From 5013890ed5c2f14a04cf4d0b5bdc18062dbed9f4 Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Tue, 21 Jun 2016 21:51:20 -0700 Subject: [PATCH 1/2] Apply new mechanism to PythonInterpreter --- .../zeppelin/python/PythonInterpreter.java | 17 ++--------- .../apache/zeppelin/python/PythonProcess.java | 2 +- .../main/resources/interpreter-setting.json | 15 ++++++++++ .../python/PythonInterpreterTest.java | 29 +++++++++---------- 4 files changed, 32 insertions(+), 31 deletions(-) create mode 100644 python/src/main/resources/interpreter-setting.json diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java index b1da98192d4..13d4a3c6cc8 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java @@ -20,7 +20,6 @@ import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; @@ -49,28 +48,16 @@ public class PythonInterpreter extends Interpreter { public static final String BOOTSTRAP_PY = "/bootstrap.py"; public static final String BOOTSTRAP_INPUT_PY = "/bootstrap_input.py"; public static final String ZEPPELIN_PYTHON = "zeppelin.python"; - public static final String DEFAULT_ZEPPELIN_PYTHON = "python"; private Integer port; private GatewayServer gatewayServer; private long pythonPid; + private Boolean py4J = false; private InterpreterContext context; PythonProcess process = null; - static { - Interpreter.register( - "python", - "python", - PythonInterpreter.class.getName(), - new InterpreterPropertyBuilder() - .add(ZEPPELIN_PYTHON, DEFAULT_ZEPPELIN_PYTHON, - "Python directory. Default : python (assume python is in your $PATH)") - .build() - ); - } - public PythonInterpreter(Properties property) { super(property); } @@ -223,7 +210,7 @@ public GUI getGui() { return context.getGui(); } - public Integer getPy4JPort() { + public Integer getPy4jPort() { return port; } diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java b/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java index 364d372f366..348ced68a45 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java @@ -92,7 +92,7 @@ public String sendAndGetResult(String cmd) throws IOException { String output = ""; String line; while (!(line = reader.readLine()).contains("*!?flush reader!?*")) { - logger.debug("Readed line from python shell : " + line); + logger.debug("Read line from python shell : " + line); if (line.equals("...")) { logger.warn("Syntax error ! "); output += "Syntax error ! "; diff --git a/python/src/main/resources/interpreter-setting.json b/python/src/main/resources/interpreter-setting.json new file mode 100644 index 00000000000..bc3dec58a3c --- /dev/null +++ b/python/src/main/resources/interpreter-setting.json @@ -0,0 +1,15 @@ +[ + { + "group": "python", + "name": "python", + "className": "org.apache.zeppelin.python.PythonInterpreter", + "properties": { + "zeppelin.python": { + "envName": null, + "propertyName": "zeppelin.python", + "defaultValue": "python", + "description": "Python directory. It is set to python by default.(assume python is in your $PATH)" + } + } + } +] diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java index 294490309bd..0e025a751df 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java @@ -60,6 +60,12 @@ public class PythonInterpreterTest { PythonProcess mockPythonProcess; String cmdHistory; + public static Properties getPythonTestProperties() { + Properties p = new Properties(); + p.setProperty(ZEPPELIN_PYTHON, DEFAULT_ZEPPELIN_PYTHON); + return p; + } + @Before public void beforeTest() { cmdHistory = ""; @@ -79,20 +85,15 @@ public String answer(InvocationOnMock invocationOnMock) throws Throwable { logger.error("Can't initiate python process", e); } - Properties properties = new Properties(); - properties.put(ZEPPELIN_PYTHON, DEFAULT_ZEPPELIN_PYTHON); - pythonInterpreter = spy(new PythonInterpreter(properties)); + pythonInterpreter = spy(new PythonInterpreter(getPythonTestProperties())); when(pythonInterpreter.getPythonProcess()).thenReturn(mockPythonProcess); - try { when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("ImportError"); } catch (IOException e) { e.printStackTrace(); } - - } @Test @@ -111,7 +112,7 @@ public void testPy4jIsNotInstalled() { py4j JavaGateway is not running */ pythonInterpreter.open(); - assertNull(pythonInterpreter.getPy4JPort()); + assertNull(pythonInterpreter.getPy4jPort()); assertTrue(cmdHistory.contains("def help()")); assertTrue(cmdHistory.contains("class PyZeppelinContext():")); @@ -122,8 +123,7 @@ public void testPy4jIsNotInstalled() { } @Test - public void testPy4JInstalled() { - + public void testPy4jInstalled() { /* If Py4J installed, bootstrap_input.py @@ -137,7 +137,7 @@ public void testPy4JInstalled() { e.printStackTrace(); } pythonInterpreter.open(); - Integer py4jPort = pythonInterpreter.getPy4JPort(); + Integer py4jPort = pythonInterpreter.getPy4jPort(); assertNotNull(py4jPort); assertTrue(cmdHistory.contains("def help()")); @@ -147,8 +147,7 @@ public void testPy4JInstalled() { assertTrue(cmdHistory.contains("GatewayClient(port=" + py4jPort + ")")); assertTrue(cmdHistory.contains("org.apache.zeppelin.display.Input")); - - assertTrue(checkSocketAdress(py4jPort)); + assertTrue(checkSocketAddress(py4jPort)); } @@ -162,12 +161,12 @@ public void testClose() { e.printStackTrace(); } pythonInterpreter.open(); - Integer py4jPort = pythonInterpreter.getPy4JPort(); + Integer py4jPort = pythonInterpreter.getPy4jPort(); assertNotNull(py4jPort); pythonInterpreter.close(); - assertFalse(checkSocketAdress(py4jPort)); + assertFalse(checkSocketAddress(py4jPort)); try { verify(mockPythonProcess, times(1)).close(); } catch (IOException e) { @@ -189,7 +188,7 @@ public void testInterpret() { - private boolean checkSocketAdress(Integer py4jPort) { + private boolean checkSocketAddress(Integer py4jPort) { Socket s = new Socket(); SocketAddress sa = new InetSocketAddress("localhost", py4jPort); Boolean working = null; From 66b8f738f161e18ee20cb8df0c380b06bfd1db5a Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Wed, 22 Jun 2016 15:41:16 -0700 Subject: [PATCH 2/2] Add zeppelin.python.maxResult property to python interpreter --- docs/interpreter/python.md | 7 ++++++- .../org/apache/zeppelin/python/PythonInterpreter.java | 9 +++++++-- python/src/main/resources/bootstrap.py | 5 ++--- python/src/main/resources/interpreter-setting.json | 6 ++++++ .../apache/zeppelin/python/PythonInterpreterTest.java | 5 ++--- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/docs/interpreter/python.md b/docs/interpreter/python.md index 0476c680b7f..34134a1d731 100644 --- a/docs/interpreter/python.md +++ b/docs/interpreter/python.md @@ -16,12 +16,17 @@ group: manual Description - python + zeppelin.python python Path of the already installed Python binary (could be python2 or python3). If python is not in your $PATH you can set the absolute directory (example : /usr/bin/python) + + zeppelin.python.maxResult + 1000 + Max number of dataframe rows to display. + ## Enabling Python Interpreter diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java index 13d4a3c6cc8..9dd4ed78b18 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java @@ -48,13 +48,14 @@ public class PythonInterpreter extends Interpreter { public static final String BOOTSTRAP_PY = "/bootstrap.py"; public static final String BOOTSTRAP_INPUT_PY = "/bootstrap_input.py"; public static final String ZEPPELIN_PYTHON = "zeppelin.python"; + public static final String DEFAULT_ZEPPELIN_PYTHON = "python"; + public static final String MAX_RESULT = "zeppelin.python.maxResult"; private Integer port; private GatewayServer gatewayServer; - private long pythonPid; - private Boolean py4J = false; private InterpreterContext context; + private int maxResult; PythonProcess process = null; @@ -67,6 +68,7 @@ public void open() { logger.info("Starting Python interpreter ....."); logger.info("Python path is set to:" + property.getProperty(ZEPPELIN_PYTHON)); + maxResult = Integer.valueOf(getProperty(MAX_RESULT)); process = getPythonProcess(); try { @@ -234,4 +236,7 @@ private int findRandomOpenPortOnAllLocalInterfaces() { return port; } + public int getMaxResult() { + return maxResult; + } } diff --git a/python/src/main/resources/bootstrap.py b/python/src/main/resources/bootstrap.py index 4f0dc5e09f4..04a5f537c16 100644 --- a/python/src/main/resources/bootstrap.py +++ b/python/src/main/resources/bootstrap.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# PYTHON 2 / 3 comptability : +# PYTHON 2 / 3 compatibility : # bootstrap.py must be runnable with Python 2 or 3 # Remove interactive mode displayhook @@ -36,7 +36,7 @@ def intHandler(signum, frame): # Set the signal handler def help(): print ('%html') print ('

Python Interpreter help

') - print ('

Python 2 & 3 comptability

') + print ('

Python 2 & 3 compatibility

') print ('

The interpreter is compatible with Python 2 & 3.
') print ('To change Python version, ') print ('change in the interpreter configuration the python to the ') @@ -100,4 +100,3 @@ def checkbox(self, name, options, defaultChecked=[]): print (self.errorMsg) z = PyZeppelinContext("") - diff --git a/python/src/main/resources/interpreter-setting.json b/python/src/main/resources/interpreter-setting.json index bc3dec58a3c..8508bd09d6c 100644 --- a/python/src/main/resources/interpreter-setting.json +++ b/python/src/main/resources/interpreter-setting.json @@ -9,6 +9,12 @@ "propertyName": "zeppelin.python", "defaultValue": "python", "description": "Python directory. It is set to python by default.(assume python is in your $PATH)" + }, + "zeppelin.python.maxResult": { + "envName": null, + "propertyName": "zeppelin.python.maxResult", + "defaultValue": "1000", + "description": "Max number of dataframe rows to display." } } } diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java index 0e025a751df..dbd13462966 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.python; +import static org.apache.zeppelin.python.PythonInterpreter.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -53,9 +54,6 @@ public class PythonInterpreterTest { Logger logger = LoggerFactory.getLogger(PythonProcess.class); - public static final String ZEPPELIN_PYTHON = "zeppelin.python"; - public static final String DEFAULT_ZEPPELIN_PYTHON = "python"; - PythonInterpreter pythonInterpreter = null; PythonProcess mockPythonProcess; String cmdHistory; @@ -63,6 +61,7 @@ public class PythonInterpreterTest { public static Properties getPythonTestProperties() { Properties p = new Properties(); p.setProperty(ZEPPELIN_PYTHON, DEFAULT_ZEPPELIN_PYTHON); + p.setProperty(MAX_RESULT, "1000"); return p; }