From 6480d1d9dfae89fde357f79501adc19d3a502910 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 24 Aug 2016 12:40:11 +0900 Subject: [PATCH 01/34] resolved conflicts --- .../interpreter/InterpreterFactory.java | 26 +++++++------- .../org/apache/zeppelin/notebook/Note.java | 2 +- .../apache/zeppelin/notebook/Paragraph.java | 6 +++- .../interpreter/InterpreterFactoryTest.java | 4 +-- .../notebook/NoteInterpreterLoaderTest.java | 34 +++++++++---------- .../apache/zeppelin/notebook/NoteTest.java | 4 +-- .../zeppelin/notebook/NotebookTest.java | 4 +-- .../zeppelin/notebook/ParagraphTest.java | 26 ++++++++++++++ 8 files changed, 69 insertions(+), 37 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 5d9f301cae8..39b843c970e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -1132,7 +1132,7 @@ public void closeNote(String noteId) { } } - private String getInterpreterInstanceKey(String noteId, InterpreterSetting setting) { + private String getInterpreterInstanceKey(String user, String noteId, InterpreterSetting setting) { if (setting.getOption().isExistingProcess()) { return Constants.EXISTING_PROCESS; } else if (setting.getOption().isPerNoteSession() || setting.getOption().isPerNoteProcess()) { @@ -1142,14 +1142,15 @@ private String getInterpreterInstanceKey(String noteId, InterpreterSetting setti } } - private List createOrGetInterpreterList(String noteId, InterpreterSetting setting) { + private List createOrGetInterpreterList(String user, String noteId, + InterpreterSetting setting) { InterpreterGroup interpreterGroup = setting.getInterpreterGroup(noteId); synchronized (interpreterGroup) { - String key = getInterpreterInstanceKey(noteId, setting); + String key = getInterpreterInstanceKey(user, noteId, setting); if (!interpreterGroup.containsKey(key)) { createInterpretersForNote(setting, noteId, key); } - return interpreterGroup.get(getInterpreterInstanceKey(noteId, setting)); + return interpreterGroup.get(getInterpreterInstanceKey(user, noteId, setting)); } } @@ -1190,14 +1191,15 @@ private String getInterpreterClassFromInterpreterSetting(InterpreterSetting sett return null; } - private Interpreter getInterpreter(String noteId, InterpreterSetting setting, String name) { + private Interpreter getInterpreter(String user, String noteId, InterpreterSetting setting, + String name) { Preconditions.checkNotNull(noteId, "noteId should be not null"); Preconditions.checkNotNull(setting, "setting should be not null"); Preconditions.checkNotNull(name, "name should be not null"); String className; if (null != (className = getInterpreterClassFromInterpreterSetting(setting, name))) { - List interpreterGroup = createOrGetInterpreterList(noteId, setting); + List interpreterGroup = createOrGetInterpreterList(user, noteId, setting); for (Interpreter interpreter : interpreterGroup) { if (className.equals(interpreter.getClassName())) { return interpreter; @@ -1207,7 +1209,7 @@ private Interpreter getInterpreter(String noteId, InterpreterSetting setting, St return null; } - public Interpreter getInterpreter(String noteId, String replName) { + public Interpreter getInterpreter(String user, String noteId, String replName) { List settings = getInterpreterSettings(noteId); InterpreterSetting setting; Interpreter interpreter; @@ -1220,7 +1222,7 @@ public Interpreter getInterpreter(String noteId, String replName) { // get default settings (first available) // TODO(jl): Fix it in case of returning null InterpreterSetting defaultSettings = getDefaultInterpreterSetting(settings); - return createOrGetInterpreterList(noteId, defaultSettings).get(0); + return createOrGetInterpreterList(user, noteId, defaultSettings).get(0); } String[] replNameSplit = replName.split("\\."); @@ -1233,7 +1235,7 @@ public Interpreter getInterpreter(String noteId, String replName) { setting = getInterpreterSettingByGroup(settings, group); if (null != setting) { - interpreter = getInterpreter(noteId, setting, name); + interpreter = getInterpreter(user, noteId, setting, name); if (null != interpreter) { return interpreter; @@ -1248,7 +1250,7 @@ public Interpreter getInterpreter(String noteId, String replName) { // TODO(jl): Handle with noteId to support defaultInterpreter per note. setting = getDefaultInterpreterSetting(settings); - interpreter = getInterpreter(noteId, setting, replName); + interpreter = getInterpreter(user, noteId, setting, replName); if (null != interpreter) { return interpreter; @@ -1259,7 +1261,7 @@ public Interpreter getInterpreter(String noteId, String replName) { setting = getInterpreterSettingByGroup(settings, replName); if (null != setting) { - List interpreters = createOrGetInterpreterList(noteId, setting); + List interpreters = createOrGetInterpreterList(user, noteId, setting); if (null != interpreters) { return interpreters.get(0); } @@ -1268,7 +1270,7 @@ public Interpreter getInterpreter(String noteId, String replName) { // Support the legacy way to use it for (InterpreterSetting s : settings) { if (s.getGroup().equals(replName)) { - List interpreters = createOrGetInterpreterList(noteId, s); + List interpreters = createOrGetInterpreterList(user, noteId, s); if (null != interpreters) { return interpreters.get(0); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 1281e71589d..95088b92a11 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -468,7 +468,7 @@ public void runAll() { AuthenticationInfo authenticationInfo = new AuthenticationInfo(); authenticationInfo.setUser(cronExecutingUser); p.setAuthenticationInfo(authenticationInfo); - run(p.getId()); + run(cronExecutingUser, p.getId()); } } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 7807abba416..335ce2c2fd2 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -105,6 +105,10 @@ private static String generateId() { + new Random(System.currentTimeMillis()).nextInt(); } + public String getUser() { + return user; + } + public String getText() { return text; } @@ -193,7 +197,7 @@ public static String getScriptBody(String text) { } public Interpreter getRepl(String name) { - return factory.getInterpreter(note.getId(), name); + return factory.getInterpreter(user, note.getId(), name); } public Interpreter getCurrentRepl() { diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 09031a59351..71d18261334 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -210,8 +210,8 @@ public void testInterpreterAliases() throws IOException, RepositoryException { add(setting2.getId()); }}); - assertEquals("className1", factory.getInterpreter("note", "test-group1").getClassName()); - assertEquals("className1", factory.getInterpreter("note", "group1").getClassName()); + assertEquals("className1", factory.getInterpreter("user1", "note", "test-group1").getClassName()); + assertEquals("className1", factory.getInterpreter("user1", "note", "group1").getClassName()); } @Test diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java index 245089963ea..9cc03e435d0 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java @@ -74,21 +74,21 @@ public void testGetInterpreter() throws IOException { factory.setInterpreters("note", factory.getDefaultInterpreterSettingList()); // when there're no interpreter selection directive - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", null).getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "").getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", " ").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", null).getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", " ").getClassName()); // when group name is omitted - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("note", "mock11").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "mock11").getClassName()); // when 'name' is ommitted - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "group1").getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("note", "group2").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "group1").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2").getClassName()); // when nothing is ommitted - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "group1.mock1").getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("note", "group1.mock11").getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("note", "group2.mock2").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "group1.mock1").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "group1.mock11").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2.mock2").getClassName()); factory.closeNote("note"); } @@ -105,12 +105,12 @@ public void testNoteSession() throws IOException { assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA")); assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("shared_process").get("noteB")); - factory.getInterpreter("noteA", null).open(); - factory.getInterpreter("noteB", null).open(); + factory.getInterpreter("user", "noteA", null).open(); + factory.getInterpreter("user", "noteB", null).open(); assertTrue( - factory.getInterpreter("noteA", null).getInterpreterGroup().getId().equals( - factory.getInterpreter("noteB", null).getInterpreterGroup().getId())); + factory.getInterpreter("user", "noteA", null).getInterpreterGroup().getId().equals( + factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId())); // interpreters are created after accessing it assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA")); @@ -138,13 +138,13 @@ public void testNotePerInterpreterProcess() throws IOException { assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA")); assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("noteB").get("noteB")); - factory.getInterpreter("noteA", null).open(); - factory.getInterpreter("noteB", null).open(); + factory.getInterpreter("user", "noteA", null).open(); + factory.getInterpreter("user", "noteB", null).open(); // per note interpreter process assertFalse( - factory.getInterpreter("noteA", null).getInterpreterGroup().getId().equals( - factory.getInterpreter("noteB", null).getInterpreterGroup().getId())); + factory.getInterpreter("user", "noteA", null).getInterpreterGroup().getId().equals( + factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId())); // interpreters are created after accessing it assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA")); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java index a44bfad478a..1be0774c5e9 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java @@ -60,7 +60,7 @@ public class NoteTest { @Test public void runNormalTest() { - when(interpreterFactory.getInterpreter(anyString(), eq("spark"))).thenReturn(interpreter); + when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("spark"))).thenReturn(interpreter); when(interpreter.getScheduler()).thenReturn(scheduler); String pText = "%spark sc.version"; @@ -72,7 +72,7 @@ public void runNormalTest() { ArgumentCaptor pCaptor = ArgumentCaptor.forClass(Paragraph.class); verify(scheduler, only()).submit(pCaptor.capture()); - verify(interpreterFactory, only()).getInterpreter(anyString(), eq("spark")); + verify(interpreterFactory, only()).getInterpreter(anyString(), anyString(), eq("spark")); assertEquals("Paragraph text", pText, pCaptor.getValue().getText()); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 86d6c06f55f..39eab5c28df 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -341,11 +341,11 @@ public void testAutoRestartInterpreterAfterSchedule() throws InterruptedExceptio MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter) - ((LazyOpenInterpreter) factory.getInterpreter(note.getId(), "mock1")).getInnerInterpreter()) + ((LazyOpenInterpreter) factory.getInterpreter("user", note.getId(), "mock1")).getInnerInterpreter()) .getInnerInterpreter())); MockInterpreter2 mock2 = ((MockInterpreter2) (((ClassloaderInterpreter) - ((LazyOpenInterpreter) factory.getInterpreter(note.getId(), "mock2")).getInnerInterpreter()) + ((LazyOpenInterpreter) factory.getInterpreter("user", note.getId(), "mock2")).getInnerInterpreter()) .getInnerInterpreter())); // wait until interpreters are started diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java index 668914ae38a..da7752c7192 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java @@ -73,6 +73,32 @@ public void replNameEndsWithWhitespace() { assertEquals("md", Paragraph.getRequiredReplName(text)); } + @Test + public void effectiveTextTest() { + InterpreterFactory interpreterFactory = mock(InterpreterFactory.class); + Interpreter interpreter = mock(Interpreter.class); + Note note = mock(Note.class); + + Paragraph p = new Paragraph("paragraph", note, null, interpreterFactory); + p.setText("%h2 show databases"); + p.setEffectiveText("%jdbc(h2) show databases"); + assertEquals("Get right replName", "jdbc", p.getRequiredReplName()); + assertEquals("Get right scriptBody", "(h2) show databases", p.getScriptBody()); + + when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("jdbc"))).thenReturn(interpreter); + when(interpreter.getFormType()).thenReturn(Interpreter.FormType.NATIVE); + when(note.getId()).thenReturn("noteId"); + + try { + p.jobRun(); + } catch (Throwable throwable) { + // Do nothing + } + + assertEquals("Erase effective Text", "h2", p.getRequiredReplName()); + assertEquals("Erase effective Text", "show databases", p.getScriptBody()); + } + @Test public void should_extract_variable_from_angular_object_registry() throws Exception { //Given From 94dfed27c430992d6aa21f2a2128ae6f5a81950a Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Fri, 22 Jul 2016 19:29:07 +0900 Subject: [PATCH 02/34] WIP --- .../apache/zeppelin/rest/NotebookRestApi.java | 5 ++++ .../zeppelin/socket/NotebookServer.java | 13 +++------ .../src/main/resources/log4j.properties | 25 ++++++++++++++++ zeppelin-server/src/main/resources/shiro.ini | 13 ++++++--- .../zeppelin/conf/ZeppelinConfiguration.java | 2 +- .../interpreter/InterpreterFactory.java | 10 +++++-- .../interpreter/InterpreterSetting.java | 29 +++++++++++++------ .../org/apache/zeppelin/notebook/Note.java | 4 +-- 8 files changed, 73 insertions(+), 28 deletions(-) create mode 100644 zeppelin-server/src/main/resources/log4j.properties diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index b83a8891d0b..5a1d5d9c3f9 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -599,6 +599,11 @@ public Response runParagraph(@PathParam("notebookId") String notebookId, // handle params if presented handleParagraphParams(message, note, paragraph); + AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); + + paragraph.setAuthenticationInfo(subject); + note.persist(subject); + note.run(paragraph.getId()); return new JsonResponse<>(Status.OK).build(); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 2eee99e1e4f..c571afc7af3 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -134,7 +134,7 @@ public void onMessage(NotebookSocket conn, String msg) { if (LOG.isTraceEnabled()) { LOG.trace("RECEIVE MSG = " + messagereceived); } - + String ticket = TicketContainer.instance.getTicket(messagereceived.principal); if (ticket != null && !ticket.equals(messagereceived.ticket)){ /* not to pollute logs, log instead of exception */ @@ -1181,14 +1181,9 @@ private void runParagraph(NotebookSocket conn, HashSet userAndRoles, Not String text = (String) fromMessage.get("paragraph"); p.setText(text); p.setTitle((String) fromMessage.get("title")); - if (!fromMessage.principal.equals("anonymous")) { - AuthenticationInfo authenticationInfo = new AuthenticationInfo(fromMessage.principal, - fromMessage.ticket); - p.setAuthenticationInfo(authenticationInfo); - - } else { - p.setAuthenticationInfo(new AuthenticationInfo()); - } + AuthenticationInfo authenticationInfo = + new AuthenticationInfo(fromMessage.principal, fromMessage.ticket); + p.setAuthenticationInfo(authenticationInfo); Map params = (Map) fromMessage .get("params"); diff --git a/zeppelin-server/src/main/resources/log4j.properties b/zeppelin-server/src/main/resources/log4j.properties new file mode 100644 index 00000000000..2f644074c89 --- /dev/null +++ b/zeppelin-server/src/main/resources/log4j.properties @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger = INFO, stdout + +log4j.appender.stdout = org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout = org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n + +log4j.additivity.org.apache.zeppelin.interpreter = false +log4j.logger.org.apache.zeppelin.interpreter = DEBUG, stdout diff --git a/zeppelin-server/src/main/resources/shiro.ini b/zeppelin-server/src/main/resources/shiro.ini index 371a44e11e1..d4f6cbb4bf0 100644 --- a/zeppelin-server/src/main/resources/shiro.ini +++ b/zeppelin-server/src/main/resources/shiro.ini @@ -18,14 +18,19 @@ [users] # List of users with their password allowed to access Zeppelin. # To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections -admin = password +admin = password, admin +user1 = user1, role1 [urls] - # anon means the access is anonymous. # authcBasic means Basic Auth Security # To enfore security, comment the line below and uncomment the next one -/** = anon -#/** = authcBasic +#/** = anon +/** = authc +[roles] +role1 = * +role2 = * +role3 = * +admin = * diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 414aed2a5bb..6dea3998bd3 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -573,7 +573,7 @@ public static enum ConfVars { // Allows a way to specify a ',' separated list of allowed origins for rest and websockets // i.e. http://localhost:8080 ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"), - ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true), + ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", false), ZEPPELIN_CREDENTIALS_PERSIST("zeppelin.credentials.persist", true), ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000"); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 39b843c970e..d8d9aad1797 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -1133,13 +1133,17 @@ public void closeNote(String noteId) { } private String getInterpreterInstanceKey(String user, String noteId, InterpreterSetting setting) { + String key; if (setting.getOption().isExistingProcess()) { - return Constants.EXISTING_PROCESS; + key = user + ":" + Constants.EXISTING_PROCESS; } else if (setting.getOption().isPerNoteSession() || setting.getOption().isPerNoteProcess()) { - return noteId; + key = user + ":" + noteId; } else { - return SHARED_SESSION; + key = user + ":" + SHARED_SESSION; } + + logger.debug("Interpreter instance key: {}", key); + return key; } private List createOrGetInterpreterList(String user, String noteId, diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 65f60cd1a6e..bb8c089c7de 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -26,6 +26,8 @@ import java.util.Properties; import com.google.gson.annotations.SerializedName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.zeppelin.dep.Dependency; @@ -35,6 +37,7 @@ * Interpreter settings */ public class InterpreterSetting { + private static final Logger logger = LoggerFactory.getLogger(InterpreterSetting.class); private static final String SHARED_PROCESS = "shared_process"; private String id; private String name; @@ -96,18 +99,22 @@ String getGroup() { return group; } - private String getInterpreterProcessKey(String noteId) { + private String getInterpreterProcessKey(String user, String noteId) { + String key; if (getOption().isExistingProcess) { - return Constants.EXISTING_PROCESS; + key = user + ":" + Constants.EXISTING_PROCESS; } else if (getOption().isPerNoteProcess()) { - return noteId; + key = user + ":" + noteId; } else { - return SHARED_PROCESS; + key = user + ":" + SHARED_PROCESS; } + + logger.debug("getInterpreterProcessKey: {}", key); + return key; } - public InterpreterGroup getInterpreterGroup(String noteId) { - String key = getInterpreterProcessKey(noteId); + public InterpreterGroup getInterpreterGroup(String user, String noteId) { + String key = getInterpreterProcessKey(user, noteId); synchronized (interpreterGroupRef) { if (!interpreterGroupRef.containsKey(key)) { String interpreterGroupId = getId() + ":" + key; @@ -126,10 +133,14 @@ public Collection getAllInterpreterGroups() { } void closeAndRemoveInterpreterGroup(String noteId) { - String key = getInterpreterProcessKey(noteId); - InterpreterGroup groupToRemove; + String key = getInterpreterProcessKey("", noteId); + InterpreterGroup groupToRemove = null; synchronized (interpreterGroupRef) { - groupToRemove = interpreterGroupRef.remove(key); + for(String intpKey: interpreterGroupRef.keySet()) { + if(intpKey.contains(key)) { + groupToRemove = interpreterGroupRef.remove(intpKey); + } + } } if (groupToRemove != null) { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 95088b92a11..7e53a00cb03 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -350,8 +350,8 @@ public void moveParagraph(String paragraphId, int index, boolean throwWhenIndexI if (index < 0 || index >= paragraphs.size()) { if (throwWhenIndexIsOutOfBound) { - throw new IndexOutOfBoundsException( - "paragraph size is " + paragraphs.size() + " , index is " + index); + throw new IndexOutOfBoundsException("paragraph size is " + paragraphs.size() + + " , index is " + index); } else { return; } From ccbedc17b44bb921d83f6549af7a6740c470d810 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 28 Jul 2016 22:21:23 +0900 Subject: [PATCH 03/34] WIP --- .../apache/zeppelin/rest/NotebookRestApi.java | 4 +- .../zeppelin/socket/NotebookServer.java | 20 +-- .../zeppelin/socket/NotebookServerTest.java | 2 +- .../interpreter/InterpreterFactory.java | 41 ++--- .../interpreter/InterpreterSetting.java | 53 ++++--- .../org/apache/zeppelin/notebook/Note.java | 22 ++- .../apache/zeppelin/notebook/Notebook.java | 22 ++- .../apache/zeppelin/notebook/Paragraph.java | 4 +- .../helium/HeliumApplicationFactoryTest.java | 14 +- .../interpreter/InterpreterFactoryTest.java | 32 +++- .../notebook/NoteInterpreterLoaderTest.java | 44 +++--- .../zeppelin/notebook/NotebookTest.java | 144 +++++++++--------- .../notebook/repo/VFSNotebookRepoTest.java | 2 +- 13 files changed, 230 insertions(+), 174 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 5a1d5d9c3f9..d9af812f164 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -177,7 +177,7 @@ public Response putNotePermissions(@PathParam("noteId") String noteId, String re public Response bind(@PathParam("noteId") String noteId, String req) throws IOException { List settingIdList = gson.fromJson(req, new TypeToken>() { }.getType()); - notebook.bindInterpretersToNote(noteId, settingIdList); + notebook.bindInterpretersToNote(SecurityUtils.getPrincipal(), noteId, settingIdList); return new JsonResponse<>(Status.OK).build(); } @@ -458,7 +458,7 @@ public Response deleteParagraph(@PathParam("notebookId") String notebookId, } AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); - note.removeParagraph(paragraphId); + note.removeParagraph(SecurityUtils.getPrincipal(), paragraphId); note.persist(subject); notebookServer.broadcastNote(note); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index c571afc7af3..aaace7b4bb9 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -813,7 +813,7 @@ private void removeParagraph(NotebookSocket conn, HashSet userAndRoles, /** We dont want to remove the last paragraph */ if (!note.isLastParagraph(paragraphId)) { - note.removeParagraph(paragraphId); + note.removeParagraph(subject.getUser(), paragraphId); note.persist(subject); broadcastNote(note); } @@ -869,6 +869,7 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo String interpreterGroupId = (String) fromMessage.get("interpreterGroupId"); String varName = (String) fromMessage.get("name"); Object varValue = fromMessage.get("value"); + String user = SecurityUtils.getPrincipal(); AngularObject ao = null; boolean global = false; // propagate change to (Remote) AngularObjectRegistry @@ -877,12 +878,12 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo List settings = notebook.getInterpreterFactory() .getInterpreterSettings(note.getId()); for (InterpreterSetting setting : settings) { - if (setting.getInterpreterGroup(note.getId()) == null) { + if (setting.getInterpreterGroup(user, note.getId()) == null) { continue; } - if (interpreterGroupId.equals(setting.getInterpreterGroup(note.getId()).getId())) { + if (interpreterGroupId.equals(setting.getInterpreterGroup(user, note.getId()).getId())) { AngularObjectRegistry angularObjectRegistry = setting - .getInterpreterGroup(note.getId()).getAngularObjectRegistry(); + .getInterpreterGroup(user, note.getId()).getAngularObjectRegistry(); // first trying to get local registry ao = angularObjectRegistry.get(varName, noteId, paragraphId); @@ -919,12 +920,12 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo List settings = notebook.getInterpreterFactory() .getInterpreterSettings(note.getId()); for (InterpreterSetting setting : settings) { - if (setting.getInterpreterGroup(n.getId()) == null) { + if (setting.getInterpreterGroup(user, n.getId()) == null) { continue; } - if (interpreterGroupId.equals(setting.getInterpreterGroup(n.getId()).getId())) { + if (interpreterGroupId.equals(setting.getInterpreterGroup(user, n.getId()).getId())) { AngularObjectRegistry angularObjectRegistry = setting - .getInterpreterGroup(n.getId()).getAngularObjectRegistry(); + .getInterpreterGroup(user, n.getId()).getAngularObjectRegistry(); this.broadcastExcept( n.getId(), new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao) @@ -1565,6 +1566,7 @@ public NotebookEventListener getNotebookInformationListener() { } private void sendAllAngularObjects(Note note, NotebookSocket conn) throws IOException { + String user = SecurityUtils.getPrincipal(); List settings = notebook().getInterpreterFactory().getInterpreterSettings(note.getId()); if (settings == null || settings.size() == 0) { @@ -1572,14 +1574,14 @@ private void sendAllAngularObjects(Note note, NotebookSocket conn) throws IOExce } for (InterpreterSetting intpSetting : settings) { - AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.getId()) + AngularObjectRegistry registry = intpSetting.getInterpreterGroup(user, note.getId()) .getAngularObjectRegistry(); List objects = registry.getAllWithGlobal(note.getId()); for (AngularObject object : objects) { conn.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE) .put("angularObject", object) .put("interpreterGroupId", - intpSetting.getInterpreterGroup(note.getId()).getId()) + intpSetting.getInterpreterGroup(user, note.getId()).getId()) .put("noteId", note.getId()) .put("paragraphId", object.getParagraphId()) )); diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java index 01a24e2e1de..9f39df6c80d 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java @@ -101,7 +101,7 @@ public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() thr List settings = notebook.getInterpreterFactory().getInterpreterSettings(note1.getId()); for (InterpreterSetting setting : settings) { if (setting.getName().equals("md")) { - interpreterGroup = setting.getInterpreterGroup("sharedProcess"); + interpreterGroup = setting.getInterpreterGroup("anonymous", "sharedProcess"); break; } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index d8d9aad1797..12ed4c302bb 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -698,16 +698,17 @@ public InterpreterGroup createInterpreterGroup(String id, InterpreterOption opti return interpreterGroup; } - public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String noteId) { + public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user, + String noteId) { if (interpreterSetting.getOption().isPerNoteProcess()) { interpreterSetting.closeAndRemoveInterpreterGroup(noteId); } else if (interpreterSetting.getOption().isPerNoteSession()) { - InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(noteId); - - interpreterGroup.close(noteId); - interpreterGroup.destroy(noteId); + InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId); + String key = getInterpreterInstanceKey(user, noteId, interpreterSetting); + interpreterGroup.close(key); + interpreterGroup.destroy(key); synchronized (interpreterGroup) { - interpreterGroup.remove(noteId); + interpreterGroup.remove(key); interpreterGroup.notifyAll(); // notify createInterpreterForNote() } logger.info("Interpreter instance {} for note {} is removed", interpreterSetting.getName(), @@ -715,9 +716,9 @@ public void removeInterpretersForNote(InterpreterSetting interpreterSetting, Str } } - public void createInterpretersForNote(InterpreterSetting interpreterSetting, String noteId, - String key) { - InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(noteId); + public void createInterpretersForNote(InterpreterSetting interpreterSetting, String user, + String noteId, String key) { + InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId); InterpreterOption option = interpreterSetting.getOption(); Properties properties = interpreterSetting.getProperties(); if (option.isExistingProcess) { @@ -861,8 +862,8 @@ public InterpreterSetting get(String name) { } } - private void putNoteInterpreterSettingBinding(String noteId, List settingList) - throws IOException { + private void putNoteInterpreterSettingBinding(String user, String noteId, + List settingList) throws IOException { List unBindedSettings = new LinkedList<>(); synchronized (interpreterSettings) { @@ -879,18 +880,18 @@ private void putNoteInterpreterSettingBinding(String noteId, List settin for (String settingId : unBindedSettings) { InterpreterSetting setting = get(settingId); - removeInterpretersForNote(setting, noteId); + removeInterpretersForNote(setting, user, noteId); } } } - public void removeNoteInterpreterSettingBinding(String noteId) { + public void removeNoteInterpreterSettingBinding(String user, String noteId) { synchronized (interpreterSettings) { List settingIds = (interpreterBindings.containsKey(noteId) ? interpreterBindings.remove(noteId) : Collections.emptyList()); for (String settingId : settingIds) { - this.removeInterpretersForNote(get(settingId), noteId); + this.removeInterpretersForNote(get(settingId), user, noteId); } } } @@ -1093,8 +1094,8 @@ private Interpreter createRemoteRepl(String interpreterPath, String noteId, Stri * @param ids InterpreterSetting id list * @throws IOException */ - public void setInterpreters(String noteId, List ids) throws IOException { - putNoteInterpreterSettingBinding(noteId, ids); + public void setInterpreters(String user, String noteId, List ids) throws IOException { + putNoteInterpreterSettingBinding(user, noteId, ids); } public List getInterpreters(String noteId) { @@ -1119,7 +1120,7 @@ public List getInterpreterSettings(String noteId) { return settings; } - public void closeNote(String noteId) { + public void closeNote(String user, String noteId) { // close interpreters in this note session List settings = getInterpreterSettings(noteId); if (settings == null || settings.size() == 0) { @@ -1128,7 +1129,7 @@ public void closeNote(String noteId) { logger.info("closeNote: {}", noteId); for (InterpreterSetting setting : settings) { - removeInterpretersForNote(setting, noteId); + removeInterpretersForNote(setting, user, noteId); } } @@ -1148,11 +1149,11 @@ private String getInterpreterInstanceKey(String user, String noteId, Interpreter private List createOrGetInterpreterList(String user, String noteId, InterpreterSetting setting) { - InterpreterGroup interpreterGroup = setting.getInterpreterGroup(noteId); + InterpreterGroup interpreterGroup = setting.getInterpreterGroup(user, noteId); synchronized (interpreterGroup) { String key = getInterpreterInstanceKey(user, noteId, setting); if (!interpreterGroup.containsKey(key)) { - createInterpretersForNote(setting, noteId, key); + createInterpretersForNote(setting, user, noteId, key); } return interpreterGroup.get(getInterpreterInstanceKey(user, noteId, setting)); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index bb8c089c7de..ffa144d5430 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.concurrent.locks.ReentrantReadWriteLock; import com.google.gson.annotations.SerializedName; import org.slf4j.Logger; @@ -54,13 +55,19 @@ public class InterpreterSetting { @Deprecated private transient InterpreterGroupFactory interpreterGroupFactory; - public InterpreterSetting() { + private final transient ReentrantReadWriteLock.ReadLock interpreterGroupReadLock; + private final transient ReentrantReadWriteLock.WriteLock interpreterGroupWriteLock; + public InterpreterSetting() { + ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + interpreterGroupReadLock = lock.readLock(); + interpreterGroupWriteLock = lock.writeLock(); } public InterpreterSetting(String id, String name, String group, List interpreterInfos, Properties properties, List dependencies, InterpreterOption option, String path) { + this(); this.id = id; this.name = name; this.group = group; @@ -115,31 +122,41 @@ private String getInterpreterProcessKey(String user, String noteId) { public InterpreterGroup getInterpreterGroup(String user, String noteId) { String key = getInterpreterProcessKey(user, noteId); - synchronized (interpreterGroupRef) { - if (!interpreterGroupRef.containsKey(key)) { - String interpreterGroupId = getId() + ":" + key; - InterpreterGroup intpGroup = - interpreterGroupFactory.createInterpreterGroup(interpreterGroupId, getOption()); - interpreterGroupRef.put(key, intpGroup); - } + if (!interpreterGroupRef.containsKey(key)) { + String interpreterGroupId = getId() + ":" + key; + InterpreterGroup intpGroup = + interpreterGroupFactory.createInterpreterGroup(interpreterGroupId, getOption()); + + interpreterGroupWriteLock.lock(); + interpreterGroupRef.put(key, intpGroup); + interpreterGroupWriteLock.unlock(); + } + try { + interpreterGroupReadLock.lock(); return interpreterGroupRef.get(key); + } finally { + interpreterGroupReadLock.unlock(); } } public Collection getAllInterpreterGroups() { - synchronized (interpreterGroupRef) { + try { + interpreterGroupReadLock.lock(); return new LinkedList<>(interpreterGroupRef.values()); + } finally { + interpreterGroupReadLock.unlock(); } } void closeAndRemoveInterpreterGroup(String noteId) { String key = getInterpreterProcessKey("", noteId); + InterpreterGroup groupToRemove = null; - synchronized (interpreterGroupRef) { - for(String intpKey: interpreterGroupRef.keySet()) { - if(intpKey.contains(key)) { - groupToRemove = interpreterGroupRef.remove(intpKey); - } + for (String intpKey : new HashSet<>(interpreterGroupRef.keySet())) { + if (intpKey.contains(key)) { + interpreterGroupWriteLock.lock(); + groupToRemove = interpreterGroupRef.remove(intpKey); + interpreterGroupWriteLock.unlock(); } } @@ -150,11 +167,9 @@ void closeAndRemoveInterpreterGroup(String noteId) { } void closeAndRmoveAllInterpreterGroups() { - synchronized (interpreterGroupRef) { - HashSet groupsToRemove = new HashSet<>(interpreterGroupRef.keySet()); - for (String key : groupsToRemove) { - closeAndRemoveInterpreterGroup(key); - } + HashSet groupsToRemove = new HashSet<>(interpreterGroupRef.keySet()); + for (String key : groupsToRemove) { + closeAndRemoveInterpreterGroup(key); } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 7e53a00cb03..632191b8a4b 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -30,6 +30,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import com.google.common.base.Preconditions; import com.google.gson.Gson; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -286,8 +287,8 @@ public Paragraph insertParagraph(int index) { * @param paragraphId ID of paragraph * @return a paragraph that was deleted, or null otherwise */ - public Paragraph removeParagraph(String paragraphId) { - removeAllAngularObjectInParagraph(paragraphId); + public Paragraph removeParagraph(String user, String paragraphId) { + removeAllAngularObjectInParagraph(user, paragraphId); ResourcePoolUtils.removeResourcesBelongsToParagraph(getId(), paragraphId); synchronized (paragraphs) { Iterator i = paragraphs.iterator(); @@ -460,6 +461,9 @@ private void setParagraphMagic(Paragraph p, int index) { */ public void runAll() { String cronExecutingUser = (String) getConfig().get("cronExecutingUser"); + if (null == cronExecutingUser) { + cronExecutingUser = "anonymous"; + } synchronized (paragraphs) { for (Paragraph p : paragraphs) { if (!p.isEnabled()) { @@ -482,7 +486,8 @@ public void run(String paragraphId) { Paragraph p = getParagraph(paragraphId); p.setListener(jobListenerFactory.getParagraphJobListener(this)); String requiredReplName = p.getRequiredReplName(); - Interpreter intp = factory.getInterpreter(getId(), requiredReplName); + Interpreter intp = factory.getInterpreter(p.getUser(), getId(), requiredReplName); + if (intp == null) { String intpExceptionMsg = p.getJobName() + "'s Interpreter " + requiredReplName + " not found"; @@ -526,7 +531,7 @@ public List getParagraphs() { } } - private void snapshotAngularObjectRegistry() { + private void snapshotAngularObjectRegistry(String user) { angularObjects = new HashMap<>(); List settings = factory.getInterpreterSettings(getId()); @@ -535,13 +540,13 @@ private void snapshotAngularObjectRegistry() { } for (InterpreterSetting setting : settings) { - InterpreterGroup intpGroup = setting.getInterpreterGroup(id); + InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id); AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry(); angularObjects.put(intpGroup.getId(), registry.getAllWithGlobal(id)); } } - private void removeAllAngularObjectInParagraph(String paragraphId) { + private void removeAllAngularObjectInParagraph(String user, String paragraphId) { angularObjects = new HashMap<>(); List settings = factory.getInterpreterSettings(getId()); @@ -550,7 +555,7 @@ private void removeAllAngularObjectInParagraph(String paragraphId) { } for (InterpreterSetting setting : settings) { - InterpreterGroup intpGroup = setting.getInterpreterGroup(id); + InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id); AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry(); if (registry instanceof RemoteAngularObjectRegistry) { @@ -580,8 +585,9 @@ private void removeAllAngularObjectInParagraph(String paragraphId) { } public void persist(AuthenticationInfo subject) throws IOException { + Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null"); stopDelayedPersistTimer(); - snapshotAngularObjectRegistry(); + snapshotAngularObjectRegistry(subject.getUser()); index.updateIndexDoc(this); repo.save(this, subject); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 1e65a86e21a..d996488cb61 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -31,6 +31,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; + +import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.FluentIterable; import com.google.common.collect.Sets; @@ -135,6 +137,7 @@ public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo, * @throws IOException */ public Note createNote(AuthenticationInfo subject) throws IOException { + Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null"); Note note; if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) { note = createNote(replFactory.getDefaultInterpreterSettingList(), subject); @@ -158,7 +161,7 @@ public Note createNote(List interpreterIds, AuthenticationInfo subject) notes.put(note.getId(), note); } if (interpreterIds != null) { - bindInterpretersToNote(note.getId(), interpreterIds); + bindInterpretersToNote(subject.getUser(), note.getId(), interpreterIds); } if (subject != null && !"anonymous".equals(subject.getUser())) { @@ -252,7 +255,7 @@ public Note cloneNote(String sourceNoteID, String newNoteName, AuthenticationInf } // Copy the interpreter bindings List boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.getId()); - bindInterpretersToNote(newNote.getId(), boundInterpreterSettingsIds); + bindInterpretersToNote(subject.getUser(), newNote.getId(), boundInterpreterSettingsIds); List paragraphs = sourceNote.getParagraphs(); for (Paragraph p : paragraphs) { @@ -264,7 +267,7 @@ public Note cloneNote(String sourceNoteID, String newNoteName, AuthenticationInf return newNote; } - public void bindInterpretersToNote(String id, List interpreterSettingIds) + public void bindInterpretersToNote(String user, String id, List interpreterSettingIds) throws IOException { Note note = getNote(id); if (note != null) { @@ -275,7 +278,7 @@ public void bindInterpretersToNote(String id, List interpreterSettingIds } } - replFactory.setInterpreters(note.getId(), interpreterSettingIds); + replFactory.setInterpreters(user, note.getId(), interpreterSettingIds); // comment out while note.getNoteReplLoader().setInterpreters(...) do the same // replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds); } @@ -306,18 +309,21 @@ public Note getNote(String id) { } public void removeNote(String id, AuthenticationInfo subject) { + Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null"); + Note note; synchronized (notes) { note = notes.remove(id); } - replFactory.removeNoteInterpreterSettingBinding(id); + replFactory.removeNoteInterpreterSettingBinding(subject.getUser(), id); notebookIndex.deleteIndexDocs(note); notebookAuthorization.removeNote(id); // remove from all interpreter instance's angular object registry for (InterpreterSetting settings : replFactory.get()) { - AngularObjectRegistry registry = settings.getInterpreterGroup(id).getAngularObjectRegistry(); + AngularObjectRegistry registry = + settings.getInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry(); if (registry instanceof RemoteAngularObjectRegistry) { // remove paragraph scope object for (Paragraph p : note.getParagraphs()) { @@ -437,7 +443,7 @@ private Note loadNoteFromRepo(String id, AuthenticationInfo subject) { SnapshotAngularObject snapshot = angularObjectSnapshot.get(name); List settings = replFactory.get(); for (InterpreterSetting setting : settings) { - InterpreterGroup intpGroup = setting.getInterpreterGroup(note.getId()); + InterpreterGroup intpGroup = setting.getInterpreterGroup(subject.getUser(), note.getId()); if (intpGroup.getId().equals(snapshot.getIntpGroupId())) { AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry(); String noteId = snapshot.getAngularObject().getNoteId(); @@ -533,7 +539,7 @@ public int compare(Note note1, Note note2) { return noteList; } } - + public List getAllNotes(AuthenticationInfo subject) { final Set entities = Sets.newHashSet(); if (subject != null) { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 335ce2c2fd2..9907a122956 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -446,8 +446,8 @@ private InterpreterContext getInterpreterContext(InterpreterOutput output) { if (!factory.getInterpreterSettings(note.getId()).isEmpty()) { InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0); - registry = intpGroup.getInterpreterGroup(note.getId()).getAngularObjectRegistry(); - resourcePool = intpGroup.getInterpreterGroup(note.getId()).getResourcePool(); + registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry(); + resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool(); } List runners = new LinkedList(); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 29cdf554de6..ada1e515475 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -132,7 +132,7 @@ public void testLoadRunUnloadApplication() new String[][]{}); Note note1 = notebook.createNote(null); - factory.setInterpreters(note1.getId(),factory.getDefaultInterpreterSettingList()); + factory.setInterpreters("user", note1.getId(),factory.getDefaultInterpreterSettingList()); Paragraph p1 = note1.addParagraph(); @@ -176,7 +176,7 @@ public void testUnloadOnParagraphRemove() throws IOException { new String[][]{}); Note note1 = notebook.createNote(null); - factory.setInterpreters(note1.getId(), factory.getDefaultInterpreterSettingList()); + factory.setInterpreters("user", note1.getId(), factory.getDefaultInterpreterSettingList()); Paragraph p1 = note1.addParagraph(); @@ -193,7 +193,7 @@ public void testUnloadOnParagraphRemove() throws IOException { } // when remove paragraph - note1.removeParagraph(p1.getId()); + note1.removeParagraph("user", p1.getId()); // then assertEquals(ApplicationState.Status.UNLOADED, app.getStatus()); @@ -214,7 +214,7 @@ public void testUnloadOnInterpreterUnbind() throws IOException { new String[][]{}); Note note1 = notebook.createNote(null); - notebook.bindInterpretersToNote(note1.getId(), factory.getDefaultInterpreterSettingList()); + notebook.bindInterpretersToNote("user", note1.getId(), factory.getDefaultInterpreterSettingList()); Paragraph p1 = note1.addParagraph(); @@ -231,7 +231,7 @@ public void testUnloadOnInterpreterUnbind() throws IOException { } // when unbind interpreter - notebook.bindInterpretersToNote(note1.getId(), new LinkedList()); + notebook.bindInterpretersToNote("user", note1.getId(), new LinkedList()); // then assertEquals(ApplicationState.Status.UNLOADED, app.getStatus()); @@ -255,7 +255,7 @@ public void testInterpreterUnbindOfNullReplParagraph() throws IOException { // Unbind all interpreter from note // NullPointerException shouldn't occur here - notebook.bindInterpretersToNote(note1.getId(), new LinkedList()); + notebook.bindInterpretersToNote("user", note1.getId(), new LinkedList()); // remove note notebook.removeNote(note1.getId(), null); @@ -273,7 +273,7 @@ public void testUnloadOnInterpreterRestart() throws IOException { new String[][]{}); Note note1 = notebook.createNote(null); - notebook.bindInterpretersToNote(note1.getId(), factory.getDefaultInterpreterSettingList()); + notebook.bindInterpretersToNote("user", note1.getId(), factory.getDefaultInterpreterSettingList()); String mock1IntpSettingId = null; for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) { if (setting.getName().equals("mock1")) { diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 71d18261334..3d0bb83d193 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -33,6 +33,8 @@ import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.interpreter.mock.MockInterpreter1; import org.apache.zeppelin.interpreter.mock.MockInterpreter2; +import org.apache.zeppelin.notebook.repo.zeppelinhub.security.Authentication; +import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.notebook.JobListenerFactory; import org.apache.zeppelin.notebook.Note; @@ -114,8 +116,8 @@ public void testBasic() { // mock1Setting = factory.createNewSetting("mock11", "mock1", new ArrayList(), new InterpreterOption(false), new Properties()); - InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("sharedProcess"); - factory.createInterpretersForNote(mock1Setting, "sharedProcess", "session"); + InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user", "sharedProcess"); + factory.createInterpretersForNote(mock1Setting, "user", "sharedProcess", "session"); // get interpreter assertNotNull("get Interpreter", interpreterGroup.get("session").get(0)); @@ -125,7 +127,7 @@ public void testBasic() { // restart interpreter factory.restart(mock1Setting.getId()); - assertNull(mock1Setting.getInterpreterGroup("sharedProcess").get("session")); + assertNull(mock1Setting.getInterpreterGroup("user", "sharedProcess").get("session")); } @Test @@ -205,7 +207,7 @@ public void testInterpreterAliases() throws IOException, RepositoryException { final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList(), new InterpreterOption(true), new Properties()); final InterpreterSetting setting2 = factory.createNewSetting("test-group2", "group1", new ArrayList(), new InterpreterOption(true), new Properties()); - factory.setInterpreters("note", new ArrayList() {{ + factory.setInterpreters("user", "note", new ArrayList() {{ add(setting1.getId()); add(setting2.getId()); }}); @@ -214,6 +216,28 @@ public void testInterpreterAliases() throws IOException, RepositoryException { assertEquals("className1", factory.getInterpreter("user1", "note", "group1").getClassName()); } + @Test + public void testMultiUser() throws IOException, RepositoryException { + factory = new InterpreterFactory(conf, null, null, null, depResolver); + final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true); + factory.add("group1", new ArrayList(){{ + add(info1); + }}, new ArrayList(), new InterpreterOption(true), new Properties(), "/path1"); + + final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList(), new InterpreterOption(true), new Properties()); + + factory.setInterpreters("user1", "note", new ArrayList() {{ + add(setting1.getId()); + }}); + + factory.setInterpreters("user2", "note", new ArrayList() {{ + add(setting1.getId()); + }}); + + assertNotEquals(factory.getInterpreter("user1", "note", "test-group1"), factory.getInterpreter("user2", "note", "test-group1")); + } + + @Test public void testInvalidInterpreterSettingName() { try { diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java index 9cc03e435d0..b3ea9fcdec6 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java @@ -71,7 +71,7 @@ public void tearDown() throws Exception { @Test public void testGetInterpreter() throws IOException { - factory.setInterpreters("note", factory.getDefaultInterpreterSettingList()); + factory.setInterpreters("user", "note", factory.getDefaultInterpreterSettingList()); // when there're no interpreter selection directive assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", null).getClassName()); @@ -90,20 +90,20 @@ public void testGetInterpreter() throws IOException { assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "group1.mock11").getClassName()); assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2.mock2").getClassName()); - factory.closeNote("note"); + factory.closeNote("user", "note"); } @Test public void testNoteSession() throws IOException { - factory.setInterpreters("noteA", factory.getDefaultInterpreterSettingList()); + factory.setInterpreters("user", "noteA", factory.getDefaultInterpreterSettingList()); factory.getInterpreterSettings("noteA").get(0).getOption().setPerNoteSession(true); - factory.setInterpreters("noteB", factory.getDefaultInterpreterSettingList()); + factory.setInterpreters("user", "noteB", factory.getDefaultInterpreterSettingList()); factory.getInterpreterSettings("noteB").get(0).getOption().setPerNoteSession(true); // interpreters are not created before accessing it - assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA")); - assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("shared_process").get("noteB")); + assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA")); + assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB")); factory.getInterpreter("user", "noteA", null).open(); factory.getInterpreter("user", "noteB", null).open(); @@ -113,30 +113,30 @@ public void testNoteSession() throws IOException { factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId())); // interpreters are created after accessing it - assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA")); - assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("shared_process").get("noteB")); + assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA")); + assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB")); // when - factory.closeNote("noteA"); - factory.closeNote("noteB"); + factory.closeNote("user", "noteA"); + factory.closeNote("user", "noteB"); // interpreters are destroyed after close - assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA")); - assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("shared_process").get("noteB")); + assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA")); + assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB")); } @Test public void testNotePerInterpreterProcess() throws IOException { - factory.setInterpreters("noteA", factory.getDefaultInterpreterSettingList()); + factory.setInterpreters("user", "noteA", factory.getDefaultInterpreterSettingList()); factory.getInterpreterSettings("noteA").get(0).getOption().setPerNoteProcess(true); - factory.setInterpreters("noteB", factory.getDefaultInterpreterSettingList()); + factory.setInterpreters("user", "noteB", factory.getDefaultInterpreterSettingList()); factory.getInterpreterSettings("noteB").get(0).getOption().setPerNoteProcess(true); // interpreters are not created before accessing it - assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA")); - assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("noteB").get("noteB")); + assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA")); + assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB")); factory.getInterpreter("user", "noteA", null).open(); factory.getInterpreter("user", "noteB", null).open(); @@ -147,16 +147,16 @@ public void testNotePerInterpreterProcess() throws IOException { factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId())); // interpreters are created after accessing it - assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA")); - assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("noteB").get("noteB")); + assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA")); + assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB")); // when - factory.closeNote("noteA"); - factory.closeNote("noteB"); + factory.closeNote("user", "noteA"); + factory.closeNote("user", "noteB"); // interpreters are destroyed after close - assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA")); - assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("noteB").get("noteB")); + assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA")); + assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB")); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 39eab5c28df..46478c5234b 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -65,6 +65,7 @@ public class NotebookTest implements JobListenerFactory{ private DependencyResolver depResolver; private NotebookAuthorization notebookAuthorization; private Credentials credentials; + private AuthenticationInfo anonymous = new AuthenticationInfo("anonymous"); @Before public void setUp() throws Exception { @@ -106,8 +107,8 @@ public void tearDown() throws Exception { @Test public void testSelectingReplImplementation() throws IOException { - Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + Note note = notebook.createNote(anonymous); + factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); // run with default repl Paragraph p1 = note.addParagraph(); @@ -137,7 +138,7 @@ public void testReloadAndSetInterpreter() throws IOException { FileUtils.copyDirectory(srcDir, destDir); // when load - notebook.reloadAllNotes(null); + notebook.reloadAllNotes(anonymous); assertEquals(1, notebook.getAllNotes().size()); // then interpreter factory should be injected into all the paragraphs @@ -169,8 +170,8 @@ public void testReloadAllNotes() throws IOException { assertEquals(notes.size(), 0); // load copied notebook on memory when reloadAllNotes() is called - Note copiedNote = notebookRepo.get("2A94M5J1Z", null); - notebook.reloadAllNotes(null); + Note copiedNote = notebookRepo.get("2A94M5J1Z", anonymous); + notebook.reloadAllNotes(anonymous); notes = notebook.getAllNotes(); assertEquals(notes.size(), 2); assertEquals(notes.get(1).getId(), copiedNote.getId()); @@ -188,14 +189,14 @@ public void testReloadAllNotes() throws IOException { assertEquals(notes.size(), 2); // delete notebook from notebook list when reloadAllNotes() is called - notebook.reloadAllNotes(null); + notebook.reloadAllNotes(anonymous); notes = notebook.getAllNotes(); assertEquals(notes.size(), 0); } @Test public void testPersist() throws IOException, SchedulerException, RepositoryException { - Note note = notebook.createNote(null); + Note note = notebook.createNote(anonymous); // run with default repl Paragraph p1 = note.addParagraph(); @@ -203,7 +204,7 @@ public void testPersist() throws IOException, SchedulerException, RepositoryExce config.put("enabled", true); p1.setConfig(config); p1.setText("hello world"); - note.persist(null); + note.persist(anonymous); Notebook notebook2 = new Notebook( conf, notebookRepo, schedulerFactory, @@ -228,7 +229,7 @@ public void testCreateNoteWithSubject() throws IOException, SchedulerException, @Test public void testClearParagraphOutput() throws IOException, SchedulerException{ - Note note = notebook.createNote(null); + Note note = notebook.createNote(anonymous); Paragraph p1 = note.addParagraph(); Map config = p1.getConfig(); config.put("enabled", true); @@ -247,8 +248,8 @@ public void testClearParagraphOutput() throws IOException, SchedulerException{ @Test public void testRunAll() throws IOException { - Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + Note note = notebook.createNote(anonymous); + factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList()); // p1 Paragraph p1 = note.addParagraph(); @@ -280,14 +281,14 @@ public void testRunAll() throws IOException { assertNull(p2.getResult()); assertEquals("repl1: p3", p3.getResult().message()); - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); } @Test public void testSchedule() throws InterruptedException, IOException { // create a note and a paragraph - Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + Note note = notebook.createNote(anonymous); + factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList()); Paragraph p = note.addParagraph(); Map config = new HashMap(); @@ -319,8 +320,8 @@ public void testSchedule() throws InterruptedException, IOException { @Test public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{ // create a note and a paragraph - Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + Note note = notebook.createNote(anonymous); + factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); Paragraph p = note.addParagraph(); Map config = new HashMap(); @@ -341,11 +342,11 @@ public void testAutoRestartInterpreterAfterSchedule() throws InterruptedExceptio MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter) - ((LazyOpenInterpreter) factory.getInterpreter("user", note.getId(), "mock1")).getInnerInterpreter()) + ((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock1")).getInnerInterpreter()) .getInnerInterpreter())); MockInterpreter2 mock2 = ((MockInterpreter2) (((ClassloaderInterpreter) - ((LazyOpenInterpreter) factory.getInterpreter("user", note.getId(), "mock2")).getInnerInterpreter()) + ((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock2")).getInnerInterpreter()) .getInnerInterpreter())); // wait until interpreters are started @@ -372,8 +373,8 @@ public void testAutoRestartInterpreterAfterSchedule() throws InterruptedExceptio @Test public void testExportAndImportNote() throws IOException, CloneNotSupportedException, InterruptedException, InterpreterException, SchedulerException, RepositoryException { - Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + Note note = notebook.createNote(anonymous); + factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList()); final Paragraph p = note.addParagraph(); String simpleText = "hello world"; @@ -386,7 +387,7 @@ public void testExportAndImportNote() throws IOException, CloneNotSupportedExcep String exportedNoteJson = notebook.exportNote(note.getId()); - Note importedNote = notebook.importNote(exportedNoteJson, "Title", null); + Note importedNote = notebook.importNote(exportedNoteJson, "Title", anonymous); Paragraph p2 = importedNote.getParagraphs().get(0); @@ -411,8 +412,8 @@ public void testExportAndImportNote() throws IOException, CloneNotSupportedExcep @Test public void testCloneNote() throws IOException, CloneNotSupportedException, InterruptedException, InterpreterException, SchedulerException, RepositoryException { - Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + Note note = notebook.createNote(anonymous); + factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList()); final Paragraph p = note.addParagraph(); p.setText("hello world"); @@ -420,7 +421,7 @@ public void testCloneNote() throws IOException, CloneNotSupportedException, while(p.isTerminated()==false || p.getResult()==null) Thread.yield(); p.setStatus(Status.RUNNING); - Note cloneNote = notebook.cloneNote(note.getId(), "clone note", null); + Note cloneNote = notebook.cloneNote(note.getId(), "clone note", anonymous); Paragraph cp = cloneNote.paragraphs.get(0); assertEquals(cp.getStatus(), Status.READY); @@ -457,8 +458,8 @@ public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedExcep @Test public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSupportedException, InterruptedException { - Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + Note note = notebook.createNote(anonymous); + factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); final Paragraph p = note.addParagraph(); p.setText("hello world"); @@ -469,7 +470,7 @@ public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSuppo // Force paragraph to have String type object p.setResult("Exception"); - Note cloneNote = notebook.cloneNote(note.getId(), "clone note with Exception result", null); + Note cloneNote = notebook.cloneNote(note.getId(), "clone note with Exception result", anonymous); Paragraph cp = cloneNote.paragraphs.get(0); // Keep same ParagraphID @@ -482,8 +483,8 @@ public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSuppo @Test public void testResourceRemovealOnParagraphNoteRemove() throws IOException { - Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + Note note = notebook.createNote(anonymous); + factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) { intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId())); } @@ -499,11 +500,11 @@ public void testResourceRemovealOnParagraphNoteRemove() throws IOException { assertEquals(2, ResourcePoolUtils.getAllResources().size()); // remove a paragraph - note.removeParagraph(p1.getId()); + note.removeParagraph(anonymous.getUser(), p1.getId()); assertEquals(1, ResourcePoolUtils.getAllResources().size()); // remove note - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); assertEquals(0, ResourcePoolUtils.getAllResources().size()); } @@ -511,11 +512,11 @@ public void testResourceRemovealOnParagraphNoteRemove() throws IOException { public void testAngularObjectRemovalOnNotebookRemove() throws InterruptedException, IOException { // create a note and a paragraph - Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + Note note = notebook.createNote(anonymous); + factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); AngularObjectRegistry registry = factory - .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess") + .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") .getAngularObjectRegistry(); Paragraph p1 = note.addParagraph(); @@ -530,7 +531,7 @@ public void testAngularObjectRemovalOnNotebookRemove() throws InterruptedExcepti registry.add("o3", "object3", null, null); // remove notebook - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); // notebook scope or paragraph scope object should be removed assertNull(registry.get("o1", note.getId(), null)); @@ -544,11 +545,11 @@ public void testAngularObjectRemovalOnNotebookRemove() throws InterruptedExcepti public void testAngularObjectRemovalOnParagraphRemove() throws InterruptedException, IOException { // create a note and a paragraph - Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + Note note = notebook.createNote(anonymous); + factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); AngularObjectRegistry registry = factory - .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess") + .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") .getAngularObjectRegistry(); Paragraph p1 = note.addParagraph(); @@ -563,7 +564,7 @@ public void testAngularObjectRemovalOnParagraphRemove() throws InterruptedExcept registry.add("o3", "object3", null, null); // remove notebook - note.removeParagraph(p1.getId()); + note.removeParagraph(anonymous.getUser(), p1.getId()); // paragraph scope should be removed assertNull(registry.get("o1", note.getId(), null)); @@ -578,11 +579,11 @@ public void testAngularObjectRemovalOnParagraphRemove() throws InterruptedExcept public void testAngularObjectRemovalOnInterpreterRestart() throws InterruptedException, IOException { // create a note and a paragraph - Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + Note note = notebook.createNote(anonymous); + factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); AngularObjectRegistry registry = factory - .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess") + .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") .getAngularObjectRegistry(); // add local scope object @@ -592,19 +593,19 @@ public void testAngularObjectRemovalOnInterpreterRestart() throws InterruptedExc // restart interpreter factory.restart(factory.getInterpreterSettings(note.getId()).get(0).getId()); - registry = factory.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess") + registry = factory.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") .getAngularObjectRegistry(); // local and global scope object should be removed assertNull(registry.get("o1", note.getId(), null)); assertNull(registry.get("o2", null, null)); - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); } @Test public void testPermissions() throws IOException { // create a note and a paragraph - Note note = notebook.createNote(null); + Note note = notebook.createNote(anonymous); NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); // empty owners, readers or writers means note is public assertEquals(notebookAuthorization.isOwner(note.getId(), @@ -643,14 +644,14 @@ public void testPermissions() throws IOException { assertEquals(notebookAuthorization.isReader(note.getId(), new HashSet(Arrays.asList("user3"))), true); - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); } @Test public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException, IOException { - Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + Note note = notebook.createNote(anonymous); + factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); ArrayList paragraphs = new ArrayList<>(); for (int i = 0; i < 100; i++) { @@ -687,7 +688,7 @@ public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedExc @Test public void testPerSessionInterpreterCloseOnNoteRemoval() throws IOException { // create a notes - Note note1 = notebook.createNote(null); + Note note1 = notebook.createNote(new AuthenticationInfo()); Paragraph p1 = note1.addParagraph(); p1.setText("getId"); @@ -702,8 +703,8 @@ public void testPerSessionInterpreterCloseOnNoteRemoval() throws IOException { InterpreterResult result = p1.getResult(); // remove note and recreate - notebook.removeNote(note1.getId(), null); - note1 = notebook.createNote(null); + notebook.removeNote(note1.getId(), anonymous); + note1 = notebook.createNote(anonymous); p1 = note1.addParagraph(); p1.setText("getId"); @@ -711,16 +712,16 @@ public void testPerSessionInterpreterCloseOnNoteRemoval() throws IOException { while (p1.getStatus() != Status.FINISHED) Thread.yield(); assertNotEquals(p1.getResult().message(), result.message()); - notebook.removeNote(note1.getId(), null); + notebook.removeNote(note1.getId(), anonymous); } @Test public void testPerSessionInterpreter() throws IOException { // create two notes - Note note1 = notebook.createNote(null); + Note note1 = notebook.createNote(anonymous); Paragraph p1 = note1.addParagraph(); - Note note2 = notebook.createNote(null); + Note note2 = notebook.createNote(anonymous); Paragraph p2 = note2.addParagraph(); p1.setText("getId"); @@ -751,15 +752,16 @@ public void testPerSessionInterpreter() throws IOException { assertNotEquals(p1.getResult().message(), p2.getResult().message()); - notebook.removeNote(note1.getId(), null); - notebook.removeNote(note2.getId(), null); + notebook.removeNote(note1.getId(), anonymous); + notebook.removeNote(note2.getId(), anonymous); } @Test public void testPerSessionInterpreterCloseOnUnbindInterpreterSetting() throws IOException { // create a notes - Note note1 = notebook.createNote(null); + Note note1 = notebook.createNote(anonymous); Paragraph p1 = note1.addParagraph(); + p1.setAuthenticationInfo(anonymous); p1.setText("getId"); // restart interpreter with per note session enabled @@ -775,15 +777,15 @@ public void testPerSessionInterpreterCloseOnUnbindInterpreterSetting() throws IO // unbind, and rebind setting. that result interpreter instance close List bindedSettings = notebook.getBindedInterpreterSettingsIds(note1.getId()); - notebook.bindInterpretersToNote(note1.getId(), new LinkedList()); - notebook.bindInterpretersToNote(note1.getId(), bindedSettings); + notebook.bindInterpretersToNote(anonymous.getUser(), note1.getId(), new LinkedList()); + notebook.bindInterpretersToNote(anonymous.getUser(), note1.getId(), bindedSettings); note1.run(p1.getId()); while (p1.getStatus() != Status.FINISHED) Thread.yield(); assertNotEquals(result.message(), p1.getResult().message()); - notebook.removeNote(note1.getId(), null); + notebook.removeNote(note1.getId(), anonymous); } @Test @@ -825,7 +827,7 @@ public void onParagraphStatusChange(Paragraph p, Status status) { } }); - Note note1 = notebook.createNote(null); + Note note1 = notebook.createNote(anonymous); assertEquals(1, onNoteCreate.get()); Paragraph p1 = note1.addParagraph(); @@ -834,14 +836,14 @@ public void onParagraphStatusChange(Paragraph p, Status status) { note1.addCloneParagraph(p1); assertEquals(2, onParagraphCreate.get()); - note1.removeParagraph(p1.getId()); + note1.removeParagraph(anonymous.getUser(), p1.getId()); assertEquals(1, onParagraphRemove.get()); List settings = notebook.getBindedInterpreterSettingsIds(note1.getId()); - notebook.bindInterpretersToNote(note1.getId(), new LinkedList()); + notebook.bindInterpretersToNote(anonymous.getUser(), note1.getId(), new LinkedList()); assertEquals(settings.size(), unbindInterpreter.get()); - notebook.removeNote(note1.getId(), null); + notebook.removeNote(note1.getId(), anonymous); assertEquals(1, onNoteRemove.get()); assertEquals(1, onParagraphRemove.get()); } @@ -849,7 +851,7 @@ public void onParagraphStatusChange(Paragraph p, Status status) { @Test public void testNormalizeNoteName() throws IOException { // create a notes - Note note1 = notebook.createNote(null); + Note note1 = notebook.createNote(anonymous); note1.setName("MyNote"); assertEquals(note1.getName(), "MyNote"); @@ -869,25 +871,25 @@ public void testNormalizeNoteName() throws IOException { note1.setName("\\\\\\MyNote///sub"); assertEquals(note1.getName(), "/MyNote/sub"); - notebook.removeNote(note1.getId(), null); + notebook.removeNote(note1.getId(), anonymous); } @Test public void testGetAllNotes() throws Exception { - Note note1 = notebook.createNote(null); - Note note2 = notebook.createNote(null); - assertEquals(2, notebook.getAllNotes(new AuthenticationInfo("anonymous")).size()); + Note note1 = notebook.createNote(anonymous); + Note note2 = notebook.createNote(anonymous); + assertEquals(2, notebook.getAllNotes(anonymous).size()); notebook.getNotebookAuthorization().setOwners(note1.getId(), Sets.newHashSet("user1")); notebook.getNotebookAuthorization().setWriters(note1.getId(), Sets.newHashSet("user1")); notebook.getNotebookAuthorization().setReaders(note1.getId(), Sets.newHashSet("user1")); - assertEquals(1, notebook.getAllNotes(new AuthenticationInfo("anonymous")).size()); + assertEquals(1, notebook.getAllNotes(anonymous).size()); assertEquals(2, notebook.getAllNotes(new AuthenticationInfo("user1")).size()); notebook.getNotebookAuthorization().setOwners(note2.getId(), Sets.newHashSet("user2")); notebook.getNotebookAuthorization().setWriters(note2.getId(), Sets.newHashSet("user2")); notebook.getNotebookAuthorization().setReaders(note2.getId(), Sets.newHashSet("user2")); - assertEquals(0, notebook.getAllNotes(new AuthenticationInfo("anonymous")).size()); + assertEquals(0, notebook.getAllNotes(anonymous).size()); assertEquals(1, notebook.getAllNotes(new AuthenticationInfo("user1")).size()); assertEquals(1, notebook.getAllNotes(new AuthenticationInfo("user2")).size()); notebook.removeNote(note1.getId(), null); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index 9fc2d826251..e9c3e9a3bc3 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -107,7 +107,7 @@ public void testInvalidJsonFile() throws IOException { @Test public void testSaveNotebook() throws IOException, InterruptedException { Note note = notebook.createNote(null); - factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); + factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList()); Paragraph p1 = note.addParagraph(); Map config = p1.getConfig(); From 8589545ee3df67527aeaecc810ba3eef3fc8c266 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 3 Aug 2016 00:10:18 +0900 Subject: [PATCH 04/34] Added option in UI --- .../zeppelin/socket/NotebookServer.java | 3 ++- .../interpreter-create.html | 15 ++++++++++---- .../src/app/interpreter/interpreter.html | 9 +++++++++ .../interpreter/InterpreterFactory.java | 15 +++++++------- .../interpreter/InterpreterOption.java | 20 +++++++++++++++++++ .../interpreter/InterpreterSetting.java | 9 +++++---- .../org/apache/zeppelin/notebook/Note.java | 1 + .../notebook/NoteInterpreterLoaderTest.java | 8 ++++---- .../zeppelin/notebook/NotebookTest.java | 6 +++--- 9 files changed, 63 insertions(+), 23 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index aaace7b4bb9..24a91e9aae2 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -54,6 +54,7 @@ import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.xml.PrettyPrinter; import javax.servlet.http.HttpServletRequest; import java.io.IOException; @@ -1509,7 +1510,7 @@ public void afterStatusChange(Job job, Status before, Status after) { LOG.info("Job {} is finished", job.getId()); try { //TODO(khalid): may change interface for JobListener and pass subject from interpreter - note.persist(null); + note.persist(job instanceof Paragraph ? ((Paragraph) job).getAuthenticationInfo() : null); } catch (IOException e) { LOG.error(e.toString(), e); } diff --git a/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html index d0e0749360f..2638dd84e3a 100644 --- a/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html +++ b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html @@ -37,23 +37,31 @@

Create new interpreter

Option
+ + + + + + - Interpreter for note
diff --git a/zeppelin-web/src/app/interpreter/interpreter.html b/zeppelin-web/src/app/interpreter/interpreter.html index 57b305ee67c..68c2afc7939 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.html +++ b/zeppelin-web/src/app/interpreter/interpreter.html @@ -141,9 +141,18 @@

{{setting.name}}

+
Option
+ + + + + + - - - Interpreter for note +
+
+ The interpreter will be instantiated + + + + + in + + + + + process. +
+
From 787a366b259dae31768dcd4f4aeda2ccd8182e46 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Tue, 18 Oct 2016 00:54:16 +0900 Subject: [PATCH 26/34] change Back-end test cases and member type (perNote, perUser) --- .../interpreter/remote/RemoteInterpreter.java | 1 + .../interpreter/InterpreterOption.java | 47 ++++++++++++++++--- .../interpreter/InterpreterFactoryTest.java | 2 +- .../notebook/NoteInterpreterLoaderTest.java | 14 ++++-- .../zeppelin/notebook/NotebookTest.java | 8 ++-- 5 files changed, 58 insertions(+), 14 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index 073b84bbda2..e0cdaa338b1 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -298,6 +298,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) { if (logger.isDebugEnabled()) { logger.debug("st:\n{}", st); } + FormType form = getFormType(); RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); Client client = null; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java index 9fe14fa063c..6c13a4c4b08 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java @@ -23,12 +23,16 @@ * */ public class InterpreterOption { + public final transient String SHARED = "shared"; + public final transient String SCOPED = "scoped"; + public final transient String ISOLATED = "isolated"; + boolean remote; String host = null; int port = -1; - boolean perNote; - boolean perUser; + String perNote; + String perUser; boolean session; boolean process; @@ -37,19 +41,46 @@ public class InterpreterOption { boolean setPermission; List users; + public boolean isGlobally() { + if (perNote != null && perNote.equals(SHARED) + && perUser != null && perUser.equals(SHARED)) { + return true; + } + return false; + } + public boolean isPerNote() { - return perNote; + if (isGlobally() == true) { + return false; + } + + if (perNote != null && !perNote.equals("")) { + return true; + } + + return false; } - public void setPerNote(boolean perNote) { + public void setPerNote(String perNote) { this.perNote = perNote; } public boolean isPerUser() { - return perUser; + if (isGlobally() == true) { + return false; + } + + if (isPerNote() == true) { + return false; + } + + if (perUser != null && !perUser.equals("")) { + return true; + } + return false; } - public void setPerUser(boolean perUser) { + public void setPerUser(String perUser) { this.perUser = perUser; } @@ -82,10 +113,14 @@ public List getUsers() { } public InterpreterOption() { + this.perNote = null; + this.perUser = null; remote = false; } public InterpreterOption(boolean remote) { + this.perNote = null; + this.perUser = null; this.remote = remote; } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 66967176f3d..662a27b712f 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -226,7 +226,7 @@ public void testMultiUser() throws IOException, RepositoryException { InterpreterOption perUserInterpreterOption = new InterpreterOption(true); perUserInterpreterOption.setSession(true); - perUserInterpreterOption.setPerUser(true); + perUserInterpreterOption.setPerUser(perUserInterpreterOption.ISOLATED); final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList(), perUserInterpreterOption, new Properties()); factory.setInterpreters("user1", "note", new ArrayList() {{ diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java index a826120b4eb..307801ffe53 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java @@ -95,13 +95,16 @@ public void testGetInterpreter() throws IOException { @Test public void testNoteSession() throws IOException { + InterpreterOption dumyInterpreterOption = new InterpreterOption(); factory.setInterpreters("user", "noteA", factory.getDefaultInterpreterSettingList()); factory.getInterpreterSettings("noteA").get(0).getOption().setSession(true); - factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(true); + factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(dumyInterpreterOption.SCOPED); + factory.getInterpreterSettings("noteA").get(0).getOption().setPerUser(""); factory.setInterpreters("user", "noteB", factory.getDefaultInterpreterSettingList()); factory.getInterpreterSettings("noteB").get(0).getOption().setSession(true); - factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(true); + factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(dumyInterpreterOption.SCOPED); + factory.getInterpreterSettings("noteB").get(0).getOption().setPerUser(""); // interpreters are not created before accessing it assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get(":noteA")); @@ -130,13 +133,16 @@ public void testNoteSession() throws IOException { @Test public void testNotePerInterpreterProcess() throws IOException { + InterpreterOption dumyInterpreterOption = new InterpreterOption(); factory.setInterpreters("user", "noteA", factory.getDefaultInterpreterSettingList()); factory.getInterpreterSettings("noteA").get(0).getOption().setProcess(true); - factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(true); + factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(dumyInterpreterOption.SCOPED); + factory.getInterpreterSettings("noteA").get(0).getOption().setPerUser(""); factory.setInterpreters("user", "noteB", factory.getDefaultInterpreterSettingList()); factory.getInterpreterSettings("noteB").get(0).getOption().setProcess(true); - factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(true); + factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(dumyInterpreterOption.SCOPED); + factory.getInterpreterSettings("noteB").get(0).getOption().setPerUser(""); // interpreters are not created before accessing it assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get(":noteA")); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index d786a977e63..96d31ccd18c 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -696,10 +696,11 @@ public void testPerSessionInterpreterCloseOnNoteRemoval() throws IOException { p1.setText("getId"); p1.setAuthenticationInfo(anonymous); - // restart interpreter with per note session enabled + // restart interpreter with per user session enabled for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) { setting.getOption().setSession(true); - setting.getOption().setPerUser(true); + setting.getOption().setPerNote(setting.getOption().SCOPED); + setting.getOption().setPerUser(""); notebook.getInterpreterFactory().restart(setting.getId()); } @@ -748,7 +749,8 @@ public void testPerSessionInterpreter() throws IOException { // restart interpreter with per note session enabled for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) { setting.getOption().setSession(true); - setting.getOption().setPerNote(true); + setting.getOption().setPerNote(setting.getOption().SCOPED); + setting.getOption().setPerUser(setting.getOption().SCOPED); notebook.getInterpreterFactory().restart(setting.getId()); } From 1f64e52ed2ba3003c0ad41b4259019a895fb4807 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Tue, 18 Oct 2016 03:21:15 +0900 Subject: [PATCH 27/34] change default value for pernote and peruser --- .../apache/zeppelin/interpreter/InterpreterOption.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java index 6c13a4c4b08..00c7533354e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java @@ -113,14 +113,14 @@ public List getUsers() { } public InterpreterOption() { - this.perNote = null; - this.perUser = null; + this.perNote = SCOPED; + this.perUser = SCOPED; remote = false; } public InterpreterOption(boolean remote) { - this.perNote = null; - this.perUser = null; + this.perNote = SCOPED; + this.perUser = SCOPED; this.remote = remote; } From b18bff4a1f72fe0add7276e886c6ce9b56bced7d Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Tue, 18 Oct 2016 03:22:11 +0900 Subject: [PATCH 28/34] implement frontend for interpreter per user mode and misc mode --- .../app/interpreter/interpreter.controller.js | 106 +++++++++++---- .../src/app/interpreter/interpreter.html | 126 +++++++++++++++--- 2 files changed, 189 insertions(+), 43 deletions(-) diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js index 37ba3131446..57c92c04b5f 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.controller.js +++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js @@ -156,7 +156,7 @@ interpreterSettingsTmp[index] = angular.copy($scope.interpreterSettings[index]); }; - $scope.setSessionOption = function(settingId, sessionOption) { + $scope.setPerNoteOption = function(settingId, sessionOption) { var option; if (settingId === undefined) { option = $scope.newInterpreterSetting.option; @@ -167,18 +167,21 @@ } if (sessionOption === 'isolated') { + option.perNote = sessionOption; option.session = false; option.process = true; } else if (sessionOption === 'scoped') { + option.perNote = sessionOption; option.session = true; option.process = false; } else { + option.perNote = 'shared'; option.session = false; option.process = false; } }; - $scope.getSessionOption = function(settingId) { + $scope.setPerUserOption = function(settingId, sessionOption) { var option; if (settingId === undefined) { option = $scope.newInterpreterSetting.option; @@ -187,16 +190,42 @@ var setting = $scope.interpreterSettings[index]; option = setting.option; } - if (option.session) { + + if (sessionOption === 'isolated') { + option.perUser = sessionOption; + option.session = false; + option.process = true; + } else if (sessionOption === 'scoped') { + option.perUser = sessionOption; + option.session = true; + option.process = false; + } else { + option.perUser = 'shared'; + option.session = false; + option.process = false; + } + }; + + $scope.getPerNoteOption = function(settingId) { + var option; + if (settingId === undefined) { + option = $scope.newInterpreterSetting.option; + } else { + var index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + var setting = $scope.interpreterSettings[index]; + option = setting.option; + } + + if (option.perNote == 'scoped') { return 'scoped'; - } else if (option.process) { + } else if (option.perNote == 'isolated') { return 'isolated'; } else { return 'shared'; } }; - $scope.getInterpreterRunningOption = function(settingId) { + $scope.getPerUserOption = function(settingId) { var option; if (settingId === undefined) { option = $scope.newInterpreterSetting.option; @@ -206,37 +235,58 @@ option = setting.option; } - var isPerNote = option.perNote; - var isPerUser = option.perUser; + if (option.perUser == 'scoped') { + return 'scoped'; + } else if (option.perUser == 'isolated') { + return 'isolated'; + } else { + return 'shared'; + } + }; - if (isPerNote === true && isPerUser === false) { - if (option.session === false && option.process === false) { - option.session = true; - } + $scope.getInterpreterRunningOption = function(settingId) { + var sharedModeName = 'shared'; - return {value: 'Per Note', isPerNote: isPerNote, isPerUser: isPerUser}; - } else if (isPerNote !== undefined && isPerUser === true) { - if (option.session === false && option.process === false) { - option.session = true; - } - console.log('clover ', $rootScope.ticket); - if ($rootScope.ticket.ticket === 'anonymous' && $rootScope.ticket.roles === '[]') { - option.perNote = true; - option.perUser = false; - return {value: 'Per Note', isPerNote: isPerNote, isPerUser: isPerUser}; - } - return {value: 'Per User', isPerNote: isPerNote, isPerUser: isPerUser}; + var globallyModeName = 'Globally'; + var perNoteModeName = 'Per Note'; + var perUserModeName = 'Per User'; + var option; + if (settingId === undefined) { + option = $scope.newInterpreterSetting.option; } else { - // fixed shared on Globally - option.session = false; - option.process = false; + var index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + var setting = $scope.interpreterSettings[index]; + option = setting.option; + } + + var perNote = option.perNote; + var perUser = option.perUser; - return {value: 'Globally', isPerNote: isPerNote, isPerUser: isPerUser}; + // Globally == shared_perNote + shared_perUser + if (perNote === sharedModeName && perUser === sharedModeName) { + return globallyModeName; } + + if ($rootScope.ticket.ticket === 'anonymous' && $rootScope.ticket.roles === '[]') { + if (perNote !== undefined && typeof perNote === 'string' && perNote !== '') { + return perNoteModeName; + } + } else if ($rootScope.ticket.ticket !== 'anonymous') { + if (perNote !== undefined && typeof perNote === 'string' && perNote !== '') { + if (perUser !== undefined && typeof perUser === 'string' && perUser !== '') { + return perUserModeName; + } + return perNoteModeName; + } + } + + option.perNote = sharedModeName; + option.perUser = sharedModeName; + return globallyModeName; }; - $scope.setInterpreterRunningOption = function (settingId, isPerNoteMode, isPerUserMode) { + $scope.setInterpreterRunningOption = function(settingId, isPerNoteMode, isPerUserMode) { var option; if (settingId === undefined) { option = $scope.newInterpreterSetting.option; diff --git a/zeppelin-web/src/app/interpreter/interpreter.html b/zeppelin-web/src/app/interpreter/interpreter.html index 0336ae06202..e0e861343b9 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.html +++ b/zeppelin-web/src/app/interpreter/interpreter.html @@ -146,30 +146,30 @@

{{setting.name}}
Option
-
+
The interpreter will be instantiated
+
+   +
+
+
+
+ + + + And + + + + in + + + + + process. + + + +
+
+  
From d20195017c8a8f0f780a3d4076259a0ead8cc575 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Tue, 18 Oct 2016 03:29:22 +0900 Subject: [PATCH 29/34] fix eqeqeq issue for frontweb --- .../src/app/interpreter/interpreter.controller.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js index 57c92c04b5f..0669ff8f38a 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.controller.js +++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js @@ -216,9 +216,9 @@ option = setting.option; } - if (option.perNote == 'scoped') { + if (option.perNote === 'scoped') { return 'scoped'; - } else if (option.perNote == 'isolated') { + } else if (option.perNote === 'isolated') { return 'isolated'; } else { return 'shared'; @@ -235,9 +235,9 @@ option = setting.option; } - if (option.perUser == 'scoped') { + if (option.perUser === 'scoped') { return 'scoped'; - } else if (option.perUser == 'isolated') { + } else if (option.perUser === 'isolated') { return 'isolated'; } else { return 'shared'; From 5e7da341e04327bc06cce2c2afee0f92b3d88d5f Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Tue, 18 Oct 2016 21:03:47 +0900 Subject: [PATCH 30/34] Changed instanceKey and processKey for dealing with new UI --- .../apache/zeppelin/utils/SecurityUtils.java | 6 + .../interpreter/InterpreterFactory.java | 26 ++-- .../interpreter/InterpreterOption.java | 117 ++++++++---------- .../interpreter/InterpreterSetting.java | 2 +- .../helium/HeliumApplicationFactoryTest.java | 2 +- .../interpreter/InterpreterFactoryTest.java | 14 +-- .../notebook/NoteInterpreterLoaderTest.java | 44 +++---- .../zeppelin/notebook/NotebookTest.java | 12 +- .../notebook/repo/NotebookRepoSyncTest.java | 2 +- .../notebook/repo/VFSNotebookRepoTest.java | 2 +- 10 files changed, 106 insertions(+), 121 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java b/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java index f9e5929a882..d81d2e6150b 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java @@ -119,4 +119,10 @@ public static HashSet getRoles() { return roles; } + /** + * Checked if shiro enabled or not + */ + public static boolean isAuthenticated() { + return org.apache.shiro.SecurityUtils.getSubject().isAuthenticated(); + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index a6e7f4371ff..3765c493cf7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -121,6 +121,8 @@ public class InterpreterFactory implements InterpreterGroupFactory { private DependencyResolver depResolver; + private boolean shiroEnabled; + private Map env = new HashMap<>(); private Interpreter devInterpreter; @@ -128,18 +130,18 @@ public class InterpreterFactory implements InterpreterGroupFactory { public InterpreterFactory(ZeppelinConfiguration conf, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appEventListener, DependencyResolver depResolver) - throws InterpreterException, IOException, RepositoryException { + ApplicationEventListener appEventListener, DependencyResolver depResolver, + boolean shiroEnabled) throws InterpreterException, IOException, RepositoryException { this(conf, new InterpreterOption(true), angularObjectRegistryListener, - remoteInterpreterProcessListener, appEventListener, depResolver); + remoteInterpreterProcessListener, appEventListener, depResolver, shiroEnabled); } public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultOption, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appEventListener, DependencyResolver depResolver) - throws InterpreterException, IOException, RepositoryException { + ApplicationEventListener appEventListener, DependencyResolver depResolver, + boolean shiroEnabled) throws InterpreterException, IOException, RepositoryException { this.conf = conf; this.defaultOption = defaultOption; this.angularObjectRegistryListener = angularObjectRegistryListener; @@ -147,6 +149,7 @@ public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultO this.interpreterRepositories = depResolver.getRepos(); this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; this.appEventListener = appEventListener; + this.shiroEnabled = shiroEnabled; String replsConf = conf.getString(ConfVars.ZEPPELIN_INTERPRETERS); interpreterClassList = replsConf.split(","); String groupOrder = conf.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER); @@ -700,9 +703,10 @@ public InterpreterGroup createInterpreterGroup(String id, InterpreterOption opti public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user, String noteId) { - if (interpreterSetting.getOption().isProcess()) { + InterpreterOption option = interpreterSetting.getOption(); + if (option.isProcess()) { interpreterSetting.closeAndRemoveInterpreterGroup(noteId); - } else if (interpreterSetting.getOption().isSession()) { + } else if (option.isSession()) { InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId); String key = getInterpreterInstanceKey(user, noteId, interpreterSetting); interpreterGroup.close(key); @@ -1138,8 +1142,12 @@ private String getInterpreterInstanceKey(String user, String noteId, Interpreter String key; if (option.isExistingProcess()) { key = Constants.EXISTING_PROCESS; - } else if (option.isSession() || option.isProcess()) { - key = (option.isPerUser() ? user : "") + ":" + (option.isPerNote() ? noteId : ""); + } else if (!(shiroEnabled && option.perUserShared()) || !option.perNoteShared()) { + if (shiroEnabled) { + key = (!option.perUserShared() ? user : "") + ":" + (!option.perNoteShared() ? noteId : ""); + } else { + key = (!option.perNoteShared() ? noteId : ""); + } } else { key = SHARED_SESSION; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java index 00c7533354e..e5c0f51a6d0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java @@ -17,15 +17,17 @@ package org.apache.zeppelin.interpreter; +import com.google.common.base.Preconditions; + import java.util.List; /** * */ public class InterpreterOption { - public final transient String SHARED = "shared"; - public final transient String SCOPED = "scoped"; - public final transient String ISOLATED = "isolated"; + public static final transient String SHARED = "shared"; + public static final transient String SCOPED = "scoped"; + public static final transient String ISOLATED = "isolated"; boolean remote; String host = null; @@ -34,56 +36,10 @@ public class InterpreterOption { String perNote; String perUser; - boolean session; - boolean process; - boolean isExistingProcess; boolean setPermission; List users; - public boolean isGlobally() { - if (perNote != null && perNote.equals(SHARED) - && perUser != null && perUser.equals(SHARED)) { - return true; - } - return false; - } - - public boolean isPerNote() { - if (isGlobally() == true) { - return false; - } - - if (perNote != null && !perNote.equals("")) { - return true; - } - - return false; - } - - public void setPerNote(String perNote) { - this.perNote = perNote; - } - - public boolean isPerUser() { - if (isGlobally() == true) { - return false; - } - - if (isPerNote() == true) { - return false; - } - - if (perUser != null && !perUser.equals("")) { - return true; - } - return false; - } - - public void setPerUser(String perUser) { - this.perUser = perUser; - } - public boolean isExistingProcess() { return isExistingProcess; } @@ -113,15 +69,21 @@ public List getUsers() { } public InterpreterOption() { - this.perNote = SCOPED; - this.perUser = SCOPED; - remote = false; + this(false); } public InterpreterOption(boolean remote) { - this.perNote = SCOPED; - this.perUser = SCOPED; + this(remote, SHARED, SHARED); + } + + public InterpreterOption(boolean remote, String perUser, String perNote) { + Preconditions.checkNotNull(remote); + Preconditions.checkNotNull(perUser); + Preconditions.checkNotNull(perNote); + this.remote = remote; + this.perUser = perUser; + this.perNote = perNote; } public boolean isRemote() { @@ -132,14 +94,6 @@ public void setRemote(boolean remote) { this.remote = remote; } - public boolean isSession() { - return session; - } - - public void setSession(boolean session) { - this.session = session; - } - public String getHost() { return host; } @@ -148,11 +102,44 @@ public int getPort() { return port; } + + public boolean perUserShared() { + return SHARED.equals(perUser); + } + + public boolean perUserScoped() { + return SCOPED.equals(perUser); + } + + public boolean perUserIsolated() { + return ISOLATED.equals(perUser); + } + + public boolean perNoteShared() { + return SHARED.equals(perNote); + } + + public boolean perNoteScoped() { + return SCOPED.equals(perNote); + } + + public boolean perNoteIsolated() { + return ISOLATED.equals(perNote); + } + public boolean isProcess() { - return process; + return perUserIsolated() || perNoteIsolated(); + } + + public boolean isSession() { + return perUserScoped() || perNoteScoped(); } - public void setProcess(boolean process) { - this.process = process; + public void setPerNote(String perNote) { + this.perNote = perNote; + } + + public void setPerUser(String perUser) { + this.perUser = perUser; } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 955182c4add..4611559b0b7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -112,7 +112,7 @@ private String getInterpreterProcessKey(String user, String noteId) { if (getOption().isExistingProcess) { key = Constants.EXISTING_PROCESS; } else if (getOption().isProcess()) { - key = (option.isPerUser() ? user : "") + ":" + (option.isPerNote() ? noteId : ""); + key = (option.perUserIsolated() ? user : "") + ":" + (option.perNoteIsolated() ? noteId : ""); } else { key = SHARED_PROCESS; } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 6641787ece4..294817c1908 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -85,7 +85,7 @@ public void setUp() throws Exception { heliumAppFactory = new HeliumApplicationFactory(); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); factory = new InterpreterFactory(conf, - new InterpreterOption(true), null, null, heliumAppFactory, depResolver); + new InterpreterOption(true), null, null, heliumAppFactory, depResolver, false); HashMap env = new HashMap(); env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); factory.setEnv(env); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 662a27b712f..a2a799ae8a0 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -89,7 +89,7 @@ public void setUp() throws Exception { conf = new ZeppelinConfiguration(); schedulerFactory = new SchedulerFactory(); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false); context = new InterpreterContext("note", "id", "title", "text", null, null, null, null, null, null, null); SearchService search = mock(SearchService.class); @@ -132,7 +132,7 @@ public void testBasic() { @Test public void testRemoteRepl() throws Exception { - factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false); List all = factory.get(); InterpreterSetting mock1Setting = null; for (InterpreterSetting setting : all) { @@ -188,13 +188,13 @@ public void testSaveLoad() throws IOException, RepositoryException { factory.createNewSetting("new-mock1", "mock1", new LinkedList(), new InterpreterOption(false), new Properties()); assertEquals(numInterpreters + 1, factory.get().size()); - InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver); + InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver, false); assertEquals(numInterpreters + 1, factory2.get().size()); } @Test public void testInterpreterAliases() throws IOException, RepositoryException { - factory = new InterpreterFactory(conf, null, null, null, depResolver); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false); final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null); final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true, null); factory.add("group1", new ArrayList(){{ @@ -218,15 +218,13 @@ public void testInterpreterAliases() throws IOException, RepositoryException { @Test public void testMultiUser() throws IOException, RepositoryException { - factory = new InterpreterFactory(conf, null, null, null, depResolver); + factory = new InterpreterFactory(conf, null, null, null, depResolver, true); final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null); factory.add("group1", new ArrayList(){{ add(info1); }}, new ArrayList(), new InterpreterOption(true), new Properties(), "/path1"); - InterpreterOption perUserInterpreterOption = new InterpreterOption(true); - perUserInterpreterOption.setSession(true); - perUserInterpreterOption.setPerUser(perUserInterpreterOption.ISOLATED); + InterpreterOption perUserInterpreterOption = new InterpreterOption(true, InterpreterOption.ISOLATED, InterpreterOption.SHARED); final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList(), perUserInterpreterOption, new Properties()); factory.setInterpreters("user1", "note", new ArrayList() {{ diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java index 307801ffe53..d19ceae08e4 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java @@ -60,7 +60,7 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "group2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false); } @After @@ -95,20 +95,15 @@ public void testGetInterpreter() throws IOException { @Test public void testNoteSession() throws IOException { - InterpreterOption dumyInterpreterOption = new InterpreterOption(); factory.setInterpreters("user", "noteA", factory.getDefaultInterpreterSettingList()); - factory.getInterpreterSettings("noteA").get(0).getOption().setSession(true); - factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(dumyInterpreterOption.SCOPED); - factory.getInterpreterSettings("noteA").get(0).getOption().setPerUser(""); + factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.SCOPED); factory.setInterpreters("user", "noteB", factory.getDefaultInterpreterSettingList()); - factory.getInterpreterSettings("noteB").get(0).getOption().setSession(true); - factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(dumyInterpreterOption.SCOPED); - factory.getInterpreterSettings("noteB").get(0).getOption().setPerUser(""); + factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.SCOPED); // interpreters are not created before accessing it - assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get(":noteA")); - assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get(":noteB")); + assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA")); + assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB")); factory.getInterpreter("user", "noteA", null).open(); factory.getInterpreter("user", "noteB", null).open(); @@ -118,35 +113,30 @@ public void testNoteSession() throws IOException { factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId())); // interpreters are created after accessing it - assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get(":noteA")); - assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get(":noteB")); + assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA")); + assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB")); // when factory.closeNote("user", "noteA"); factory.closeNote("user", "noteB"); // interpreters are destroyed after close - assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get(":noteA")); - assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get(":noteB")); + assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA")); + assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB")); } @Test public void testNotePerInterpreterProcess() throws IOException { - InterpreterOption dumyInterpreterOption = new InterpreterOption(); factory.setInterpreters("user", "noteA", factory.getDefaultInterpreterSettingList()); - factory.getInterpreterSettings("noteA").get(0).getOption().setProcess(true); - factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(dumyInterpreterOption.SCOPED); - factory.getInterpreterSettings("noteA").get(0).getOption().setPerUser(""); + factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.ISOLATED); factory.setInterpreters("user", "noteB", factory.getDefaultInterpreterSettingList()); - factory.getInterpreterSettings("noteB").get(0).getOption().setProcess(true); - factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(dumyInterpreterOption.SCOPED); - factory.getInterpreterSettings("noteB").get(0).getOption().setPerUser(""); + factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.ISOLATED); // interpreters are not created before accessing it - assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get(":noteA")); - assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get(":noteB")); + assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA")); + assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB")); factory.getInterpreter("user", "noteA", null).open(); factory.getInterpreter("user", "noteB", null).open(); @@ -157,16 +147,16 @@ public void testNotePerInterpreterProcess() throws IOException { factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId())); // interpreters are created after accessing it - assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get(":noteA")); - assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get(":noteB")); + assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA")); + assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB")); // when factory.closeNote("user", "noteA"); factory.closeNote("user", "noteB"); // interpreters are destroyed after close - assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get(":noteA")); - assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get(":noteB")); + assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA")); + assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB")); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 96d31ccd18c..b1d622fdef0 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -89,7 +89,7 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); @@ -210,7 +210,7 @@ public void testPersist() throws IOException, SchedulerException, RepositoryExce Notebook notebook2 = new Notebook( conf, notebookRepo, schedulerFactory, - new InterpreterFactory(conf, null, null, null, depResolver), this, null, null, null); + new InterpreterFactory(conf, null, null, null, depResolver, false), this, null, null, null); assertEquals(1, notebook2.getAllNotes().size()); notebook.removeNote(note.getId(), null); @@ -698,9 +698,7 @@ public void testPerSessionInterpreterCloseOnNoteRemoval() throws IOException { // restart interpreter with per user session enabled for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) { - setting.getOption().setSession(true); setting.getOption().setPerNote(setting.getOption().SCOPED); - setting.getOption().setPerUser(""); notebook.getInterpreterFactory().restart(setting.getId()); } @@ -748,9 +746,7 @@ public void testPerSessionInterpreter() throws IOException { // restart interpreter with per note session enabled for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) { - setting.getOption().setSession(true); - setting.getOption().setPerNote(setting.getOption().SCOPED); - setting.getOption().setPerUser(setting.getOption().SCOPED); + setting.getOption().setPerNote(InterpreterOption.SCOPED); notebook.getInterpreterFactory().restart(setting.getId()); } @@ -777,7 +773,7 @@ public void testPerSessionInterpreterCloseOnUnbindInterpreterSetting() throws IO // restart interpreter with per note session enabled for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) { - setting.getOption().setSession(true); + setting.getOption().setPerNote(InterpreterOption.SCOPED); notebook.getInterpreterFactory().restart(setting.getId()); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 97bfb8bc75f..43ed586135e 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -97,7 +97,7 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false); search = mock(SearchService.class); notebookRepoSync = new NotebookRepoSync(conf); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index 86f5a0bf2b2..e6236c85a21 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -78,7 +78,7 @@ public void setUp() throws Exception { this.schedulerFactory = new SchedulerFactory(); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); From cee39f40be52151004cf45ddb663fb0161336c83 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Tue, 18 Oct 2016 22:01:44 +0900 Subject: [PATCH 31/34] Fixed to pass shiro information to InterpreterFactory from ZeppelinServer --- .../main/java/org/apache/zeppelin/server/ZeppelinServer.java | 2 +- .../org/apache/zeppelin/interpreter/InterpreterFactory.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index c6202355c83..de02fe0f3c8 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -89,7 +89,7 @@ public ZeppelinServer() throws Exception { this.heliumApplicationFactory = new HeliumApplicationFactory(); this.schedulerFactory = new SchedulerFactory(); this.replFactory = new InterpreterFactory(conf, notebookWsServer, - notebookWsServer, heliumApplicationFactory, depResolver); + notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated()); this.notebookRepo = new NotebookRepoSync(conf); this.notebookIndex = new LuceneSearch(); this.notebookAuthorization = NotebookAuthorization.init(conf); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 3765c493cf7..9233fcf9813 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -160,6 +160,8 @@ public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultO gson = builder.create(); init(); + + logger.info("shiroEnabled: {}", shiroEnabled); } private void init() throws InterpreterException, IOException, RepositoryException { From ad80951c5b8dad4dd716ba2222c7f7b2ac2b9df1 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 19 Oct 2016 13:19:20 +0900 Subject: [PATCH 32/34] Fixed some wrong logic of getInterpreterInstanceKey --- .../apache/zeppelin/interpreter/InterpreterFactory.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 9233fcf9813..19081c9889e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -1144,11 +1144,10 @@ private String getInterpreterInstanceKey(String user, String noteId, Interpreter String key; if (option.isExistingProcess()) { key = Constants.EXISTING_PROCESS; - } else if (!(shiroEnabled && option.perUserShared()) || !option.perNoteShared()) { - if (shiroEnabled) { - key = (!option.perUserShared() ? user : "") + ":" + (!option.perNoteShared() ? noteId : ""); - } else { - key = (!option.perNoteShared() ? noteId : ""); + } else if (!option.perNoteShared()) { + key = noteId; + if (shiroEnabled && !option.perUserShared()) { + key = user + ":" + key; } } else { key = SHARED_SESSION; From e84703d4efdf33032666f358ad84dbd64ccc57d6 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 19 Oct 2016 17:36:36 +0900 Subject: [PATCH 33/34] Fixed ZEPPELIN-1542 --- .../zeppelin/notebook/NotebookTest.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index b1d622fdef0..d0af2c90abe 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -129,7 +129,7 @@ public void testSelectingReplImplementation() throws IOException { note.run(p2.getId()); while(p2.isTerminated()==false || p2.getResult()==null) Thread.yield(); assertEquals("repl2: hello world", p2.getResult().message()); - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); } @Test @@ -213,7 +213,7 @@ public void testPersist() throws IOException, SchedulerException, RepositoryExce new InterpreterFactory(conf, null, null, null, depResolver, false), this, null, null, null); assertEquals(1, notebook2.getAllNotes().size()); - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); } @Test @@ -226,7 +226,7 @@ public void testCreateNoteWithSubject() throws IOException, SchedulerException, Set owners = new HashSet<>(); owners.add("user1"); assertEquals(owners, notebook.getNotebookAuthorization().getOwners(note.getId())); - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); } @Test @@ -246,7 +246,7 @@ public void testClearParagraphOutput() throws IOException, SchedulerException{ // clear paragraph output/result note.clearParagraphOutput(p1.getId()); assertNull(p1.getResult()); - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); } @Test @@ -317,7 +317,7 @@ public void testSchedule() throws InterruptedException, IOException { assertNotNull(dateFinished); Thread.sleep(1 * 1000); assertEquals(dateFinished, p.getDateFinished()); - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); } @Test @@ -370,7 +370,7 @@ public void testAutoRestartInterpreterAfterSchedule() throws InterruptedExceptio // make sure all paragraph has been executed assertNotNull(p.getDateFinished()); assertNotNull(p2.getDateFinished()); - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); } @Test @@ -407,9 +407,9 @@ public void testExportAndImportNote() throws IOException, CloneNotSupportedExcep Set owners = new HashSet<>(); owners.add("user1"); assertEquals(owners, notebook.getNotebookAuthorization().getOwners(importedNote2.getId())); - notebook.removeNote(note.getId(), null); - notebook.removeNote(importedNote.getId(), null); - notebook.removeNote(importedNote2.getId(), null); + notebook.removeNote(note.getId(), anonymous); + notebook.removeNote(importedNote.getId(), anonymous); + notebook.removeNote(importedNote2.getId(), anonymous); } @Test @@ -441,9 +441,9 @@ public void testCloneNote() throws IOException, CloneNotSupportedException, Set owners = new HashSet<>(); owners.add("user1"); assertEquals(owners, notebook.getNotebookAuthorization().getOwners(cloneNote2.getId())); - notebook.removeNote(note.getId(), null); - notebook.removeNote(cloneNote.getId(), null); - notebook.removeNote(cloneNote2.getId(), null); + notebook.removeNote(note.getId(), anonymous); + notebook.removeNote(cloneNote.getId(), anonymous); + notebook.removeNote(cloneNote2.getId(), anonymous); } @Test @@ -454,8 +454,8 @@ public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedExcep Note cloneNote = notebook.cloneNote(note.getId(), null, anonymous); assertEquals(cloneNote.getName(), "Note " + cloneNote.getId()); - notebook.removeNote(note.getId(), null); - notebook.removeNote(cloneNote.getId(), null); + notebook.removeNote(note.getId(), anonymous); + notebook.removeNote(cloneNote.getId(), anonymous); } @Test @@ -480,8 +480,8 @@ public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSuppo assertEquals(cp.getId(), p.getId()); assertEquals(cp.text, p.text); assertNull(cp.getResult()); - notebook.removeNote(note.getId(), null); - notebook.removeNote(cloneNote.getId(), null); + notebook.removeNote(note.getId(), anonymous); + notebook.removeNote(cloneNote.getId(), anonymous); } @Test @@ -575,7 +575,7 @@ public void testAngularObjectRemovalOnParagraphRemove() throws InterruptedExcept // notebook scope and global object sould be remained assertNotNull(registry.get("o2", note.getId(), null)); assertNotNull(registry.get("o3", null, null)); - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); } @Test @@ -685,7 +685,7 @@ public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedExc } assertTrue(isAborted); - notebook.removeNote(note.getId(), null); + notebook.removeNote(note.getId(), anonymous); } @Test @@ -899,8 +899,8 @@ public void testGetAllNotes() throws Exception { assertEquals(0, notebook.getAllNotes(anonymous).size()); assertEquals(1, notebook.getAllNotes(new AuthenticationInfo("user1")).size()); assertEquals(1, notebook.getAllNotes(new AuthenticationInfo("user2")).size()); - notebook.removeNote(note1.getId(), null); - notebook.removeNote(note2.getId(), null); + notebook.removeNote(note1.getId(), anonymous); + notebook.removeNote(note2.getId(), anonymous); } From 48a0d8ecfa25610aea3b913bcd6ab8f0ccb42c02 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Wed, 19 Oct 2016 18:08:13 +0900 Subject: [PATCH 34/34] Fixed ZEPPELIN-1542 Fixed flaky test --- .../java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java | 5 +++-- .../src/main/java/org/apache/zeppelin/notebook/Note.java | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index 0bf30b02cdb..c2606f82494 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -113,7 +113,7 @@ public void testGetNotebookInfo() throws IOException { assertTrue(paragraphs.size() > 0); assertEquals(paragraphText, paragraphs.get(0).get("text")); // - ZeppelinServer.notebook.removeNote(sourceNoteID, null); + ZeppelinServer.notebook.removeNote(sourceNoteID, anonymous); } @Test @@ -426,6 +426,7 @@ public void testGetNotebookJob() throws IOException, InterruptedException { paragraph.setConfig(config); paragraph.setText("%sh sleep 1"); + paragraph.setAuthenticationInfo(anonymous); note.persist(anonymous); String noteID = note.getId(); @@ -511,7 +512,7 @@ public void testRunParagraphWithParams() throws IOException, InterruptedExceptio } @Test - public void testCronJobs() throws InterruptedException, IOException{ + public void testJobs() throws InterruptedException, IOException{ // create a note and a paragraph Note note = ZeppelinServer.notebook.createNote(anonymous); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index de66c602f88..7ad269701f5 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -438,6 +438,8 @@ private Map populateParagraphInfo(Paragraph p) { } if (p.getStatus().isRunning()) { info.put("progress", String.valueOf(p.progress())); + } else { + info.put("progress", String.valueOf(100)); } return info; }