From 8f7b26da3465ced5295c9b6ea8525faae231248d Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Mon, 9 May 2016 19:48:33 +0900 Subject: [PATCH 01/25] implement job manager for webapp. --- zeppelin-web/src/app/app.js | 3 + .../app/jobmanager/jobmanager.controller.js | 722 ++++++++++++++++++ .../src/app/jobmanager/jobmanager.css | 35 + .../src/app/jobmanager/jobmanager.html | 117 +++ .../src/app/jobmanager/jobs/job-control.html | 31 + .../app/jobmanager/jobs/job-progressBar.html | 22 + .../src/app/jobmanager/jobs/job.controller.js | 32 + zeppelin-web/src/app/jobmanager/jobs/job.css | 113 +++ zeppelin-web/src/app/jobmanager/jobs/job.html | 78 ++ .../src/components/navbar/navbar.html | 3 + zeppelin-web/src/index.html | 4 + 11 files changed, 1160 insertions(+) create mode 100644 zeppelin-web/src/app/jobmanager/jobmanager.controller.js create mode 100644 zeppelin-web/src/app/jobmanager/jobmanager.css create mode 100644 zeppelin-web/src/app/jobmanager/jobmanager.html create mode 100644 zeppelin-web/src/app/jobmanager/jobs/job-control.html create mode 100644 zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html create mode 100644 zeppelin-web/src/app/jobmanager/jobs/job.controller.js create mode 100644 zeppelin-web/src/app/jobmanager/jobs/job.css create mode 100644 zeppelin-web/src/app/jobmanager/jobs/job.html diff --git a/zeppelin-web/src/app/app.js b/zeppelin-web/src/app/app.js index 16fad36c4dd..1fb9c17af47 100644 --- a/zeppelin-web/src/app/app.js +++ b/zeppelin-web/src/app/app.js @@ -62,6 +62,9 @@ templateUrl: 'app/notebook/notebook.html', controller: 'NotebookCtrl' }) + .when('/jobmanager', { + templateUrl: 'app/jobmanager/jobmanager.html' + }) .when('/interpreter', { templateUrl: 'app/interpreter/interpreter.html', controller: 'InterpreterCtrl' diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js new file mode 100644 index 00000000000..3656394d787 --- /dev/null +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -0,0 +1,722 @@ +/* jshint loopfunc: true */ +/* global $: false */ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp').controller('NotebookCtrl', + function($scope, $route, $routeParams, $location, $rootScope, $http, + websocketMsgSrv, baseUrlSrv, $timeout, SaveAsService) { + $scope.note = null; + $scope.showEditor = false; + $scope.editorToggled = false; + $scope.tableToggled = false; + $scope.viewOnly = false; + $scope.showSetting = false; + $scope.looknfeelOption = [ 'default', 'simple', 'report']; + $scope.cronOption = [ + {name: 'None', value : undefined}, + {name: '1m', value: '0 0/1 * * * ?'}, + {name: '5m', value: '0 0/5 * * * ?'}, + {name: '1h', value: '0 0 0/1 * * ?'}, + {name: '3h', value: '0 0 0/3 * * ?'}, + {name: '6h', value: '0 0 0/6 * * ?'}, + {name: '12h', value: '0 0 0/12 * * ?'}, + {name: '1d', value: '0 0 0 * * ?'} + ]; + + $scope.interpreterSettings = []; + $scope.interpreterBindings = []; + $scope.isNoteDirty = null; + $scope.saveTimer = null; + + var connectedOnce = false; + + $scope.$on('setConnectedStatus', function(event, param) { + if(connectedOnce && param){ + initNotebook(); + } + connectedOnce = true; + }); + + $scope.getCronOptionNameFromValue = function(value) { + if (!value) { + return ''; + } + + for (var o in $scope.cronOption) { + if ($scope.cronOption[o].value===value) { + return $scope.cronOption[o].name; + } + } + return value; + }; + + /** Init the new controller */ + var initNotebook = function() { + websocketMsgSrv.getNotebook($routeParams.noteId); + + var currentRoute = $route.current; + + if (currentRoute) { + setTimeout( + function() { + var routeParams = currentRoute.params; + var $id = $('#' + routeParams.paragraph + '_container'); + + if ($id.length > 0) { + // adjust for navbar + var top = $id.offset().top - 103; + $('html, body').scrollTo({top: top, left: 0}); + } + + }, + 1000 + ); + } + }; + + initNotebook(); + + + $scope.focusParagraphOnClick = function(clickEvent) { + if (!$scope.note) { + return; + } + for (var i=0; i<$scope.note.paragraphs.length; i++) { + var paragraphId = $scope.note.paragraphs[i].id; + if (jQuery.contains(angular.element('#' + paragraphId + '_container')[0], clickEvent.target)) { + $scope.$broadcast('focusParagraph', paragraphId, 0, true); + break; + } + } + }; + + // register mouseevent handler for focus paragraph + document.addEventListener('click', $scope.focusParagraphOnClick); + + $scope.keyboardShortcut = function(keyEvent) { + // handle keyevent + if (!$scope.viewOnly) { + $scope.$broadcast('keyEvent', keyEvent); + } + }; + + // register mouseevent handler for focus paragraph + document.addEventListener('keydown', $scope.keyboardShortcut); + + + /** Remove the note and go back tot he main page */ + /** TODO(anthony): In the nearly future, go back to the main page and telle to the dude that the note have been remove */ + $scope.removeNote = function(noteId) { + BootstrapDialog.confirm({ + closable: true, + title: '', + message: 'Do you want to delete this notebook?', + callback: function(result) { + if (result) { + websocketMsgSrv.deleteNotebook(noteId); + $location.path('/#'); + } + } + }); + }; + + //Export notebook + $scope.exportNotebook = function() { + var jsonContent = JSON.stringify($scope.note); + SaveAsService.SaveAs(jsonContent, $scope.note.name, 'json'); + }; + + //Clone note + $scope.cloneNote = function(noteId) { + BootstrapDialog.confirm({ + closable: true, + title: '', + message: 'Do you want to clone this notebook?', + callback: function(result) { + if (result) { + websocketMsgSrv.cloneNotebook(noteId); + $location.path('/#'); + } + } + }); + }; + + // checkpoint/commit notebook + $scope.checkpointNotebook = function(commitMessage) { + BootstrapDialog.confirm({ + closable: true, + title: '', + message: 'Commit notebook to current repository?', + callback: function(result) { + if (result) { + websocketMsgSrv.checkpointNotebook($routeParams.noteId, commitMessage); + } + } + }); + document.getElementById('note.checkpoint.message').value=''; + }; + + $scope.runNote = function() { + BootstrapDialog.confirm({ + closable: true, + title: '', + message: 'Run all paragraphs?', + callback: function(result) { + if (result) { + _.forEach($scope.note.paragraphs, function (n, key) { + angular.element('#' + n.id + '_paragraphColumn_main').scope().runParagraph(n.text); + }); + } + } + }); + }; + + $scope.saveNote = function() { + if ($scope.note && $scope.note.paragraphs) { + _.forEach($scope.note.paragraphs, function(n, key) { + angular.element('#' + n.id + '_paragraphColumn_main').scope().saveParagraph(); + }); + $scope.isNoteDirty = null; + } + }; + + $scope.clearAllParagraphOutput = function() { + BootstrapDialog.confirm({ + closable: true, + title: '', + message: 'Do you want to clear all output?', + callback: function(result) { + if (result) { + _.forEach($scope.note.paragraphs, function(n, key) { + angular.element('#' + n.id + '_paragraphColumn_main').scope().clearParagraphOutput(); + }); + } + } + }); + }; + + $scope.toggleAllEditor = function() { + if ($scope.editorToggled) { + $scope.$broadcast('openEditor'); + } else { + $scope.$broadcast('closeEditor'); + } + $scope.editorToggled = !$scope.editorToggled; + }; + + $scope.showAllEditor = function() { + $scope.$broadcast('openEditor'); + }; + + $scope.hideAllEditor = function() { + $scope.$broadcast('closeEditor'); + }; + + $scope.toggleAllTable = function() { + if ($scope.tableToggled) { + $scope.$broadcast('openTable'); + } else { + $scope.$broadcast('closeTable'); + } + $scope.tableToggled = !$scope.tableToggled; + }; + + $scope.showAllTable = function() { + $scope.$broadcast('openTable'); + }; + + $scope.hideAllTable = function() { + $scope.$broadcast('closeTable'); + }; + + $scope.isNoteRunning = function() { + var running = false; + if(!$scope.note){ return false; } + for (var i=0; i<$scope.note.paragraphs.length; i++) { + if ( $scope.note.paragraphs[i].status === 'PENDING' || $scope.note.paragraphs[i].status === 'RUNNING') { + running = true; + break; + } + } + return running; + }; + + $scope.killSaveTimer = function() { + if ($scope.saveTimer) { + $timeout.cancel($scope.saveTimer); + $scope.saveTimer = null; + } + }; + + $scope.startSaveTimer = function() { + $scope.killSaveTimer(); + $scope.isNoteDirty = true; + //console.log('startSaveTimer called ' + $scope.note.id); + $scope.saveTimer = $timeout(function(){ + $scope.saveNote(); + }, 10000); + }; + + angular.element(window).on('beforeunload', function(e) { + $scope.killSaveTimer(); + $scope.saveNote(); + }); + + $scope.$on('$destroy', function() { + angular.element(window).off('beforeunload'); + $scope.killSaveTimer(); + $scope.saveNote(); + + document.removeEventListener('click', $scope.focusParagraphOnClick); + document.removeEventListener('keydown', $scope.keyboardShortcut); + }); + + $scope.setLookAndFeel = function(looknfeel) { + $scope.note.config.looknfeel = looknfeel; + $scope.setConfig(); + }; + + /** Set cron expression for this note **/ + $scope.setCronScheduler = function(cronExpr) { + $scope.note.config.cron = cronExpr; + $scope.setConfig(); + }; + + /** Set release resource for this note **/ + $scope.setReleaseResource = function(value) { + $scope.note.config.releaseresource = value; + $scope.setConfig(); + }; + + /** Update note config **/ + $scope.setConfig = function(config) { + if(config) { + $scope.note.config = config; + } + websocketMsgSrv.updateNotebook($scope.note.id, $scope.note.name, $scope.note.config); + }; + + /** Update the note name */ + $scope.sendNewName = function() { + if ($scope.note.name) { + websocketMsgSrv.updateNotebook($scope.note.id, $scope.note.name, $scope.note.config); + } + }; + + /** update the current note */ + $scope.$on('setNoteContent', function(event, note) { + $scope.paragraphUrl = $routeParams.paragraphId; + $scope.asIframe = $routeParams.asIframe; + if ($scope.paragraphUrl) { + note = cleanParagraphExcept($scope.paragraphUrl, note); + $rootScope.$broadcast('setIframe', $scope.asIframe); + } + + if ($scope.note === null) { + $scope.note = note; + } else { + updateNote(note); + } + initializeLookAndFeel(); + //open interpreter binding setting when there're none selected + getInterpreterBindings(getInterpreterBindingsCallBack); + }); + + + var initializeLookAndFeel = function() { + if (!$scope.note.config.looknfeel) { + $scope.note.config.looknfeel = 'default'; + } else { + $scope.viewOnly = $scope.note.config.looknfeel === 'report' ? true : false; + } + $rootScope.$broadcast('setLookAndFeel', $scope.note.config.looknfeel); + }; + + var cleanParagraphExcept = function(paragraphId, note) { + var noteCopy = {}; + noteCopy.id = note.id; + noteCopy.name = note.name; + noteCopy.config = note.config; + noteCopy.info = note.info; + noteCopy.paragraphs = []; + for (var i=0; i $scope.note.paragraphs.length) { + return; + } + websocketMsgSrv.insertParagraph(newIndex); + }); + + $scope.$on('moveParagraphUp', function(event, paragraphId) { + var newIndex = -1; + for (var i=0; i<$scope.note.paragraphs.length; i++) { + if ($scope.note.paragraphs[i].id === paragraphId) { + newIndex = i-1; + break; + } + } + if (newIndex<0 || newIndex>=$scope.note.paragraphs.length) { + return; + } + // save dirtyText of moving paragraphs. + var prevParagraphId = $scope.note.paragraphs[newIndex].id; + angular.element('#' + paragraphId + '_paragraphColumn_main').scope().saveParagraph(); + angular.element('#' + prevParagraphId + '_paragraphColumn_main').scope().saveParagraph(); + websocketMsgSrv.moveParagraph(paragraphId, newIndex); + }); + + $scope.$on('moveParagraphDown', function(event, paragraphId) { + var newIndex = -1; + for (var i=0; i<$scope.note.paragraphs.length; i++) { + if ($scope.note.paragraphs[i].id === paragraphId) { + newIndex = i+1; + break; + } + } + + if (newIndex<0 || newIndex>=$scope.note.paragraphs.length) { + return; + } + // save dirtyText of moving paragraphs. + var nextParagraphId = $scope.note.paragraphs[newIndex].id; + angular.element('#' + paragraphId + '_paragraphColumn_main').scope().saveParagraph(); + angular.element('#' + nextParagraphId + '_paragraphColumn_main').scope().saveParagraph(); + websocketMsgSrv.moveParagraph(paragraphId, newIndex); + }); + + $scope.$on('moveFocusToPreviousParagraph', function(event, currentParagraphId){ + var focus = false; + for (var i=$scope.note.paragraphs.length-1; i>=0; i--) { + if (focus === false ) { + if ($scope.note.paragraphs[i].id === currentParagraphId) { + focus = true; + continue; + } + } else { + $scope.$broadcast('focusParagraph', $scope.note.paragraphs[i].id, -1); + break; + } + } + }); + + $scope.$on('moveFocusToNextParagraph', function(event, currentParagraphId){ + var focus = false; + for (var i=0; i<$scope.note.paragraphs.length; i++) { + if (focus === false ) { + if ($scope.note.paragraphs[i].id === currentParagraphId) { + focus = true; + continue; + } + } else { + $scope.$broadcast('focusParagraph', $scope.note.paragraphs[i].id, 0); + break; + } + } + }); + + var updateNote = function(note) { + /** update Note name */ + if (note.name !== $scope.note.name) { + console.log('change note name: %o to %o', $scope.note.name, note.name); + $scope.note.name = note.name; + } + + $scope.note.config = note.config; + $scope.note.info = note.info; + + var newParagraphIds = note.paragraphs.map(function(x) {return x.id;}); + var oldParagraphIds = $scope.note.paragraphs.map(function(x) {return x.id;}); + + var numNewParagraphs = newParagraphIds.length; + var numOldParagraphs = oldParagraphIds.length; + + var paragraphToBeFocused; + var focusedParagraph; + for (var i=0; i<$scope.note.paragraphs.length; i++) { + var paragraphId = $scope.note.paragraphs[i].id; + if (angular.element('#' + paragraphId + '_paragraphColumn_main').scope().paragraphFocused) { + focusedParagraph = paragraphId; + break; + } + } + + /** add a new paragraph */ + if (numNewParagraphs > numOldParagraphs) { + for (var index in newParagraphIds) { + if (oldParagraphIds[index] !== newParagraphIds[index]) { + $scope.note.paragraphs.splice(index, 0, note.paragraphs[index]); + paragraphToBeFocused = note.paragraphs[index].id; + break; + } + $scope.$broadcast('updateParagraph', { + note: $scope.note, // pass the note object to paragraph scope + paragraph: note.paragraphs[index]}); + } + } + + /** update or move paragraph */ + if (numNewParagraphs === numOldParagraphs) { + for (var idx in newParagraphIds) { + var newEntry = note.paragraphs[idx]; + if (oldParagraphIds[idx] === newParagraphIds[idx]) { + $scope.$broadcast('updateParagraph', { + note: $scope.note, // pass the note object to paragraph scope + paragraph: newEntry}); + } else { + // move paragraph + var oldIdx = oldParagraphIds.indexOf(newParagraphIds[idx]); + $scope.note.paragraphs.splice(oldIdx, 1); + $scope.note.paragraphs.splice(idx, 0, newEntry); + // rebuild id list since paragraph has moved. + oldParagraphIds = $scope.note.paragraphs.map(function(x) {return x.id;}); + } + + if (focusedParagraph === newParagraphIds[idx]) { + paragraphToBeFocused = focusedParagraph; + } + } + } + + /** remove paragraph */ + if (numNewParagraphs < numOldParagraphs) { + for (var oldidx in oldParagraphIds) { + if(oldParagraphIds[oldidx] !== newParagraphIds[oldidx]) { + $scope.note.paragraphs.splice(oldidx, 1); + break; + } + } + } + + // restore focus of paragraph + for (var f=0; f<$scope.note.paragraphs.length; f++) { + if (paragraphToBeFocused === $scope.note.paragraphs[f].id) { + $scope.note.paragraphs[f].focus = true; + } + } + }; + + var getInterpreterBindings = function(callback) { + $http.get(baseUrlSrv.getRestApiBase()+ '/notebook/interpreter/bind/' +$scope.note.id). + success(function(data, status, headers, config) { + $scope.interpreterBindings = data.body; + $scope.interpreterBindingsOrig = angular.copy($scope.interpreterBindings); // to check dirty + if (callback) { + callback(); + } + }). + error(function(data, status, headers, config) { + if (status !== 0) { + console.log('Error %o %o', status, data.message); + } + }); + }; + + var getInterpreterBindingsCallBack = function() { + var selected = false; + var key; + var setting; + + for (key in $scope.interpreterBindings) { + setting = $scope.interpreterBindings[key]; + if (setting.selected) { + selected = true; + break; + } + } + + if (!selected) { + // make default selection + var selectedIntp = {}; + for (key in $scope.interpreterBindings) { + setting = $scope.interpreterBindings[key]; + if (!selectedIntp[setting.group]) { + setting.selected = true; + selectedIntp[setting.group] = true; + } + } + $scope.showSetting = true; + } + }; + + $scope.interpreterSelectionListeners = { + accept : function(sourceItemHandleScope, destSortableScope) {return true;}, + itemMoved: function (event) {}, + orderChanged: function(event) {} + }; + + $scope.openSetting = function() { + $scope.showSetting = true; + getInterpreterBindings(); + }; + + $scope.closeSetting = function() { + if (isSettingDirty()) { + BootstrapDialog.confirm({ + closable: true, + title: '', + message: 'Changes will be discarded.', + callback: function(result) { + if (result) { + $scope.$apply(function() { + $scope.showSetting = false; + }); + } + } + }); + } else { + $scope.showSetting = false; + } + }; + + $scope.saveSetting = function() { + var selectedSettingIds = []; + for (var no in $scope.interpreterBindings) { + var setting = $scope.interpreterBindings[no]; + if (setting.selected) { + selectedSettingIds.push(setting.id); + } + } + + $http.put(baseUrlSrv.getRestApiBase() + '/notebook/interpreter/bind/' + $scope.note.id, + selectedSettingIds). + success(function(data, status, headers, config) { + console.log('Interpreter binding %o saved', selectedSettingIds); + $scope.showSetting = false; + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + }); + }; + + $scope.toggleSetting = function() { + if ($scope.showSetting) { + $scope.closeSetting(); + } else { + $scope.openSetting(); + } + }; + + var getPermissions = function(callback) { + $http.get(baseUrlSrv.getRestApiBase()+ '/notebook/' +$scope.note.id + '/permissions'). + success(function(data, status, headers, config) { + $scope.permissions = data.body; + $scope.permissionsOrig = angular.copy($scope.permissions); // to check dirty + if (callback) { + callback(); + } + }). + error(function(data, status, headers, config) { + if (status !== 0) { + console.log('Error %o %o', status, data.message); + } + }); + }; + + $scope.openPermissions = function() { + $scope.showPermissions = true; + getPermissions(); + }; + + + $scope.closePermissions = function() { + if (isPermissionsDirty()) { + BootstrapDialog.confirm({ + closable: true, + title: '', + message: 'Changes will be discarded.', + callback: function(result) { + if (result) { + $scope.$apply(function() { + $scope.showPermissions = false; + }); + } + } + }); + } else { + $scope.showPermissions = false; + } + }; + + $scope.savePermissions = function() { + $http.put(baseUrlSrv.getRestApiBase() + '/notebook/' +$scope.note.id + '/permissions', + $scope.permissions, {withCredentials: true}). + success(function(data, status, headers, config) { + console.log('Note permissions %o saved', $scope.permissions); + $scope.showPermissions = false; + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + BootstrapDialog.alert({ + closable: true, + title: 'Insufficient privileges', + message: data.message + }); + }); + }; + + $scope.togglePermissions = function() { + if ($scope.showPermissions) { + $scope.closePermissions(); + } else { + $scope.openPermissions(); + } + }; + + + var isSettingDirty = function() { + if (angular.equals($scope.interpreterBindings, $scope.interpreterBindingsOrig)) { + return false; + } else { + return true; + } + }; + + var isPermissionsDirty = function() { + if (angular.equals($scope.permissions, $scope.permissionsOrig)) { + return false; + } else { + return true; + } + }; + +}); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.css b/zeppelin-web/src/app/jobmanager/jobmanager.css new file mode 100644 index 00000000000..a09f91dce87 --- /dev/null +++ b/zeppelin-web/src/app/jobmanager/jobmanager.css @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.job-col { + margin: 0; + padding: 0; +} + +.job { + padding: 2px 8px 4px 8px; + min-height: 32px; +} + +.jobManagerHead { + margin: -10px -10px 20px; + padding: 10px 15px 15px 15px; + background: white; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.15); + border-bottom: 1px solid #E5E5E5; +} + +.jobManagerHead .header { + font-family: 'Roboto', sans-serif; +} diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.html b/zeppelin-web/src/app/jobmanager/jobmanager.html new file mode 100644 index 00000000000..62312543de4 --- /dev/null +++ b/zeppelin-web/src/app/jobmanager/jobmanager.html @@ -0,0 +1,117 @@ + + +
+
+
+
+

+ Job +

+
+
+
+
+ You can monitor the written notebook. Check the status of the Notebook and can control the action. +
+
+
+
+
+ + + + + {{jobStatus}} + + + + {{jobStatus}} + + + + + {{jobStatus}} + + + + + {{jobStatus}} + + + + + {{jobStatus}} + + + + + {{jobStatus}} + + +
+
+
+
+
+
+
+
+
+ diff --git a/zeppelin-web/src/app/jobmanager/jobs/job-control.html b/zeppelin-web/src/app/jobmanager/jobs/job-control.html new file mode 100644 index 00000000000..dd6f08faa4a --- /dev/null +++ b/zeppelin-web/src/app/jobmanager/jobs/job-control.html @@ -0,0 +1,31 @@ + + +
+ + {{notebook.status}} + + + + {{getProgress()}}% + + + + + + + +
diff --git a/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html b/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html new file mode 100644 index 00000000000..8520a114e8f --- /dev/null +++ b/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html @@ -0,0 +1,22 @@ + + +
+
+
+
+
+
diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js new file mode 100644 index 00000000000..b70a68e2a0f --- /dev/null +++ b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js @@ -0,0 +1,32 @@ +/*jshint loopfunc: true, unused:false */ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp') + .controller('JobCtrl', function($scope,$rootScope, $route, $window, $element, $routeParams, $location, + $timeout, $compile, websocketMsgSrv, ngToast) { + + $scope.init = function () { + $scope.jobType = 'cron'; + //$scope.jobType = 'normal' + $scope.notebook = {}; + $scope.notebook.status = 'RUNNING'; + }; + + $scope.getProgress = function () { + return 100; + } + +}); diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.css b/zeppelin-web/src/app/jobmanager/jobs/job.css new file mode 100644 index 00000000000..15479a0689f --- /dev/null +++ b/zeppelin-web/src/app/jobmanager/jobs/job.css @@ -0,0 +1,113 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + job Style +*/ + +.job-space { + margin-bottom: 5px !important; + padding: 10px 10px 10px 10px !important; + min-height: 30px; +} + +.job-margin { + margin-right: 2px; + margin-left: 2px; +} + +.job-types i { + font-weight: bold; + font-size: 10px; +} + + +/* + job Controls CSS +*/ + +.job .runControl { + font-size: 1px; + color: #AAAAAA; + height:4px; + margin: 0px 0px 0px 0px; +} + +.job .runControl .progress { + position: relative; + width: 100%; + height: 4px; + z-index: 100; + border-radius: 0; +} + +.job .control span { + margin-left: 4px; +} + +.job .control { + background: rgba(255,255,255,0.85); + float: right; + color: #999; + margin-top: 1px; + margin-right: 5px; + position: absolute; + clear: both; + right: 15px; + text-align: right; + font-size: 12px; + padding: 4px; +} + +.job .control li { + font-size: 12px; + margin-bottom: 4px; + color: #333333; +} + +.job .control .tooltip { + z-index: 10003; +} + +@-webkit-keyframes spinnerRotateAnimation +{ + from{-webkit-transform:rotate(0deg);} + to{-webkit-transform:rotate(360deg);} +} +@-moz-keyframes spinnerRotateAnimation +{ + from{-moz-transform:rotate(0deg);} + to{-moz-transform:rotate(360deg);} +} +@-ms-keyframes spinnerRotateAnimation +{ + from{-ms-transform:rotate(0deg);} + to{-ms-transform:rotate(360deg);} +} + +.spinAnimation{ + -webkit-animation-name: spinnerRotateAnimation; + -webkit-animation-duration: 1s; + -webkit-animation-iteration-count: infinite; + -webkit-animation-timing-function: linear; + -moz-animation-name: spinnerRotateAnimation; + -moz-animation-duration: 1s; + -moz-animation-iteration-count: infinite; + -moz-animation-timing-function: linear; + -ms-animation-name: spinnerRotateAnimation; + -ms-animation-duration: 1s; + -ms-animation-iteration-count: infinite; + -ms-animation-timing-function: linear; +} + diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.html b/zeppelin-web/src/app/jobmanager/jobs/job.html new file mode 100644 index 00000000000..b675a559a89 --- /dev/null +++ b/zeppelin-web/src/app/jobmanager/jobs/job.html @@ -0,0 +1,78 @@ + + +
+
+
+ + + + + +   + + Notebook Title + + +  -  + + + interpreter name + +
+
+ +
+ + + + + + + + + + + + + + +
+
diff --git a/zeppelin-web/src/components/navbar/navbar.html b/zeppelin-web/src/components/navbar/navbar.html index 272167ae6cb..b6254d26d96 100644 --- a/zeppelin-web/src/components/navbar/navbar.html +++ b/zeppelin-web/src/components/navbar/navbar.html @@ -49,6 +49,9 @@ +
  • + Job +
  • Interpreter
    • diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index 207320ead1c..3ec98633842 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -53,6 +53,8 @@ + + @@ -149,6 +151,8 @@ + + From 2fb5f7a7049f125afc6cd12cc91aceb66e8a59e1 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Tue, 10 May 2016 17:47:15 +0900 Subject: [PATCH 02/25] implement logic web for jobmanger. --- zeppelin-web/src/app/app.js | 7 +- .../app/jobmanager/jobmanager.controller.js | 957 +++++------------- .../src/app/jobmanager/jobmanager.html | 19 +- .../src/app/jobmanager/jobs/job-control.html | 27 +- .../app/jobmanager/jobs/job-progressBar.html | 8 +- .../src/app/jobmanager/jobs/job.controller.js | 17 +- zeppelin-web/src/app/jobmanager/jobs/job.html | 107 +- 7 files changed, 360 insertions(+), 782 deletions(-) diff --git a/zeppelin-web/src/app/app.js b/zeppelin-web/src/app/app.js index 1fb9c17af47..0441b1a79a2 100644 --- a/zeppelin-web/src/app/app.js +++ b/zeppelin-web/src/app/app.js @@ -63,15 +63,16 @@ controller: 'NotebookCtrl' }) .when('/jobmanager', { - templateUrl: 'app/jobmanager/jobmanager.html' + templateUrl: 'app/jobmanager/jobmanager.html', + controller: 'JobmanagerCtrl' }) .when('/interpreter', { templateUrl: 'app/interpreter/interpreter.html', controller: 'InterpreterCtrl' }) .when('/configuration', { - templateUrl: 'app/configuration/configuration.html', - controller: 'ConfigurationCtrl' + templateUrl: 'app/configuration/configuration.html', + controller: 'ConfigurationCtrl' }) .when('/search/:searchTerm', { templateUrl: 'app/search/result-list.html', diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 3656394d787..7dc4dff56ae 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -1,5 +1,4 @@ -/* jshint loopfunc: true */ -/* global $: false */ +/*jshint loopfunc: true, unused:false */ /* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,708 +14,260 @@ */ 'use strict'; -angular.module('zeppelinWebApp').controller('NotebookCtrl', - function($scope, $route, $routeParams, $location, $rootScope, $http, - websocketMsgSrv, baseUrlSrv, $timeout, SaveAsService) { - $scope.note = null; - $scope.showEditor = false; - $scope.editorToggled = false; - $scope.tableToggled = false; - $scope.viewOnly = false; - $scope.showSetting = false; - $scope.looknfeelOption = [ 'default', 'simple', 'report']; - $scope.cronOption = [ - {name: 'None', value : undefined}, - {name: '1m', value: '0 0/1 * * * ?'}, - {name: '5m', value: '0 0/5 * * * ?'}, - {name: '1h', value: '0 0 0/1 * * ?'}, - {name: '3h', value: '0 0 0/3 * * ?'}, - {name: '6h', value: '0 0 0/6 * * ?'}, - {name: '12h', value: '0 0 0/12 * * ?'}, - {name: '1d', value: '0 0 0 * * ?'} - ]; - - $scope.interpreterSettings = []; - $scope.interpreterBindings = []; - $scope.isNoteDirty = null; - $scope.saveTimer = null; - - var connectedOnce = false; - - $scope.$on('setConnectedStatus', function(event, param) { - if(connectedOnce && param){ - initNotebook(); - } - connectedOnce = true; - }); - - $scope.getCronOptionNameFromValue = function(value) { - if (!value) { - return ''; - } - - for (var o in $scope.cronOption) { - if ($scope.cronOption[o].value===value) { - return $scope.cronOption[o].name; - } - } - return value; - }; - - /** Init the new controller */ - var initNotebook = function() { - websocketMsgSrv.getNotebook($routeParams.noteId); - - var currentRoute = $route.current; - - if (currentRoute) { - setTimeout( - function() { - var routeParams = currentRoute.params; - var $id = $('#' + routeParams.paragraph + '_container'); - - if ($id.length > 0) { - // adjust for navbar - var top = $id.offset().top - 103; - $('html, body').scrollTo({top: top, left: 0}); +angular.module('zeppelinWebApp') + .controller('JobmanagerCtrl', + function($scope, $route, $routeParams, $location, $rootScope, $http, + websocketMsgSrv, baseUrlSrv, $timeout, SaveAsService) { + + $scope.debugPrint = function (value) { + console.log(value); + }; + + $scope.init = function () { + $scope.FILTER_VALUE_INTERPRETER = /.*/; + $scope.ACTIVE_INTERPRETERS = [ + { + name : 'ALL', + value : /.*/ } - - }, - 1000 - ); - } - }; - - initNotebook(); - - - $scope.focusParagraphOnClick = function(clickEvent) { - if (!$scope.note) { - return; - } - for (var i=0; i<$scope.note.paragraphs.length; i++) { - var paragraphId = $scope.note.paragraphs[i].id; - if (jQuery.contains(angular.element('#' + paragraphId + '_container')[0], clickEvent.target)) { - $scope.$broadcast('focusParagraph', paragraphId, 0, true); - break; - } - } - }; - - // register mouseevent handler for focus paragraph - document.addEventListener('click', $scope.focusParagraphOnClick); - - $scope.keyboardShortcut = function(keyEvent) { - // handle keyevent - if (!$scope.viewOnly) { - $scope.$broadcast('keyEvent', keyEvent); - } - }; - - // register mouseevent handler for focus paragraph - document.addEventListener('keydown', $scope.keyboardShortcut); - - - /** Remove the note and go back tot he main page */ - /** TODO(anthony): In the nearly future, go back to the main page and telle to the dude that the note have been remove */ - $scope.removeNote = function(noteId) { - BootstrapDialog.confirm({ - closable: true, - title: '', - message: 'Do you want to delete this notebook?', - callback: function(result) { - if (result) { - websocketMsgSrv.deleteNotebook(noteId); - $location.path('/#'); - } - } - }); - }; - - //Export notebook - $scope.exportNotebook = function() { - var jsonContent = JSON.stringify($scope.note); - SaveAsService.SaveAs(jsonContent, $scope.note.name, 'json'); - }; - - //Clone note - $scope.cloneNote = function(noteId) { - BootstrapDialog.confirm({ - closable: true, - title: '', - message: 'Do you want to clone this notebook?', - callback: function(result) { - if (result) { - websocketMsgSrv.cloneNotebook(noteId); - $location.path('/#'); - } - } - }); - }; - - // checkpoint/commit notebook - $scope.checkpointNotebook = function(commitMessage) { - BootstrapDialog.confirm({ - closable: true, - title: '', - message: 'Commit notebook to current repository?', - callback: function(result) { - if (result) { - websocketMsgSrv.checkpointNotebook($routeParams.noteId, commitMessage); - } - } - }); - document.getElementById('note.checkpoint.message').value=''; - }; - - $scope.runNote = function() { - BootstrapDialog.confirm({ - closable: true, - title: '', - message: 'Run all paragraphs?', - callback: function(result) { - if (result) { - _.forEach($scope.note.paragraphs, function (n, key) { - angular.element('#' + n.id + '_paragraphColumn_main').scope().runParagraph(n.text); - }); + ]; + $scope.jobInfomations = initTestPacket(); + + var interpreterLists = _.pluck($scope.jobInfomations, 'interpreter'); + for (var index = 0, length = interpreterLists.length; index < length; index++) { + $scope.ACTIVE_INTERPRETERS.push({ + name : interpreterLists[index], + value : interpreterLists[index] + }) } - } - }); - }; - - $scope.saveNote = function() { - if ($scope.note && $scope.note.paragraphs) { - _.forEach($scope.note.paragraphs, function(n, key) { - angular.element('#' + n.id + '_paragraphColumn_main').scope().saveParagraph(); - }); - $scope.isNoteDirty = null; - } - }; - - $scope.clearAllParagraphOutput = function() { - BootstrapDialog.confirm({ - closable: true, - title: '', - message: 'Do you want to clear all output?', - callback: function(result) { - if (result) { - _.forEach($scope.note.paragraphs, function(n, key) { - angular.element('#' + n.id + '_paragraphColumn_main').scope().clearParagraphOutput(); - }); - } - } - }); - }; - - $scope.toggleAllEditor = function() { - if ($scope.editorToggled) { - $scope.$broadcast('openEditor'); - } else { - $scope.$broadcast('closeEditor'); - } - $scope.editorToggled = !$scope.editorToggled; - }; - - $scope.showAllEditor = function() { - $scope.$broadcast('openEditor'); - }; - - $scope.hideAllEditor = function() { - $scope.$broadcast('closeEditor'); - }; - - $scope.toggleAllTable = function() { - if ($scope.tableToggled) { - $scope.$broadcast('openTable'); - } else { - $scope.$broadcast('closeTable'); - } - $scope.tableToggled = !$scope.tableToggled; - }; - - $scope.showAllTable = function() { - $scope.$broadcast('openTable'); - }; - - $scope.hideAllTable = function() { - $scope.$broadcast('closeTable'); - }; - - $scope.isNoteRunning = function() { - var running = false; - if(!$scope.note){ return false; } - for (var i=0; i<$scope.note.paragraphs.length; i++) { - if ( $scope.note.paragraphs[i].status === 'PENDING' || $scope.note.paragraphs[i].status === 'RUNNING') { - running = true; - break; - } - } - return running; - }; - - $scope.killSaveTimer = function() { - if ($scope.saveTimer) { - $timeout.cancel($scope.saveTimer); - $scope.saveTimer = null; - } - }; - - $scope.startSaveTimer = function() { - $scope.killSaveTimer(); - $scope.isNoteDirty = true; - //console.log('startSaveTimer called ' + $scope.note.id); - $scope.saveTimer = $timeout(function(){ - $scope.saveNote(); - }, 10000); - }; - - angular.element(window).on('beforeunload', function(e) { - $scope.killSaveTimer(); - $scope.saveNote(); - }); - - $scope.$on('$destroy', function() { - angular.element(window).off('beforeunload'); - $scope.killSaveTimer(); - $scope.saveNote(); - - document.removeEventListener('click', $scope.focusParagraphOnClick); - document.removeEventListener('keydown', $scope.keyboardShortcut); - }); - - $scope.setLookAndFeel = function(looknfeel) { - $scope.note.config.looknfeel = looknfeel; - $scope.setConfig(); - }; - - /** Set cron expression for this note **/ - $scope.setCronScheduler = function(cronExpr) { - $scope.note.config.cron = cronExpr; - $scope.setConfig(); - }; - - /** Set release resource for this note **/ - $scope.setReleaseResource = function(value) { - $scope.note.config.releaseresource = value; - $scope.setConfig(); - }; - - /** Update note config **/ - $scope.setConfig = function(config) { - if(config) { - $scope.note.config = config; - } - websocketMsgSrv.updateNotebook($scope.note.id, $scope.note.name, $scope.note.config); - }; - - /** Update the note name */ - $scope.sendNewName = function() { - if ($scope.note.name) { - websocketMsgSrv.updateNotebook($scope.note.id, $scope.note.name, $scope.note.config); - } - }; - - /** update the current note */ - $scope.$on('setNoteContent', function(event, note) { - $scope.paragraphUrl = $routeParams.paragraphId; - $scope.asIframe = $routeParams.asIframe; - if ($scope.paragraphUrl) { - note = cleanParagraphExcept($scope.paragraphUrl, note); - $rootScope.$broadcast('setIframe', $scope.asIframe); - } - - if ($scope.note === null) { - $scope.note = note; - } else { - updateNote(note); - } - initializeLookAndFeel(); - //open interpreter binding setting when there're none selected - getInterpreterBindings(getInterpreterBindingsCallBack); - }); - - - var initializeLookAndFeel = function() { - if (!$scope.note.config.looknfeel) { - $scope.note.config.looknfeel = 'default'; - } else { - $scope.viewOnly = $scope.note.config.looknfeel === 'report' ? true : false; - } - $rootScope.$broadcast('setLookAndFeel', $scope.note.config.looknfeel); - }; - - var cleanParagraphExcept = function(paragraphId, note) { - var noteCopy = {}; - noteCopy.id = note.id; - noteCopy.name = note.name; - noteCopy.config = note.config; - noteCopy.info = note.info; - noteCopy.paragraphs = []; - for (var i=0; i $scope.note.paragraphs.length) { - return; - } - websocketMsgSrv.insertParagraph(newIndex); - }); - - $scope.$on('moveParagraphUp', function(event, paragraphId) { - var newIndex = -1; - for (var i=0; i<$scope.note.paragraphs.length; i++) { - if ($scope.note.paragraphs[i].id === paragraphId) { - newIndex = i-1; - break; - } - } - if (newIndex<0 || newIndex>=$scope.note.paragraphs.length) { - return; - } - // save dirtyText of moving paragraphs. - var prevParagraphId = $scope.note.paragraphs[newIndex].id; - angular.element('#' + paragraphId + '_paragraphColumn_main').scope().saveParagraph(); - angular.element('#' + prevParagraphId + '_paragraphColumn_main').scope().saveParagraph(); - websocketMsgSrv.moveParagraph(paragraphId, newIndex); - }); - - $scope.$on('moveParagraphDown', function(event, paragraphId) { - var newIndex = -1; - for (var i=0; i<$scope.note.paragraphs.length; i++) { - if ($scope.note.paragraphs[i].id === paragraphId) { - newIndex = i+1; - break; - } - } - - if (newIndex<0 || newIndex>=$scope.note.paragraphs.length) { - return; - } - // save dirtyText of moving paragraphs. - var nextParagraphId = $scope.note.paragraphs[newIndex].id; - angular.element('#' + paragraphId + '_paragraphColumn_main').scope().saveParagraph(); - angular.element('#' + nextParagraphId + '_paragraphColumn_main').scope().saveParagraph(); - websocketMsgSrv.moveParagraph(paragraphId, newIndex); - }); - - $scope.$on('moveFocusToPreviousParagraph', function(event, currentParagraphId){ - var focus = false; - for (var i=$scope.note.paragraphs.length-1; i>=0; i--) { - if (focus === false ) { - if ($scope.note.paragraphs[i].id === currentParagraphId) { - focus = true; - continue; - } - } else { - $scope.$broadcast('focusParagraph', $scope.note.paragraphs[i].id, -1); - break; - } - } - }); - - $scope.$on('moveFocusToNextParagraph', function(event, currentParagraphId){ - var focus = false; - for (var i=0; i<$scope.note.paragraphs.length; i++) { - if (focus === false ) { - if ($scope.note.paragraphs[i].id === currentParagraphId) { - focus = true; - continue; - } - } else { - $scope.$broadcast('focusParagraph', $scope.note.paragraphs[i].id, 0); - break; - } - } - }); - - var updateNote = function(note) { - /** update Note name */ - if (note.name !== $scope.note.name) { - console.log('change note name: %o to %o', $scope.note.name, note.name); - $scope.note.name = note.name; - } - - $scope.note.config = note.config; - $scope.note.info = note.info; - - var newParagraphIds = note.paragraphs.map(function(x) {return x.id;}); - var oldParagraphIds = $scope.note.paragraphs.map(function(x) {return x.id;}); - - var numNewParagraphs = newParagraphIds.length; - var numOldParagraphs = oldParagraphIds.length; - - var paragraphToBeFocused; - var focusedParagraph; - for (var i=0; i<$scope.note.paragraphs.length; i++) { - var paragraphId = $scope.note.paragraphs[i].id; - if (angular.element('#' + paragraphId + '_paragraphColumn_main').scope().paragraphFocused) { - focusedParagraph = paragraphId; - break; - } - } - - /** add a new paragraph */ - if (numNewParagraphs > numOldParagraphs) { - for (var index in newParagraphIds) { - if (oldParagraphIds[index] !== newParagraphIds[index]) { - $scope.note.paragraphs.splice(index, 0, note.paragraphs[index]); - paragraphToBeFocused = note.paragraphs[index].id; - break; - } - $scope.$broadcast('updateParagraph', { - note: $scope.note, // pass the note object to paragraph scope - paragraph: note.paragraphs[index]}); - } - } - - /** update or move paragraph */ - if (numNewParagraphs === numOldParagraphs) { - for (var idx in newParagraphIds) { - var newEntry = note.paragraphs[idx]; - if (oldParagraphIds[idx] === newParagraphIds[idx]) { - $scope.$broadcast('updateParagraph', { - note: $scope.note, // pass the note object to paragraph scope - paragraph: newEntry}); - } else { - // move paragraph - var oldIdx = oldParagraphIds.indexOf(newParagraphIds[idx]); - $scope.note.paragraphs.splice(oldIdx, 1); - $scope.note.paragraphs.splice(idx, 0, newEntry); - // rebuild id list since paragraph has moved. - oldParagraphIds = $scope.note.paragraphs.map(function(x) {return x.id;}); - } - - if (focusedParagraph === newParagraphIds[idx]) { - paragraphToBeFocused = focusedParagraph; - } - } - } - - /** remove paragraph */ - if (numNewParagraphs < numOldParagraphs) { - for (var oldidx in oldParagraphIds) { - if(oldParagraphIds[oldidx] !== newParagraphIds[oldidx]) { - $scope.note.paragraphs.splice(oldidx, 1); - break; - } - } - } - - // restore focus of paragraph - for (var f=0; f<$scope.note.paragraphs.length; f++) { - if (paragraphToBeFocused === $scope.note.paragraphs[f].id) { - $scope.note.paragraphs[f].focus = true; - } - } - }; - - var getInterpreterBindings = function(callback) { - $http.get(baseUrlSrv.getRestApiBase()+ '/notebook/interpreter/bind/' +$scope.note.id). - success(function(data, status, headers, config) { - $scope.interpreterBindings = data.body; - $scope.interpreterBindingsOrig = angular.copy($scope.interpreterBindings); // to check dirty - if (callback) { - callback(); - } - }). - error(function(data, status, headers, config) { - if (status !== 0) { - console.log('Error %o %o', status, data.message); - } - }); - }; - - var getInterpreterBindingsCallBack = function() { - var selected = false; - var key; - var setting; - - for (key in $scope.interpreterBindings) { - setting = $scope.interpreterBindings[key]; - if (setting.selected) { - selected = true; - break; - } - } - - if (!selected) { - // make default selection - var selectedIntp = {}; - for (key in $scope.interpreterBindings) { - setting = $scope.interpreterBindings[key]; - if (!selectedIntp[setting.group]) { - setting.selected = true; - selectedIntp[setting.group] = true; - } - } - $scope.showSetting = true; - } - }; - - $scope.interpreterSelectionListeners = { - accept : function(sourceItemHandleScope, destSortableScope) {return true;}, - itemMoved: function (event) {}, - orderChanged: function(event) {} - }; - - $scope.openSetting = function() { - $scope.showSetting = true; - getInterpreterBindings(); - }; - - $scope.closeSetting = function() { - if (isSettingDirty()) { - BootstrapDialog.confirm({ - closable: true, - title: '', - message: 'Changes will be discarded.', - callback: function(result) { - if (result) { - $scope.$apply(function() { - $scope.showSetting = false; - }); - } - } - }); - } else { - $scope.showSetting = false; - } - }; - - $scope.saveSetting = function() { - var selectedSettingIds = []; - for (var no in $scope.interpreterBindings) { - var setting = $scope.interpreterBindings[no]; - if (setting.selected) { - selectedSettingIds.push(setting.id); - } - } - - $http.put(baseUrlSrv.getRestApiBase() + '/notebook/interpreter/bind/' + $scope.note.id, - selectedSettingIds). - success(function(data, status, headers, config) { - console.log('Interpreter binding %o saved', selectedSettingIds); - $scope.showSetting = false; - }). - error(function(data, status, headers, config) { - console.log('Error %o %o', status, data.message); - }); - }; - - $scope.toggleSetting = function() { - if ($scope.showSetting) { - $scope.closeSetting(); - } else { - $scope.openSetting(); - } - }; - - var getPermissions = function(callback) { - $http.get(baseUrlSrv.getRestApiBase()+ '/notebook/' +$scope.note.id + '/permissions'). - success(function(data, status, headers, config) { - $scope.permissions = data.body; - $scope.permissionsOrig = angular.copy($scope.permissions); // to check dirty - if (callback) { - callback(); - } - }). - error(function(data, status, headers, config) { - if (status !== 0) { - console.log('Error %o %o', status, data.message); - } - }); - }; - - $scope.openPermissions = function() { - $scope.showPermissions = true; - getPermissions(); - }; - - - $scope.closePermissions = function() { - if (isPermissionsDirty()) { - BootstrapDialog.confirm({ - closable: true, - title: '', - message: 'Changes will be discarded.', - callback: function(result) { - if (result) { - $scope.$apply(function() { - $scope.showPermissions = false; - }); + }; + + var initTestPacket = function () { + return [ + { + notebookId: "2BMFSNAAF", + notebookName: "myNotebook", + notebookType: "normal", + interpreter: "mySpark", + isRunningJob: true, + paragraphs: [ + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "FINISHED" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "FINISHED" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "RUNNING" + }, + + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "PENDING" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "PENDING" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + } + ] + }, + { + notebookId: "2BMFSNAAF", + notebookName: "hiveNotebook", + notebookType: "normal", + interpreter: "hive", + isRunningJob: false, + paragraphs: [ + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "FINISHED" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "FINISHED" + }, + + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "ABORT" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "PENDING" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + } + ] + }, + { + notebookId: "2BMFSNAAF", + notebookName: "R Notebook", + notebookType: "cron", + interpreter: "r", + isRunningJob: true, + paragraphs: [ + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "FINISHED" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "FINISHED" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "RUNNING" + }, + + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "PENDING" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "ERROR" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "ERROR" + } + ] + }, + { + notebookId: "2BMFSNAAF", + notebookName: "myNotebook", + notebookType: "normal", + interpreter: "mySpark", + isRunningJob: false, + paragraphs: [ + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + }, + { + id: "20160509-112757_1030139246", + name: "20160509-112757_1030139246", + status: "READY" + } + ] } - } - }); - } else { - $scope.showPermissions = false; - } - }; - - $scope.savePermissions = function() { - $http.put(baseUrlSrv.getRestApiBase() + '/notebook/' +$scope.note.id + '/permissions', - $scope.permissions, {withCredentials: true}). - success(function(data, status, headers, config) { - console.log('Note permissions %o saved', $scope.permissions); - $scope.showPermissions = false; - }). - error(function(data, status, headers, config) { - console.log('Error %o %o', status, data.message); - BootstrapDialog.alert({ - closable: true, - title: 'Insufficient privileges', - message: data.message - }); - }); - }; - - $scope.togglePermissions = function() { - if ($scope.showPermissions) { - $scope.closePermissions(); - } else { - $scope.openPermissions(); - } - }; - - - var isSettingDirty = function() { - if (angular.equals($scope.interpreterBindings, $scope.interpreterBindingsOrig)) { - return false; - } else { - return true; - } - }; - - var isPermissionsDirty = function() { - if (angular.equals($scope.permissions, $scope.permissionsOrig)) { - return false; - } else { - return true; - } - }; - + ]; + }; }); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.html b/zeppelin-web/src/app/jobmanager/jobmanager.html index 62312543de4..c012d6a7dae 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.html +++ b/zeppelin-web/src/app/jobmanager/jobmanager.html @@ -11,8 +11,8 @@ See the License for the specific language governing permissions and limitations under the License. --> - -
    + +
    @@ -36,10 +36,9 @@

    - {{jobStatus}} @@ -101,15 +100,21 @@

    +
    + +
    -
    diff --git a/zeppelin-web/src/app/jobmanager/jobs/job-control.html b/zeppelin-web/src/app/jobmanager/jobs/job-control.html index dd6f08faa4a..fdc0160bc32 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job-control.html +++ b/zeppelin-web/src/app/jobmanager/jobs/job-control.html @@ -12,20 +12,29 @@ limitations under the License. --> -
    +
    - {{notebook.status}} + + RUNNING + + + READY + - + {{getProgress()}}% - - - - + + + +
    diff --git a/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html b/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html index 8520a114e8f..d6b49d3038f 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html +++ b/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html @@ -12,11 +12,11 @@ limitations under the License. --> -
    -
    -
    +
    +
    -
    diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js index b70a68e2a0f..fe74fa47f16 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js @@ -16,17 +16,18 @@ angular.module('zeppelinWebApp') .controller('JobCtrl', function($scope,$rootScope, $route, $window, $element, $routeParams, $location, - $timeout, $compile, websocketMsgSrv, ngToast) { + $timeout, $compile, websocketMsgSrv, ngToast) { - $scope.init = function () { - $scope.jobType = 'cron'; - //$scope.jobType = 'normal' - $scope.notebook = {}; - $scope.notebook.status = 'RUNNING'; + $scope.init = function (jobInformation) { }; $scope.getProgress = function () { - return 100; - } + return 80; + }; + + $scope.isRunningNotebook = function (jobInformation) { + var NOT_FOUND_RUNNING_STATUS = -1; + return _.findIndex(jobInformation.paragraphs, {status : "RUNNING"}) !== NOT_FOUND_RUNNING_STATUS; + }; }); diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.html b/zeppelin-web/src/app/jobmanager/jobs/job.html index b675a559a89..ae706336f1a 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job.html +++ b/zeppelin-web/src/app/jobmanager/jobs/job.html @@ -12,67 +12,78 @@ limitations under the License. --> -
    +
    - - + ng-switch="notebookJob.notebookType"> + +   - - Notebook Title - - -  -  - - - interpreter name - + + + {{notebookJob.notebookName}} + + +  -  + + + {{notebookJob.interpreter}} + +
    - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + +
    From d23fb7b7e63cd1ffc26764ed0e59713dd2041fec Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Wed, 11 May 2016 13:40:17 +0900 Subject: [PATCH 03/25] add notebook filter in jab manager page --- .../app/jobmanager/jobmanager.controller.js | 37 ++++++++++++++++--- .../src/app/jobmanager/jobmanager.html | 12 ++++-- zeppelin-web/test/karma.conf.js | 8 ++++ 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 7dc4dff56ae..5ef790fac46 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -15,23 +15,50 @@ 'use strict'; angular.module('zeppelinWebApp') + .filter('myJob', function() { + + function filterContext(JobItems, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME) { + var filterItems = JobItems; + if (FILTER_VALUE_INTERPRETER !== '*') { + filterItems = _.where(filterItems, {interpreter : FILTER_VALUE_INTERPRETER}); + console.log(filterItems) + } + + if (FILTER_VALUE_NOTEBOOK_NAME !== "") { + + filterItems = _.filter(filterItems, function(jobItem){ + var lowerFilterValue = FILTER_VALUE_NOTEBOOK_NAME.toLocaleLowerCase(); + var lowerNotebookName = jobItem.notebookName.toLocaleLowerCase(); + return lowerNotebookName.match(new RegExp(".*"+lowerFilterValue+".*"))}); + } + + return filterItems; + } + + return filterContext; + }) .controller('JobmanagerCtrl', function($scope, $route, $routeParams, $location, $rootScope, $http, - websocketMsgSrv, baseUrlSrv, $timeout, SaveAsService) { + websocketMsgSrv, baseUrlSrv, $timeout, SaveAsService, myJobFilter) { + - $scope.debugPrint = function (value) { - console.log(value); + $scope.doFiltering = function (jobInfomations, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME) { + $scope.JobInfomationsByFilter = $scope.jobTypeFilter(jobInfomations, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME) }; $scope.init = function () { - $scope.FILTER_VALUE_INTERPRETER = /.*/; + + $scope.FILTER_VALUE_NOTEBOOK_NAME = ""; + $scope.FILTER_VALUE_INTERPRETER = "*"; + $scope.jobTypeFilter = myJobFilter; $scope.ACTIVE_INTERPRETERS = [ { name : 'ALL', - value : /.*/ + value : '*' } ]; $scope.jobInfomations = initTestPacket(); + $scope.JobInfomationsByFilter = $scope.jobInfomations; var interpreterLists = _.pluck($scope.jobInfomations, 'interpreter'); for (var index = 0, length = interpreterLists.length; index < length; index++) { diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.html b/zeppelin-web/src/app/jobmanager/jobmanager.html index c012d6a7dae..d0e23382a4d 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.html +++ b/zeppelin-web/src/app/jobmanager/jobmanager.html @@ -102,15 +102,19 @@

    +
    + +
    Date: Wed, 18 May 2016 17:58:23 +0900 Subject: [PATCH 04/25] add sort to job item --- .../app/jobmanager/jobmanager.controller.js | 282 +++++++++--------- .../src/app/jobmanager/jobmanager.html | 41 ++- .../src/app/jobmanager/jobs/job-control.html | 3 + .../src/app/jobmanager/jobs/job.controller.js | 8 +- 4 files changed, 184 insertions(+), 150 deletions(-) diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 5ef790fac46..8784117e2e1 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -21,18 +21,21 @@ angular.module('zeppelinWebApp') var filterItems = JobItems; if (FILTER_VALUE_INTERPRETER !== '*') { filterItems = _.where(filterItems, {interpreter : FILTER_VALUE_INTERPRETER}); - console.log(filterItems) + console.log(filterItems); } - if (FILTER_VALUE_NOTEBOOK_NAME !== "") { + if (FILTER_VALUE_NOTEBOOK_NAME !== '') { filterItems = _.filter(filterItems, function(jobItem){ var lowerFilterValue = FILTER_VALUE_NOTEBOOK_NAME.toLocaleLowerCase(); var lowerNotebookName = jobItem.notebookName.toLocaleLowerCase(); - return lowerNotebookName.match(new RegExp(".*"+lowerFilterValue+".*"))}); + return lowerNotebookName.match(new RegExp('.*'+lowerFilterValue+'.*')); + }); } - return filterItems; + console.log(_.sortBy(filterItems, 'isRunningJob')); + + return _.sortBy(filterItems, 'isRunningJob').reverse(); } return filterContext; @@ -43,13 +46,13 @@ angular.module('zeppelinWebApp') $scope.doFiltering = function (jobInfomations, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME) { - $scope.JobInfomationsByFilter = $scope.jobTypeFilter(jobInfomations, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME) + $scope.JobInfomationsByFilter = $scope.jobTypeFilter(jobInfomations, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME); }; $scope.init = function () { - $scope.FILTER_VALUE_NOTEBOOK_NAME = ""; - $scope.FILTER_VALUE_INTERPRETER = "*"; + $scope.FILTER_VALUE_NOTEBOOK_NAME = ''; + $scope.FILTER_VALUE_INTERPRETER = '*'; $scope.jobTypeFilter = myJobFilter; $scope.ACTIVE_INTERPRETERS = [ { @@ -59,239 +62,246 @@ angular.module('zeppelinWebApp') ]; $scope.jobInfomations = initTestPacket(); $scope.JobInfomationsByFilter = $scope.jobInfomations; - + // initialize packet + $scope.JobInfomationsByFilter = $scope.jobTypeFilter( + $scope.jobInfomations, $scope.FILTER_VALUE_INTERPRETER, $scope.FILTER_VALUE_NOTEBOOK_NAME + ); var interpreterLists = _.pluck($scope.jobInfomations, 'interpreter'); for (var index = 0, length = interpreterLists.length; index < length; index++) { $scope.ACTIVE_INTERPRETERS.push({ name : interpreterLists[index], value : interpreterLists[index] - }) + }); } }; var initTestPacket = function () { return [ { - notebookId: "2BMFSNAAF", - notebookName: "myNotebook", - notebookType: "normal", - interpreter: "mySpark", + notebookId: '2BMFSNAAF', + notebookName: 'myNotebook', + notebookType: 'normal', + interpreter: 'mySpark', isRunningJob: true, + unixTimeLastRun: 1463561562, paragraphs: [ { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "FINISHED" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'FINISHED' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "FINISHED" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'FINISHED' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "RUNNING" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'RUNNING' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "PENDING" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'PENDING' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "PENDING" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'PENDING' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' } ] }, { - notebookId: "2BMFSNAAF", - notebookName: "hiveNotebook", - notebookType: "normal", - interpreter: "hive", + notebookId: '2BMFSNAAF', + notebookName: 'hiveNotebook', + notebookType: 'normal', + interpreter: 'hive', isRunningJob: false, + unixTimeLastRun: 1463561562, paragraphs: [ { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "FINISHED" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'FINISHED' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "FINISHED" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'FINISHED' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "ABORT" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'ABORT' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "PENDING" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'PENDING' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' } ] }, { - notebookId: "2BMFSNAAF", - notebookName: "R Notebook", - notebookType: "cron", - interpreter: "r", + notebookId: '2BMFSNAAF', + notebookName: 'R Notebook', + notebookType: 'cron', + interpreter: 'r', isRunningJob: true, + unixTimeLastRun: 1463561562, paragraphs: [ { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "FINISHED" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'FINISHED' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "FINISHED" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'FINISHED' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "RUNNING" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'RUNNING' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "PENDING" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'PENDING' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "ERROR" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'ERROR' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "ERROR" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'ERROR' } ] }, { - notebookId: "2BMFSNAAF", - notebookName: "myNotebook", - notebookType: "normal", - interpreter: "mySpark", + notebookId: '2BMFSNAAF', + notebookName: 'myNotebook', + notebookType: 'normal', + interpreter: 'mySpark', isRunningJob: false, + unixTimeLastRun: 1463561562, paragraphs: [ { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' }, { - id: "20160509-112757_1030139246", - name: "20160509-112757_1030139246", - status: "READY" + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' } ] } diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.html b/zeppelin-web/src/app/jobmanager/jobmanager.html index d0e23382a4d..54b0c5d7301 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.html +++ b/zeppelin-web/src/app/jobmanager/jobmanager.html @@ -100,27 +100,44 @@

    -
    - -
    - +
    +
    +
    +
    + + +
    -
    +
    +
    +
    + Data does not exist +
    +
    diff --git a/zeppelin-web/src/app/jobmanager/jobs/job-control.html b/zeppelin-web/src/app/jobmanager/jobs/job-control.html index fdc0160bc32..2fb7eb5e048 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job-control.html +++ b/zeppelin-web/src/app/jobmanager/jobs/job-control.html @@ -13,6 +13,9 @@ -->
    + + {{lastExecuteTime(notebookJob.unixTimeLastRun)}} + RUNNING diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js index fe74fa47f16..c4043af46c4 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js @@ -16,7 +16,7 @@ angular.module('zeppelinWebApp') .controller('JobCtrl', function($scope,$rootScope, $route, $window, $element, $routeParams, $location, - $timeout, $compile, websocketMsgSrv, ngToast) { + $timeout, $compile, websocketMsgSrv, ngToast, moment) { $scope.init = function (jobInformation) { }; @@ -27,7 +27,11 @@ angular.module('zeppelinWebApp') $scope.isRunningNotebook = function (jobInformation) { var NOT_FOUND_RUNNING_STATUS = -1; - return _.findIndex(jobInformation.paragraphs, {status : "RUNNING"}) !== NOT_FOUND_RUNNING_STATUS; + return _.findIndex(jobInformation.paragraphs, {status : 'RUNNING'}) !== NOT_FOUND_RUNNING_STATUS; + }; + + $scope.lastExecuteTime = function (unixtime) { + return moment.unix(unixtime).fromNow(); }; }); From b24b4e789d148b9751979dc164ec30150cd2ddb0 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Thu, 19 May 2016 11:45:37 +0900 Subject: [PATCH 05/25] add momentjs in .jshintrc --- zeppelin-web/.jshintrc | 3 ++- zeppelin-web/src/app/jobmanager/jobs/job.controller.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/zeppelin-web/.jshintrc b/zeppelin-web/.jshintrc index bdcd213761e..8bf2b07df98 100644 --- a/zeppelin-web/.jshintrc +++ b/zeppelin-web/.jshintrc @@ -32,6 +32,7 @@ "ace": false, "d3": false, "BootstrapDialog": false, - "Handsontable": false + "Handsontable": false, + "moment" : false } } diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js index c4043af46c4..91494eaaae4 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js @@ -16,7 +16,7 @@ angular.module('zeppelinWebApp') .controller('JobCtrl', function($scope,$rootScope, $route, $window, $element, $routeParams, $location, - $timeout, $compile, websocketMsgSrv, ngToast, moment) { + $timeout, $compile, websocketMsgSrv, ngToast) { $scope.init = function (jobInformation) { }; From b90e3202666eb0ccf460ba0a8124a3f87935a4b5 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Tue, 24 May 2016 17:47:25 +0900 Subject: [PATCH 06/25] implement notebook job management backend. --- .../org/apache/zeppelin/socket/Message.java | 3 +- .../zeppelin/socket/NotebookServer.java | 122 ++++++++++++++++++ 2 files changed, 124 insertions(+), 1 deletion(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java index 913e184619e..fb744600c71 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java @@ -111,9 +111,10 @@ public static enum OP { CONFIGURATIONS_INFO, // [s-c] all key/value pairs of configurations // @param settings serialized Map object - CHECKPOINT_NOTEBOOK // [c-s] checkpoint notebook to storage repository + CHECKPOINT_NOTEBOOK, // [c-s] checkpoint notebook to storage repository // @param noteId // @param checkpointName + LISTS_NOTEBOOK_JOBS } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 9e46cd2746e..f94d8ad1fc8 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -135,6 +135,9 @@ public void onMessage(NotebookSocket conn, String msg) { case LIST_NOTES: unicastNoteList(conn); break; + case LISTS_NOTEBOOK_JOBS: + unicastNotebookJobInfo(conn); + break; case RELOAD_NOTES_FROM_REPO: broadcastReloadedNoteList(); break; @@ -347,6 +350,120 @@ private void unicast(Message m, NotebookSocket conn) { } } + /* + { + notebookId: '2BMFSNAAF', + notebookName: 'hiveNotebook', + notebookType: 'normal', + interpreter: 'hive', + isRunningJob: false, + unixTimeLastRun: 1463561562, + paragraphs: [ + { + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'FINISHED' + }, + { + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'FINISHED' + }, + + { + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'ABORT' + }, + { + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'PENDING' + }, + { + id: '20160509-112757_1030139246', + name: '20160509-112757_1030139246', + status: 'READY' + } + ] + } + */ + + public List> generateNotebooksJobInfo(boolean needsReload) { + Notebook notebook = notebook(); + + ZeppelinConfiguration conf = notebook.getConf(); + String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); + boolean hideHomeScreenNotebookFromList = conf + .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); + + if (needsReload) { + try { + notebook.reloadAllNotes(); + } catch (IOException e) { + LOG.error("Fail to reload notes from repository"); + } + } + + List notes = notebook.getAllNotes(); + List> notesInfo = new LinkedList<>(); + for (Note note : notes) { + boolean isNotebookRunning = false; + Map info = new HashMap<>(); + + if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { + continue; + } + + String cronTypeNotebookKeywork = "cron"; + info.put("notebookId", note.id()); + info.put("notebookName", note.getName()); + if (note.getConfig().containsKey(cronTypeNotebookKeywork) == true + && !note.getConfig().get(cronTypeNotebookKeywork).equals("")) { + info.put("notebookType", "cron"); + } + else { + info.put("notebookType", "normal"); + } + long lastRunningUnixTime = 0; + + List> paragraphsInfo = new LinkedList<>(); + Date lastRunningDate = null; + for (Paragraph paragraph : note.getParagraphs()) { + Map paragraphItem = new HashMap<>(); + paragraphItem.put("id", paragraph.getId()); + paragraphItem.put("name", paragraph.getTitle()); + paragraphItem.put("status", paragraph.getStatus().toString()); +// if (lastRunningDate == null) { +// lastRunningDate = paragraph.getDateStarted(); +// lastRunningUnixTime = lastRunningDate.getTime(); +// } +// else { +// LOG.info("start time {}", paragraph.()); +// if (lastRunningDate.after(paragraph.getDateFinished()) == true) { +// lastRunningDate = paragraph.getDateFinished(); +// lastRunningUnixTime = lastRunningDate.getTime(); +// } +// } + + if (paragraph.getStatus().isRunning() == true) { + isNotebookRunning = true; + } + paragraphsInfo.add(paragraphItem); + } + + info.put("interpreter", note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup()); + info.put("isRunningJob", isNotebookRunning); + //info.put("unixTimeLastRun", lastRunningUnixTime); + info.put("unixTimeLastRun", 123123123); + info.put("paragraphs", paragraphsInfo); + + notesInfo.add(info); + } + + return notesInfo; + } + public List> generateNotebooksInfo(boolean needsReload) { Notebook notebook = notebook(); @@ -394,6 +511,11 @@ public void unicastNoteList(NotebookSocket conn) { unicast(new Message(OP.NOTES_INFO).put("notes", notesInfo), conn); } + public void unicastNotebookJobInfo(NotebookSocket conn) { + List> notebookJobs = generateNotebooksJobInfo(true); + unicast(new Message(OP.NOTES_INFO).put("notebookJobs", notebookJobs), conn); + } + public void broadcastReloadedNoteList() { List> notesInfo = generateNotebooksInfo(true); broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo)); From 35887c0a3017f1c8b7483be896b6cfa2de1f6fde Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Thu, 26 May 2016 15:41:34 +0900 Subject: [PATCH 07/25] Implement cs transition for jobmanagement. --- .../org/apache/zeppelin/socket/Message.java | 2 +- .../zeppelin/socket/NotebookServer.java | 16 ++++++-- .../app/jobmanager/jobmanager.controller.js | 39 +++++++++++-------- .../websocketEvents.factory.js | 4 +- .../websocketEvents/websocketMsg.service.js | 4 ++ 5 files changed, 42 insertions(+), 23 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java index fb744600c71..5fcc99678cf 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java @@ -114,7 +114,7 @@ public static enum OP { CHECKPOINT_NOTEBOOK, // [c-s] checkpoint notebook to storage repository // @param noteId // @param checkpointName - LISTS_NOTEBOOK_JOBS + LIST_NOTEBOOK_JOBS } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index f94d8ad1fc8..c47732f9182 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -135,7 +135,7 @@ public void onMessage(NotebookSocket conn, String msg) { case LIST_NOTES: unicastNoteList(conn); break; - case LISTS_NOTEBOOK_JOBS: + case LIST_NOTEBOOK_JOBS: unicastNotebookJobInfo(conn); break; case RELOAD_NOTES_FROM_REPO: @@ -417,7 +417,13 @@ public List> generateNotebooksJobInfo(boolean needsReload) { String cronTypeNotebookKeywork = "cron"; info.put("notebookId", note.id()); - info.put("notebookName", note.getName()); + String notebookName = note.getName(); + if (notebookName != null) { + info.put("notebookName", note.getName()); + } else { + info.put("notebookName", note.id()); + } + if (note.getConfig().containsKey(cronTypeNotebookKeywork) == true && !note.getConfig().get(cronTypeNotebookKeywork).equals("")) { info.put("notebookType", "cron"); @@ -433,6 +439,7 @@ public List> generateNotebooksJobInfo(boolean needsReload) { Map paragraphItem = new HashMap<>(); paragraphItem.put("id", paragraph.getId()); paragraphItem.put("name", paragraph.getTitle()); + LOG.info("{}, {}", paragraph.getTitle(), paragraph.isRunning()); paragraphItem.put("status", paragraph.getStatus().toString()); // if (lastRunningDate == null) { // lastRunningDate = paragraph.getDateStarted(); @@ -512,8 +519,9 @@ public void unicastNoteList(NotebookSocket conn) { } public void unicastNotebookJobInfo(NotebookSocket conn) { - List> notebookJobs = generateNotebooksJobInfo(true); - unicast(new Message(OP.NOTES_INFO).put("notebookJobs", notebookJobs), conn); + LOG.info("request unicast notebook job info"); + List> notebookJobs = generateNotebooksJobInfo(false); + unicast(new Message(OP.LIST_NOTEBOOK_JOBS).put("notebookJobs", notebookJobs), conn); } public void broadcastReloadedNoteList() { diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 8784117e2e1..94050e294a0 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -44,35 +44,40 @@ angular.module('zeppelinWebApp') function($scope, $route, $routeParams, $location, $rootScope, $http, websocketMsgSrv, baseUrlSrv, $timeout, SaveAsService, myJobFilter) { - - $scope.doFiltering = function (jobInfomations, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME) { - $scope.JobInfomationsByFilter = $scope.jobTypeFilter(jobInfomations, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME); - }; - - $scope.init = function () { - - $scope.FILTER_VALUE_NOTEBOOK_NAME = ''; - $scope.FILTER_VALUE_INTERPRETER = '*'; - $scope.jobTypeFilter = myJobFilter; + $scope.$on('setNotebookJobs', function(event, note) { + console.log('packet recevied ', note); + $scope.jobInfomations = note; + $scope.JobInfomationsByFilter = $scope.jobTypeFilter( + $scope.jobInfomations, $scope.FILTER_VALUE_INTERPRETER, $scope.FILTER_VALUE_NOTEBOOK_NAME + ); $scope.ACTIVE_INTERPRETERS = [ { name : 'ALL', value : '*' } ]; - $scope.jobInfomations = initTestPacket(); - $scope.JobInfomationsByFilter = $scope.jobInfomations; - // initialize packet - $scope.JobInfomationsByFilter = $scope.jobTypeFilter( - $scope.jobInfomations, $scope.FILTER_VALUE_INTERPRETER, $scope.FILTER_VALUE_NOTEBOOK_NAME - ); - var interpreterLists = _.pluck($scope.jobInfomations, 'interpreter'); + var interpreterLists = _.uniq(_.pluck($scope.jobInfomations, 'interpreter'), true); for (var index = 0, length = interpreterLists.length; index < length; index++) { $scope.ACTIVE_INTERPRETERS.push({ name : interpreterLists[index], value : interpreterLists[index] }); } + }); + + $scope.doFiltering = function (jobInfomations, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME) { + $scope.JobInfomationsByFilter = $scope.jobTypeFilter(jobInfomations, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME); + }; + + $scope.init = function () { + $scope.FILTER_VALUE_NOTEBOOK_NAME = ''; + $scope.FILTER_VALUE_INTERPRETER = '*'; + $scope.jobTypeFilter = myJobFilter; + //$scope.jobInfomations = initTestPacket(); + $scope.jobInfomations = []; + $scope.JobInfomationsByFilter = $scope.jobInfomations; + + websocketMsgSrv.getNotebookJobsList(); }; var initTestPacket = function () { diff --git a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js index 3a261a78f74..e73a85e1102 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js +++ b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js @@ -59,10 +59,12 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, $location.path('notebook/' + data.note.id); } else if (op === 'NOTES_INFO') { $rootScope.$broadcast('setNoteMenu', data.notes); + } else if (op === 'LIST_NOTEBOOK_JOBS') { + $rootScope.$broadcast('setNotebookJobs', data.notebookJobs); } else if (op === 'AUTH_INFO') { BootstrapDialog.show({ closable: true, - title: 'Insufficient privileges', + title: 'Insufficient privileges', message: data.info.toString(), buttons: [{ label: 'Login', diff --git a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js index 3b4df03796c..13bfe9d5471 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js +++ b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js @@ -37,6 +37,10 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, websocketEvents.sendNewEvent({op: 'LIST_NOTES'}); }, + getNotebookJobsList: function() { + websocketEvents.sendNewEvent({op: 'LIST_NOTEBOOK_JOBS'}); + }, + reloadAllNotesFromRepo: function() { websocketEvents.sendNewEvent({op: 'RELOAD_NOTES_FROM_REPO'}); }, From f87217c726294c7d0e56e709db0ba3bbf7231e2e Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Thu, 26 May 2016 19:31:32 +0900 Subject: [PATCH 08/25] improve jobmanagement ui and timestamp --- .../zeppelin/socket/NotebookServer.java | 83 ++---- .../app/jobmanager/jobmanager.controller.js | 241 +----------------- .../src/app/jobmanager/jobmanager.html | 174 ++++++------- .../app/jobmanager/jobs/job-progressBar.html | 2 +- .../src/app/jobmanager/jobs/job.controller.js | 15 +- 5 files changed, 136 insertions(+), 379 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 79f41767bc7..7a5dfe7d6cd 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -350,45 +350,6 @@ private void unicast(Message m, NotebookSocket conn) { } } - /* - { - notebookId: '2BMFSNAAF', - notebookName: 'hiveNotebook', - notebookType: 'normal', - interpreter: 'hive', - isRunningJob: false, - unixTimeLastRun: 1463561562, - paragraphs: [ - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'FINISHED' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'FINISHED' - }, - - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'ABORT' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'PENDING' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - } - ] - } - */ - public List> generateNotebooksJobInfo(boolean needsReload) { Notebook notebook = notebook(); @@ -431,27 +392,40 @@ public List> generateNotebooksJobInfo(boolean needsReload) { else { info.put("notebookType", "normal"); } + + Date lastRunningDate = null; long lastRunningUnixTime = 0; List> paragraphsInfo = new LinkedList<>(); - Date lastRunningDate = null; for (Paragraph paragraph : note.getParagraphs()) { Map paragraphItem = new HashMap<>(); paragraphItem.put("id", paragraph.getId()); - paragraphItem.put("name", paragraph.getTitle()); - LOG.info("{}, {}", paragraph.getTitle(), paragraph.isRunning()); + String paragraphName = paragraph.getTitle(); + if (paragraphName != null) { + paragraphItem.put("name", paragraphName); + } else { + paragraphItem.put("name", paragraph.getId()); + } paragraphItem.put("status", paragraph.getStatus().toString()); -// if (lastRunningDate == null) { -// lastRunningDate = paragraph.getDateStarted(); -// lastRunningUnixTime = lastRunningDate.getTime(); -// } -// else { -// LOG.info("start time {}", paragraph.()); -// if (lastRunningDate.after(paragraph.getDateFinished()) == true) { -// lastRunningDate = paragraph.getDateFinished(); -// lastRunningUnixTime = lastRunningDate.getTime(); -// } -// } + + LOG.info("lastRunningDate {}", lastRunningDate); + LOG.info("{} {} {}", paragraph.getDateCreated(), + paragraph.getDateStarted(), paragraph.getDateFinished()); + + Date paragaraphDate = paragraph.getDateStarted(); + if (paragaraphDate == null) { + paragaraphDate = paragraph.getDateCreated(); + } + + if (lastRunningDate == null) { + lastRunningDate = paragaraphDate; + } else { + if (lastRunningDate.after(paragaraphDate) == true) { + lastRunningDate = paragaraphDate; + } + } + + lastRunningUnixTime = lastRunningDate.getTime(); if (paragraph.getStatus().isRunning() == true) { isNotebookRunning = true; @@ -461,8 +435,7 @@ public List> generateNotebooksJobInfo(boolean needsReload) { info.put("interpreter", note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup()); info.put("isRunningJob", isNotebookRunning); - //info.put("unixTimeLastRun", lastRunningUnixTime); - info.put("unixTimeLastRun", 123123123); + info.put("unixTimeLastRun", lastRunningUnixTime); info.put("paragraphs", paragraphsInfo); notesInfo.add(info); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 94050e294a0..4a1430d8b59 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -42,7 +42,7 @@ angular.module('zeppelinWebApp') }) .controller('JobmanagerCtrl', function($scope, $route, $routeParams, $location, $rootScope, $http, - websocketMsgSrv, baseUrlSrv, $timeout, SaveAsService, myJobFilter) { + websocketMsgSrv, baseUrlSrv, $interval, SaveAsService, myJobFilter) { $scope.$on('setNotebookJobs', function(event, note) { console.log('packet recevied ', note); @@ -56,7 +56,7 @@ angular.module('zeppelinWebApp') value : '*' } ]; - var interpreterLists = _.uniq(_.pluck($scope.jobInfomations, 'interpreter'), true); + var interpreterLists = _.uniq(_.pluck($scope.jobInfomations, 'interpreter'), false); for (var index = 0, length = interpreterLists.length; index < length; index++) { $scope.ACTIVE_INTERPRETERS.push({ name : interpreterLists[index], @@ -73,243 +73,14 @@ angular.module('zeppelinWebApp') $scope.FILTER_VALUE_NOTEBOOK_NAME = ''; $scope.FILTER_VALUE_INTERPRETER = '*'; $scope.jobTypeFilter = myJobFilter; - //$scope.jobInfomations = initTestPacket(); $scope.jobInfomations = []; $scope.JobInfomationsByFilter = $scope.jobInfomations; websocketMsgSrv.getNotebookJobsList(); - }; - - var initTestPacket = function () { - return [ - { - notebookId: '2BMFSNAAF', - notebookName: 'myNotebook', - notebookType: 'normal', - interpreter: 'mySpark', - isRunningJob: true, - unixTimeLastRun: 1463561562, - paragraphs: [ - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'FINISHED' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'FINISHED' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'RUNNING' - }, - - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'PENDING' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'PENDING' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - } - ] - }, - { - notebookId: '2BMFSNAAF', - notebookName: 'hiveNotebook', - notebookType: 'normal', - interpreter: 'hive', - isRunningJob: false, - unixTimeLastRun: 1463561562, - paragraphs: [ - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'FINISHED' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'FINISHED' - }, + var refreshObj = $interval(websocketMsgSrv.getNotebookJobsList, 1200); - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'ABORT' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'PENDING' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - } - ] - }, - { - notebookId: '2BMFSNAAF', - notebookName: 'R Notebook', - notebookType: 'cron', - interpreter: 'r', - isRunningJob: true, - unixTimeLastRun: 1463561562, - paragraphs: [ - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'FINISHED' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'FINISHED' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'RUNNING' - }, - - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'PENDING' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'ERROR' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'ERROR' - } - ] - }, - { - notebookId: '2BMFSNAAF', - notebookName: 'myNotebook', - notebookType: 'normal', - interpreter: 'mySpark', - isRunningJob: false, - unixTimeLastRun: 1463561562, - paragraphs: [ - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - }, - { - id: '20160509-112757_1030139246', - name: '20160509-112757_1030139246', - status: 'READY' - } - ] - } - ]; + $scope.$on('$destroy', function() { + $interval.cancel(refreshObj); + }); }; }); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.html b/zeppelin-web/src/app/jobmanager/jobmanager.html index 54b0c5d7301..068c79fa85b 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.html +++ b/zeppelin-web/src/app/jobmanager/jobmanager.html @@ -27,93 +27,95 @@

    -
    -
    - - - - - {{jobStatus}} - - - - {{jobStatus}} - - - - - {{jobStatus}} - - - - - {{jobStatus}} - - - - - {{jobStatus}} - - - +
    +
    +
    +
    +
    + + +
    +
    +
    + - - {{jobStatus}} - - -
    -
    -
    -
    -
    - - + + + + {{jobStatus}} + + + + {{jobStatus}} + + + + + {{jobStatus}} + + + + + {{jobStatus}} + + + + + {{jobStatus}} + + + + + {{jobStatus}} + +
    diff --git a/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html b/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html index d6b49d3038f..f0e67da253f 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html +++ b/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html @@ -13,7 +13,7 @@ -->
    -
    +
    Date: Fri, 27 May 2016 20:13:28 +0900 Subject: [PATCH 09/25] bug fixed and improve run status to always top for job manager page. --- .../apache/zeppelin/rest/NotebookRestApi.java | 20 ++++-- .../zeppelin/socket/NotebookServer.java | 5 -- .../app/jobmanager/jobmanager.controller.js | 43 ++++++----- .../src/app/jobmanager/jobmanager.html | 72 +++++++++---------- .../src/app/jobmanager/jobs/job-control.html | 4 +- .../app/jobmanager/jobs/job-progressBar.html | 2 +- .../src/app/jobmanager/jobs/job.controller.js | 72 +++++++++++++++++-- zeppelin-web/src/app/jobmanager/jobs/job.html | 12 +++- 8 files changed, 152 insertions(+), 78 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 89e84eed192..f3e63a98b67 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -32,6 +32,7 @@ import javax.ws.rs.core.Response.Status; import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Notebook; @@ -472,8 +473,13 @@ public Response runNoteJobs(@PathParam("notebookId") String notebookId) throws if (note == null) { return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } - - note.runAll(); + try { + note.runAll(); + } catch (InterpreterException intpException) { + return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, intpException.getMessage()).build(); + } catch (Exception e) { + return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage()).build(); + } return new JsonResponse<>(Status.OK).build(); } @@ -493,10 +499,14 @@ public Response stopNoteJobs(@PathParam("notebookId") String notebookId) throws return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } - for (Paragraph p : note.getParagraphs()) { - if (!p.isTerminated()) { - p.abort(); + try { + for (Paragraph p : note.getParagraphs()) { + if (!p.isTerminated()) { + p.abort(); + } } + } catch (Exception e) { + return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage()).build(); } return new JsonResponse<>(Status.OK).build(); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 7a5dfe7d6cd..249665aa91c 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -408,10 +408,6 @@ public List> generateNotebooksJobInfo(boolean needsReload) { } paragraphItem.put("status", paragraph.getStatus().toString()); - LOG.info("lastRunningDate {}", lastRunningDate); - LOG.info("{} {} {}", paragraph.getDateCreated(), - paragraph.getDateStarted(), paragraph.getDateFinished()); - Date paragaraphDate = paragraph.getDateStarted(); if (paragaraphDate == null) { paragaraphDate = paragraph.getDateCreated(); @@ -492,7 +488,6 @@ public void unicastNoteList(NotebookSocket conn) { } public void unicastNotebookJobInfo(NotebookSocket conn) { - LOG.info("request unicast notebook job info"); List> notebookJobs = generateNotebooksJobInfo(false); unicast(new Message(OP.LIST_NOTEBOOK_JOBS).put("notebookJobs", notebookJobs), conn); } diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 4a1430d8b59..6feb6e35a0e 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -17,27 +17,29 @@ angular.module('zeppelinWebApp') .filter('myJob', function() { - function filterContext(JobItems, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME) { + function filterContext(JobItems, filterConfig) { + var FILTER_VALUE_INTERPRETER = filterConfig.FILTER_VALUE_INTERPRETER; + var FILTER_VALUE_NOTEBOOK_NAME = filterConfig.FILTER_VALUE_NOTEBOOK_NAME; + var RUNNING_ALWAYS_TOP = filterConfig.RUNNING_ALWAYS_TOP; + var filterItems = JobItems; if (FILTER_VALUE_INTERPRETER !== '*') { filterItems = _.where(filterItems, {interpreter : FILTER_VALUE_INTERPRETER}); - console.log(filterItems); } - if (FILTER_VALUE_NOTEBOOK_NAME !== '') { + if (RUNNING_ALWAYS_TOP === true) { + filterItems = _.sortBy(filterItems, 'isRunningJob').reverse(); + } + if (FILTER_VALUE_NOTEBOOK_NAME !== '') { filterItems = _.filter(filterItems, function(jobItem){ var lowerFilterValue = FILTER_VALUE_NOTEBOOK_NAME.toLocaleLowerCase(); var lowerNotebookName = jobItem.notebookName.toLocaleLowerCase(); return lowerNotebookName.match(new RegExp('.*'+lowerFilterValue+'.*')); - }); + }); } - - console.log(_.sortBy(filterItems, 'isRunningJob')); - - return _.sortBy(filterItems, 'isRunningJob').reverse(); + return filterItems; } - return filterContext; }) .controller('JobmanagerCtrl', @@ -45,11 +47,8 @@ angular.module('zeppelinWebApp') websocketMsgSrv, baseUrlSrv, $interval, SaveAsService, myJobFilter) { $scope.$on('setNotebookJobs', function(event, note) { - console.log('packet recevied ', note); $scope.jobInfomations = note; - $scope.JobInfomationsByFilter = $scope.jobTypeFilter( - $scope.jobInfomations, $scope.FILTER_VALUE_INTERPRETER, $scope.FILTER_VALUE_NOTEBOOK_NAME - ); + $scope.JobInfomationsByFilter = $scope.jobTypeFilter($scope.jobInfomations, $scope.filterConfig); $scope.ACTIVE_INTERPRETERS = [ { name : 'ALL', @@ -65,19 +64,27 @@ angular.module('zeppelinWebApp') } }); - $scope.doFiltering = function (jobInfomations, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME) { - $scope.JobInfomationsByFilter = $scope.jobTypeFilter(jobInfomations, FILTER_VALUE_INTERPRETER, FILTER_VALUE_NOTEBOOK_NAME); + $scope.doFiltering = function (jobInfomations, filterConfig) { + $scope.JobInfomationsByFilter = $scope.jobTypeFilter(jobInfomations, filterConfig); + }; + + $scope.onChangeRunJobToAlwaysTopToggle = function (jobInfomations, filterConfig) { + $scope.isRunJobToAlwaysTopToggle = !$scope.isRunJobToAlwaysTopToggle; + $scope.doFiltering(jobInfomations, filterConfig); }; $scope.init = function () { - $scope.FILTER_VALUE_NOTEBOOK_NAME = ''; - $scope.FILTER_VALUE_INTERPRETER = '*'; + $scope.filterConfig = { + RUNNING_ALWAYS_TOP : true, + FILTER_VALUE_NOTEBOOK_NAME : '', + FILTER_VALUE_INTERPRETER : '*' + }; $scope.jobTypeFilter = myJobFilter; $scope.jobInfomations = []; $scope.JobInfomationsByFilter = $scope.jobInfomations; websocketMsgSrv.getNotebookJobsList(); - var refreshObj = $interval(websocketMsgSrv.getNotebookJobsList, 1200); + var refreshObj = $interval(websocketMsgSrv.getNotebookJobsList, 1000); $scope.$on('$destroy', function() { $interval.cancel(refreshObj); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.html b/zeppelin-web/src/app/jobmanager/jobmanager.html index 068c79fa85b..b09bc36e61d 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.html +++ b/zeppelin-web/src/app/jobmanager/jobmanager.html @@ -27,65 +27,60 @@

    -
    -
    +
    +
    +
    + type="text" ng-model="filterConfig.FILTER_VALUE_NOTEBOOK_NAME" + ng-change="doFiltering(jobInfomations, filterConfig)"/>
    -
    +
    + ng-switch="jobStatus"> + ng-switch-when="FINISHED"> + tooltip="FINISHED"> {{jobStatus}} + style="margin-right: 3px;"> {{jobStatus}} + style="margin-right: 3px;"> + tooltip="READY"> {{jobStatus}} + style="margin-right: 3px;"> + style="margin-right: 3px;"> + tooltip="ABORT"> {{jobStatus}} + style="margin-right: 3px;"> + tooltip="ERROR"> {{jobStatus}}
    +
    +
    + + Running the notebook always on top + + +
    +
    @@ -127,17 +128,14 @@

    class="paragraph-col">
    + ng-controller="JobCtrl">

    + class="paragraph-col">
    + class="job-space box job-margin text-center"> Data does not exist
    diff --git a/zeppelin-web/src/app/jobmanager/jobs/job-control.html b/zeppelin-web/src/app/jobmanager/jobs/job-control.html index 2fb7eb5e048..f8d56df428d 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job-control.html +++ b/zeppelin-web/src/app/jobmanager/jobs/job-control.html @@ -33,11 +33,11 @@ + ng-click="runNotebookJob(notebookJob.notebookId)"> + ng-click="stopNotebookJob(notebookJob.notebookId)">
    diff --git a/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html b/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html index f0e67da253f..11e3c172e10 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html +++ b/zeppelin-web/src/app/jobmanager/jobs/job-progressBar.html @@ -15,7 +15,7 @@
    + class="progress-bar" role="progressbar" ng-style="{width:getProgress()+'%'}">
    diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js index e75c655e862..bffcf653c38 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js @@ -15,10 +15,10 @@ 'use strict'; angular.module('zeppelinWebApp') - .controller('JobCtrl', function($scope,$rootScope, $route, $window, $element, $routeParams, $location, - $timeout, $compile, websocketMsgSrv, ngToast) { + .controller('JobCtrl', function($scope,$rootScope, $http, baseUrlSrv) { $scope.init = function (jobInformation) { + $scope.progressValue = 0; }; $scope.getProgress = function () { @@ -36,13 +36,71 @@ angular.module('zeppelinWebApp') return isNaN(result)? 0 : result; }; - $scope.isRunningNotebook = function (jobInformation) { - var NOT_FOUND_RUNNING_STATUS = -1; - return _.findIndex(jobInformation.paragraphs, {status : 'RUNNING'}) !== NOT_FOUND_RUNNING_STATUS; - }; - $scope.lastExecuteTime = function (unixtime) { return moment.unix(unixtime/1000).fromNow(); }; + $scope.runNotebookJob = function (notebookId) { + BootstrapDialog.confirm({ + closable: true, + title: '', + message: 'Run all paragraphs?', + callback: function(result) { + if (result === true) { + $http({ + method: 'POST', + url: baseUrlSrv.getRestApiBase() + '/notebook/job/' + notebookId, + headers: { + 'Content-Type': 'application/x-www-form-urlencoded' + } + }).then(function successCallback(response) { + console.log('run job at management', response); + }, function errorCallback(errorResponse) { + console.log(errorResponse); + var errorText = 'SERVER ERROR'; + if (errorResponse.data.message !== undefined) { + errorText = errorResponse.data.message; + } + BootstrapDialog.alert({ + closable: true, + title: 'Execution Failure', + message: errorText + }); + }); + } + } + }); + }; + + $scope.stopNotebookJob = function (notebookId) { + BootstrapDialog.confirm({ + closable: true, + title: '', + message: 'Stop all paragraphs?', + callback: function(result) { + if (result === true) { + $http({ + method: 'DELETE', + url: baseUrlSrv.getRestApiBase() + '/notebook/job/' + notebookId, + headers: { + 'Content-Type': 'application/x-www-form-urlencoded' + } + }).then(function successCallback(response) { + console.log('run job at management', response); + }, function errorCallback(errorResponse) { + var errorText = 'SERVER ERROR'; + if (errorResponse.data.message !== undefined) { + errorText = errorResponse.data.message; + } + BootstrapDialog.alert({ + closable: true, + title: 'Stop Failure', + message: errorText + }); + }); + }; + } + }); + }; + }); diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.html b/zeppelin-web/src/app/jobmanager/jobs/job.html index ae706336f1a..be0e8532fff 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job.html +++ b/zeppelin-web/src/app/jobmanager/jobs/job.html @@ -23,7 +23,7 @@   - + {{notebookJob.notebookName}} @@ -40,45 +40,51 @@
    + ng-switch="paragraphJob.status"> From 75c03efd04262e23751373dc132185fffa6e2d55 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Fri, 27 May 2016 20:16:30 +0900 Subject: [PATCH 10/25] remove semicolon for job manager --- zeppelin-web/src/app/jobmanager/jobs/job.controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js index bffcf653c38..9d70665ada9 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js @@ -98,7 +98,7 @@ angular.module('zeppelinWebApp') message: errorText }); }); - }; + } } }); }; From 0bcb9ff1a27c360b75d299f54d11e1c381daf9b4 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Mon, 30 May 2016 11:44:10 +0900 Subject: [PATCH 11/25] Additional feedback in the absence of the interpreter for jobmanager --- .../org/apache/zeppelin/socket/NotebookServer.java | 9 ++++++++- .../src/app/jobmanager/jobmanager.controller.js | 11 ++++++++--- zeppelin-web/src/app/jobmanager/jobs/job.html | 9 +++++++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index d0b9ba06868..bf1f138c34a 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -431,7 +431,14 @@ public List> generateNotebooksJobInfo(boolean needsReload) { paragraphsInfo.add(paragraphItem); } - info.put("interpreter", note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup()); + // Interpreter is set does not exist. + String interpreterGroupName = null; + if (note.getNoteReplLoader().getInterpreterSettings() != null + && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { + interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); + } + + info.put("interpreter", interpreterGroupName); info.put("isRunningJob", isNotebookRunning); info.put("unixTimeLastRun", lastRunningUnixTime); info.put("paragraphs", paragraphsInfo); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 6feb6e35a0e..e08696003f4 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -17,13 +17,18 @@ angular.module('zeppelinWebApp') .filter('myJob', function() { - function filterContext(JobItems, filterConfig) { + function filterContext(jobItems, filterConfig) { var FILTER_VALUE_INTERPRETER = filterConfig.FILTER_VALUE_INTERPRETER; var FILTER_VALUE_NOTEBOOK_NAME = filterConfig.FILTER_VALUE_NOTEBOOK_NAME; var RUNNING_ALWAYS_TOP = filterConfig.RUNNING_ALWAYS_TOP; - var filterItems = JobItems; - if (FILTER_VALUE_INTERPRETER !== '*') { + var filterItems = jobItems; + + if (FILTER_VALUE_INTERPRETER === undefined) { + filterItems = _.filter(filterItems, function (jobItem) { + return jobItem.interpreter === undefined? true : false; + }); + } else if (FILTER_VALUE_INTERPRETER !== '*') { filterItems = _.where(filterItems, {interpreter : FILTER_VALUE_INTERPRETER}); } diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.html b/zeppelin-web/src/app/jobmanager/jobs/job.html index be0e8532fff..7bc8f1547fb 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job.html +++ b/zeppelin-web/src/app/jobmanager/jobs/job.html @@ -30,8 +30,13 @@  -  - - {{notebookJob.interpreter}} + + + Interpreter is not set + + + {{notebookJob.interpreter}} +
    From 877b203fcf610275255113c7f9a49bb2ee832784 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Mon, 30 May 2016 19:28:08 +0900 Subject: [PATCH 12/25] optimize job management - slow down issue. --- .../zeppelin/socket/NotebookServer.java | 158 +++++++++++++++++- .../app/jobmanager/jobmanager.controller.js | 58 ++++++- .../src/app/jobmanager/jobmanager.html | 2 +- .../src/app/jobmanager/jobs/job.controller.js | 5 +- zeppelin-web/src/app/jobmanager/jobs/job.html | 38 ++--- .../websocketEvents.factory.js | 2 + .../websocketEvents/websocketMsg.service.js | 6 + .../zeppelin/notebook/socket/Message.java | 4 +- 8 files changed, 234 insertions(+), 39 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 61b045fb425..a2347b31169 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -141,6 +141,9 @@ public void onMessage(NotebookSocket conn, String msg) { case LIST_NOTEBOOK_JOBS: unicastNotebookJobInfo(conn); break; + case LIST_UPDATE_NOTEBOOK_JOBS: + unicastUpdateNotebookJobInfo(conn, messagereceived); + break; case RELOAD_NOTES_FROM_REPO: broadcastReloadedNoteList(); break; @@ -353,7 +356,8 @@ private void unicast(Message m, NotebookSocket conn) { } } - public List> generateNotebooksJobInfo(boolean needsReload) { + public List> generateNotebooksJobInfo( + boolean needsReload, boolean isOnlyRunningJob) { Notebook notebook = notebook(); ZeppelinConfiguration conf = notebook.getConf(); @@ -401,6 +405,13 @@ public List> generateNotebooksJobInfo(boolean needsReload) { List> paragraphsInfo = new LinkedList<>(); for (Paragraph paragraph : note.getParagraphs()) { + if (paragraph.getStatus().isRunning() == true) { + isNotebookRunning = true; + } else { + if (isOnlyRunningJob == true) { + continue; + } + } Map paragraphItem = new HashMap<>(); paragraphItem.put("id", paragraph.getId()); String paragraphName = paragraph.getTitle(); @@ -423,19 +434,136 @@ public List> generateNotebooksJobInfo(boolean needsReload) { lastRunningDate = paragaraphDate; } } - lastRunningUnixTime = lastRunningDate.getTime(); + paragraphsInfo.add(paragraphItem); + } + + if (isOnlyRunningJob == true && isNotebookRunning == false) { + continue; + } + + // Interpreter is set does not exist. + String interpreterGroupName = null; + if (note.getNoteReplLoader().getInterpreterSettings() != null + && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { + interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); + } + + info.put("interpreter", interpreterGroupName); + info.put("isRunningJob", isNotebookRunning); + info.put("unixTimeLastRun", lastRunningUnixTime); + info.put("paragraphs", paragraphsInfo); + notesInfo.add(info); + } + + return notesInfo; + } + + + public List> generateUpdateNotebooksJobInfo( + boolean needsReload, long lastUpdateServerUnixTime) { + Notebook notebook = notebook(); + + ZeppelinConfiguration conf = notebook.getConf(); + String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); + boolean hideHomeScreenNotebookFromList = conf + .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); + + if (needsReload) { + try { + notebook.reloadAllNotes(); + } catch (IOException e) { + LOG.error("Fail to reload notes from repository"); + } + } + + List notes = notebook.getAllNotes(); + List> notesInfo = new LinkedList<>(); + for (Note note : notes) { + boolean isNotebookRunning = false; + boolean isUpdateNotebook = false; + Map info = new HashMap<>(); + + if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { + continue; + } + + String cronTypeNotebookKeywork = "cron"; + info.put("notebookId", note.id()); + String notebookName = note.getName(); + if (notebookName != null) { + info.put("notebookName", note.getName()); + } else { + info.put("notebookName", note.id()); + } + + if (note.getConfig().containsKey(cronTypeNotebookKeywork) == true + && !note.getConfig().get(cronTypeNotebookKeywork).equals("")) { + info.put("notebookType", "cron"); + } + else { + info.put("notebookType", "normal"); + } + + Date lastRunningDate = null; + long lastRunningUnixTime = 0; + + List> paragraphsInfo = new LinkedList<>(); + for (Paragraph paragraph : note.getParagraphs()) { + + Date startedDate = paragraph.getDateStarted(); + Date createdDate = paragraph.getDateCreated(); + Date finishedDate = paragraph.getDateFinished(); + + if (startedDate != null && startedDate.getTime() > lastUpdateServerUnixTime) { + isUpdateNotebook = true; + } + if (createdDate != null && createdDate.getTime() > lastUpdateServerUnixTime) { + isUpdateNotebook = true; + } + if (finishedDate != null && finishedDate.getTime() > lastUpdateServerUnixTime) { + isUpdateNotebook = true; + } + + Map paragraphItem = new HashMap<>(); + paragraphItem.put("id", paragraph.getId()); + String paragraphName = paragraph.getTitle(); + if (paragraphName != null) { + paragraphItem.put("name", paragraphName); + } else { + paragraphItem.put("name", paragraph.getId()); + } + paragraphItem.put("status", paragraph.getStatus().toString()); + + Date paragaraphDate = startedDate; + if (paragaraphDate == null) { + paragaraphDate = createdDate; + } + + if (lastRunningDate == null) { + lastRunningDate = paragaraphDate; + } else { + if (lastRunningDate.after(paragaraphDate) == true) { + lastRunningDate = paragaraphDate; + } + } + lastRunningUnixTime = lastRunningDate.getTime(); if (paragraph.getStatus().isRunning() == true) { isNotebookRunning = true; + isUpdateNotebook = true; } paragraphsInfo.add(paragraphItem); } + if (isUpdateNotebook != true) { + continue; + } + // Interpreter is set does not exist. String interpreterGroupName = null; if (note.getNoteReplLoader().getInterpreterSettings() != null - && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { + && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); } @@ -443,7 +571,6 @@ public List> generateNotebooksJobInfo(boolean needsReload) { info.put("isRunningJob", isNotebookRunning); info.put("unixTimeLastRun", lastRunningUnixTime); info.put("paragraphs", paragraphsInfo); - notesInfo.add(info); } @@ -498,8 +625,27 @@ public void unicastNoteList(NotebookSocket conn) { } public void unicastNotebookJobInfo(NotebookSocket conn) { - List> notebookJobs = generateNotebooksJobInfo(false); - unicast(new Message(OP.LIST_NOTEBOOK_JOBS).put("notebookJobs", notebookJobs), conn); + List> notebookJobs = generateNotebooksJobInfo(false, false); + Map response = new HashMap<>(); + + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", notebookJobs); + + unicast(new Message(OP.LIST_NOTEBOOK_JOBS).put("notebookJobs", response), conn); + } + + public void unicastUpdateNotebookJobInfo(NotebookSocket conn, Message fromMessage) { + double lastUpdateUnixTimeRaw = (double) fromMessage.get("lastUpdateUnixTime"); + long lastUpdateUnixTime = new Double(lastUpdateUnixTimeRaw).longValue(); + List> notebookJobs; + notebookJobs = generateUpdateNotebooksJobInfo(false, lastUpdateUnixTime); + Map response = new HashMap<>(); + + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", notebookJobs); + + unicast(new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS) + .put("notebookRunningJobs", response), conn); } public void broadcastReloadedNoteList() { diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index e08696003f4..0d25e45c88c 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -24,6 +24,14 @@ angular.module('zeppelinWebApp') var filterItems = jobItems; + if (RUNNING_ALWAYS_TOP === true) { + var runningJobList = _.where(filterItems, {isRunningJob : true}); + filterItems = _.reject(filterItems, {isRunningJob : true}); + runningJobList.map(function (runningJob) { + filterItems.splice(0,0, runningJob); + }); + } + if (FILTER_VALUE_INTERPRETER === undefined) { filterItems = _.filter(filterItems, function (jobItem) { return jobItem.interpreter === undefined? true : false; @@ -32,10 +40,6 @@ angular.module('zeppelinWebApp') filterItems = _.where(filterItems, {interpreter : FILTER_VALUE_INTERPRETER}); } - if (RUNNING_ALWAYS_TOP === true) { - filterItems = _.sortBy(filterItems, 'isRunningJob').reverse(); - } - if (FILTER_VALUE_NOTEBOOK_NAME !== '') { filterItems = _.filter(filterItems, function(jobItem){ var lowerFilterValue = FILTER_VALUE_NOTEBOOK_NAME.toLocaleLowerCase(); @@ -49,10 +53,12 @@ angular.module('zeppelinWebApp') }) .controller('JobmanagerCtrl', function($scope, $route, $routeParams, $location, $rootScope, $http, - websocketMsgSrv, baseUrlSrv, $interval, SaveAsService, myJobFilter) { + websocketMsgSrv, baseUrlSrv, $interval, $timeout, SaveAsService, myJobFilter) { - $scope.$on('setNotebookJobs', function(event, note) { - $scope.jobInfomations = note; + $scope.$on('setNotebookJobs', function(event, responseData) { + $scope.lastJobServerUnixTime = responseData.lastResponseUnixTime; + $scope.jobInfomations = responseData.jobs; + $scope.jobInfomationsIndexs = $scope.jobInfomations? _.indexBy($scope.jobInfomations, 'notebookId') : {}; $scope.JobInfomationsByFilter = $scope.jobTypeFilter($scope.jobInfomations, $scope.filterConfig); $scope.ACTIVE_INTERPRETERS = [ { @@ -69,6 +75,28 @@ angular.module('zeppelinWebApp') } }); + $scope.$on('setUpdateNotebookJobs', function(event, responseData) { + var jobInfomations = $scope.jobInfomations; + var indexStore = $scope.jobInfomationsIndexs; + $scope.lastJobServerUnixTime = responseData.lastResponseUnixTime; + var notes = responseData.jobs; + notes.map(function (changedItem) { + if (indexStore[changedItem.notebookId] === undefined) { + var newItem = angular.copy(changedItem); + jobInfomations.push(newItem); + indexStore[changedItem.notebookId] = newItem; + } else { + var changeOriginTarget = indexStore[changedItem.notebookId]; + changeOriginTarget.isRunningJob = changedItem.isRunningJob; + changeOriginTarget.notebookName = changedItem.notebookName; + changeOriginTarget.notebookType = changedItem.notebookType; + changeOriginTarget.unixTimeLastRun = changedItem.unixTimeLastRun; + changeOriginTarget.paragraphs = changedItem.paragraphs; + } + $scope.doFiltering(jobInfomations, $scope.filterConfig); + }); + }); + $scope.doFiltering = function (jobInfomations, filterConfig) { $scope.JobInfomationsByFilter = $scope.jobTypeFilter(jobInfomations, filterConfig); }; @@ -78,6 +106,16 @@ angular.module('zeppelinWebApp') $scope.doFiltering(jobInfomations, filterConfig); }; + $scope.doFilterInputTyping = function (keyEvent, jobInfomations, filterConfig) { + var returnKey = 13; + $timeout.cancel($scope.dofilterTimeoutObject); + $scope.dofilterTimeoutObject = $timeout(function(){$scope.doFiltering(jobInfomations, filterConfig);}, 1000); + if (keyEvent.which === returnKey) { + $timeout.cancel($scope.dofilterTimeoutObject); + $scope.doFiltering(jobInfomations, filterConfig); + } + }; + $scope.init = function () { $scope.filterConfig = { RUNNING_ALWAYS_TOP : true, @@ -89,10 +127,12 @@ angular.module('zeppelinWebApp') $scope.JobInfomationsByFilter = $scope.jobInfomations; websocketMsgSrv.getNotebookJobsList(); - var refreshObj = $interval(websocketMsgSrv.getNotebookJobsList, 1000); - + var refreshObj = $interval(function () { + websocketMsgSrv.getUpdateNotebookJobsList($scope.lastJobServerUnixTime); + }, 1000); $scope.$on('$destroy', function() { $interval.cancel(refreshObj); }); + }; }); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.html b/zeppelin-web/src/app/jobmanager/jobmanager.html index b09bc36e61d..31d1f1c3d84 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.html +++ b/zeppelin-web/src/app/jobmanager/jobmanager.html @@ -43,7 +43,7 @@

    + ng-keypress="doFilterInputTyping($event, jobInfomations, filterConfig)"/>

    @@ -48,51 +48,51 @@ ng-switch="paragraphJob.status"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{::paragraphJob.id}}"> + tooltip="{{::paragraphJob.name}} is READY"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{::paragraphJob.id}}"> + tooltip="{{::paragraphJob.name}} is FINISHED"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{::paragraphJob.id}}"> + tooltip="{{::paragraphJob.name}} is ABORT"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{::paragraphJob.id}}"> + tooltip="{{::paragraphJob.name}} is ERROR"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{::paragraphJob.id}}"> + tooltip="{{::paragraphJob.name}} is PENDING"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{::paragraphJob.id}}"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{::paragraphJob.id}}"> + tooltip="{{::paragraphJob.name}} is {{::paragraphJob.status}}"> diff --git a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js index e73a85e1102..9f09858113d 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js +++ b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js @@ -61,6 +61,8 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, $rootScope.$broadcast('setNoteMenu', data.notes); } else if (op === 'LIST_NOTEBOOK_JOBS') { $rootScope.$broadcast('setNotebookJobs', data.notebookJobs); + } else if (op === 'LIST_UPDATE_NOTEBOOK_JOBS') { + $rootScope.$broadcast('setUpdateNotebookJobs', data.notebookRunningJobs); } else if (op === 'AUTH_INFO') { BootstrapDialog.show({ closable: true, diff --git a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js index 13bfe9d5471..35bffd22188 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js +++ b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js @@ -41,6 +41,12 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, websocketEvents.sendNewEvent({op: 'LIST_NOTEBOOK_JOBS'}); }, + getUpdateNotebookJobsList: function(lastUpdateServerUnixTime) { + websocketEvents.sendNewEvent( + {op: 'LIST_UPDATE_NOTEBOOK_JOBS', data : {lastUpdateUnixTime : lastUpdateServerUnixTime*1}} + ); + }, + reloadAllNotesFromRepo: function() { websocketEvents.sendNewEvent({op: 'RELOAD_NOTES_FROM_REPO'}); }, diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java index 36e373a09c8..74a0b9dc063 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java @@ -114,7 +114,9 @@ public static enum OP { CHECKPOINT_NOTEBOOK, // [c-s] checkpoint notebook to storage repository // @param noteId // @param checkpointName - LIST_NOTEBOOK_JOBS + LIST_NOTEBOOK_JOBS, // [c-s] get notebook job management infomations + LIST_UPDATE_NOTEBOOK_JOBS // [c-s] get job management informations for until unixtime + // @param unixTime } From 5a781369771e0039f4bde73c0f9db44fc1e38817 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Tue, 31 May 2016 11:46:02 +0900 Subject: [PATCH 13/25] add comment and change variable name. --- .../zeppelin/socket/NotebookServer.java | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index a2347b31169..56727b7ab29 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -356,8 +356,7 @@ private void unicast(Message m, NotebookSocket conn) { } } - public List> generateNotebooksJobInfo( - boolean needsReload, boolean isOnlyRunningJob) { + public List> generateNotebooksJobInfo(boolean needsReload) { Notebook notebook = notebook(); ZeppelinConfiguration conf = notebook.getConf(); @@ -383,7 +382,7 @@ public List> generateNotebooksJobInfo( continue; } - String cronTypeNotebookKeywork = "cron"; + String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; info.put("notebookId", note.id()); String notebookName = note.getName(); if (notebookName != null) { @@ -392,8 +391,8 @@ public List> generateNotebooksJobInfo( info.put("notebookName", note.id()); } - if (note.getConfig().containsKey(cronTypeNotebookKeywork) == true - && !note.getConfig().get(cronTypeNotebookKeywork).equals("")) { + if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true + && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { info.put("notebookType", "cron"); } else { @@ -407,26 +406,27 @@ public List> generateNotebooksJobInfo( for (Paragraph paragraph : note.getParagraphs()) { if (paragraph.getStatus().isRunning() == true) { isNotebookRunning = true; - } else { - if (isOnlyRunningJob == true) { - continue; - } } Map paragraphItem = new HashMap<>(); + // set paragraph id paragraphItem.put("id", paragraph.getId()); + + // set paragraph name String paragraphName = paragraph.getTitle(); if (paragraphName != null) { paragraphItem.put("name", paragraphName); } else { paragraphItem.put("name", paragraph.getId()); } + + // set status for paragraph. paragraphItem.put("status", paragraph.getStatus().toString()); + // get last update time. Date paragaraphDate = paragraph.getDateStarted(); if (paragaraphDate == null) { paragaraphDate = paragraph.getDateCreated(); } - if (lastRunningDate == null) { lastRunningDate = paragaraphDate; } else { @@ -434,15 +434,13 @@ public List> generateNotebooksJobInfo( lastRunningDate = paragaraphDate; } } + + // convert date to unixtime(ms). lastRunningUnixTime = lastRunningDate.getTime(); paragraphsInfo.add(paragraphItem); } - if (isOnlyRunningJob == true && isNotebookRunning == false) { - continue; - } - // Interpreter is set does not exist. String interpreterGroupName = null; if (note.getNoteReplLoader().getInterpreterSettings() != null @@ -450,6 +448,7 @@ public List> generateNotebooksJobInfo( interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); } + // notebook json object root information. info.put("interpreter", interpreterGroupName); info.put("isRunningJob", isNotebookRunning); info.put("unixTimeLastRun", lastRunningUnixTime); @@ -483,13 +482,15 @@ public List> generateUpdateNotebooksJobInfo( for (Note note : notes) { boolean isNotebookRunning = false; boolean isUpdateNotebook = false; + Map info = new HashMap<>(); if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { continue; } - String cronTypeNotebookKeywork = "cron"; + // set const keyword for cron type + String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; info.put("notebookId", note.id()); String notebookName = note.getName(); if (notebookName != null) { @@ -498,8 +499,8 @@ public List> generateUpdateNotebooksJobInfo( info.put("notebookName", note.id()); } - if (note.getConfig().containsKey(cronTypeNotebookKeywork) == true - && !note.getConfig().get(cronTypeNotebookKeywork).equals("")) { + if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true + && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { info.put("notebookType", "cron"); } else { @@ -512,6 +513,7 @@ public List> generateUpdateNotebooksJobInfo( List> paragraphsInfo = new LinkedList<>(); for (Paragraph paragraph : note.getParagraphs()) { + // check date for update time. Date startedDate = paragraph.getDateStarted(); Date createdDate = paragraph.getDateCreated(); Date finishedDate = paragraph.getDateFinished(); @@ -527,13 +529,19 @@ public List> generateUpdateNotebooksJobInfo( } Map paragraphItem = new HashMap<>(); + + // set paragraph id paragraphItem.put("id", paragraph.getId()); + + // set paragraph name String paragraphName = paragraph.getTitle(); if (paragraphName != null) { paragraphItem.put("name", paragraphName); } else { paragraphItem.put("name", paragraph.getId()); } + + // set status for paragraph paragraphItem.put("status", paragraph.getStatus().toString()); Date paragaraphDate = startedDate; @@ -541,6 +549,7 @@ public List> generateUpdateNotebooksJobInfo( paragaraphDate = createdDate; } + // set last update unixtime(ms). if (lastRunningDate == null) { lastRunningDate = paragaraphDate; } else { @@ -556,6 +565,7 @@ public List> generateUpdateNotebooksJobInfo( paragraphsInfo.add(paragraphItem); } + // Insert only data that has changed. if (isUpdateNotebook != true) { continue; } @@ -567,6 +577,7 @@ public List> generateUpdateNotebooksJobInfo( interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); } + // set notebook root information. info.put("interpreter", interpreterGroupName); info.put("isRunningJob", isNotebookRunning); info.put("unixTimeLastRun", lastRunningUnixTime); @@ -625,7 +636,7 @@ public void unicastNoteList(NotebookSocket conn) { } public void unicastNotebookJobInfo(NotebookSocket conn) { - List> notebookJobs = generateNotebooksJobInfo(false, false); + List> notebookJobs = generateNotebooksJobInfo(false); Map response = new HashMap<>(); response.put("lastResponseUnixTime", System.currentTimeMillis()); From caf4f3112f4541232556dd0dbc5ae21500d8f7d0 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Tue, 31 May 2016 17:04:59 +0900 Subject: [PATCH 14/25] was divided zeppelin-notebook server - job manager server. --- .../zeppelin/server/ZeppelinServer.java | 18 + .../zeppelin/socket/JobManagerServer.java | 509 ++++++++++++++++++ .../zeppelin/socket/JobManagerSocket.java | 71 +++ .../socket/JobManagerSocketListener.java | 26 + .../socket/JobManagerWebSocketCreator.java | 36 ++ .../zeppelin/socket/NotebookServer.java | 263 +-------- .../app/jobmanager/jobmanager.controller.js | 3 + .../src/app/jobmanager/jobmanager.html | 2 +- zeppelin-web/src/app/jobmanager/jobs/job.html | 4 +- .../src/app/notebook/notebook.controller.js | 4 +- .../src/components/baseUrl/baseUrl.service.js | 5 + .../components/navbar/navbar.controller.js | 2 +- .../websocketEvents.factory.js | 100 +++- .../websocketEvents/websocketMsg.service.js | 64 +-- 14 files changed, 783 insertions(+), 324 deletions(-) create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerServer.java create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocket.java create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocketListener.java create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerWebSocketCreator.java diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index c8fc60a46b4..8b075592914 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -30,6 +30,7 @@ import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.search.LuceneSearch; import org.apache.zeppelin.search.SearchService; +import org.apache.zeppelin.socket.JobManagerServer; import org.apache.zeppelin.socket.NotebookServer; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.server.*; @@ -61,6 +62,7 @@ public class ZeppelinServer extends Application { public static Notebook notebook; public static Server jettyWebServer; public static NotebookServer notebookWsServer; + public static JobManagerServer jobManagerWsServer; private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; @@ -101,6 +103,9 @@ public static void main(String[] args) throws InterruptedException { // REST api setupRestApiContextHandler(webApp, conf); + // Job Manager server + setupJobManagerServer(webApp, conf); + // Notebook server setupNotebookServer(webApp, conf); @@ -202,6 +207,19 @@ private static void setupNotebookServer(WebAppContext webapp, webapp.addServlet(servletHolder, "/ws/*"); } + private static void setupJobManagerServer(WebAppContext webapp, + ZeppelinConfiguration conf) { + jobManagerWsServer = new JobManagerServer(); + String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); + final ServletHolder servletHolder = new ServletHolder(jobManagerWsServer); + servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); + + final ServletContextHandler cxfContext = new ServletContextHandler( + ServletContextHandler.SESSIONS); + + webapp.addServlet(servletHolder, "/ws/job/*"); + } + private static SslContextFactory getSslContextFactory(ZeppelinConfiguration conf) { SslContextFactory sslContextFactory = new SslContextFactory(); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerServer.java new file mode 100644 index 00000000000..52c49a0e4da --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerServer.java @@ -0,0 +1,509 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.socket; + +import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import org.apache.zeppelin.notebook.*; +import org.apache.zeppelin.notebook.socket.Message; +import org.apache.zeppelin.notebook.socket.Message.OP; +import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.server.ZeppelinServer; +import org.apache.zeppelin.ticket.TicketContainer; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.apache.zeppelin.utils.SecurityUtils; +import org.eclipse.jetty.websocket.servlet.WebSocketServlet; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.quartz.SchedulerException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * Zeppelin websocket service. + */ +public class JobManagerServer extends WebSocketServlet implements + JobManagerSocketListener, JobListenerFactory { + private static final Logger LOG = LoggerFactory.getLogger(JobManagerServer.class); + Gson gson = new GsonBuilder() + .setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); + final Map> jobManagerSocketMap = new HashMap<>(); + final Queue connectedSockets = new ConcurrentLinkedQueue<>(); + + private Notebook notebook() { + return ZeppelinServer.notebook; + } + + @Override + public void configure(WebSocketServletFactory factory) { + factory.setCreator(new JobManagerWebSocketCreator(this)); + } + + public boolean checkOrigin(HttpServletRequest request, String origin) { + try { + return SecurityUtils.isValidOrigin(origin, ZeppelinConfiguration.create()); + } catch (UnknownHostException e) { + LOG.error(e.toString(), e); + } catch (URISyntaxException e) { + LOG.error(e.toString(), e); + } + return false; + } + + public JobManagerSocket doWebSocketConnect(HttpServletRequest req, String protocol) { + return new JobManagerSocket(req, protocol, this); + } + + @Override + public void onOpen(JobManagerSocket conn) { + LOG.info("New connection from {} : {}", conn.getRequest().getRemoteAddr(), + conn.getRequest().getRemotePort()); + connectedSockets.add(conn); + } + + @Override + public void onMessage(JobManagerSocket conn, String msg) { + Notebook notebook = notebook(); + try { + Message messagereceived = deserializeMessage(msg); + LOG.debug("RECEIVE << " + messagereceived.op); + LOG.debug("RECEIVE PRINCIPAL << " + messagereceived.principal); + LOG.debug("RECEIVE TICKET << " + messagereceived.ticket); + LOG.debug("RECEIVE ROLES << " + messagereceived.roles); + + if (LOG.isTraceEnabled()) { + LOG.trace("RECEIVE MSG = " + messagereceived); + } + + String ticket = TicketContainer.instance.getTicket(messagereceived.principal); + if (ticket != null && !ticket.equals(messagereceived.ticket)) + throw new Exception("Invalid ticket " + messagereceived.ticket + " != " + ticket); + + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + boolean allowAnonymous = conf. + getBoolean(ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED); + if (!allowAnonymous && messagereceived.principal.equals("anonymous")) { + throw new Exception("Anonymous access not allowed "); + } + + HashSet userAndRoles = new HashSet(); + userAndRoles.add(messagereceived.principal); + if (!messagereceived.roles.equals("")) { + HashSet roles = gson.fromJson(messagereceived.roles, + new TypeToken>(){}.getType()); + if (roles != null) { + userAndRoles.addAll(roles); + } + } + + LOG.info("lcs job received"); + /** Lets be elegant here */ + switch (messagereceived.op) { + case LIST_NOTEBOOK_JOBS: + unicastNotebookJobInfo(conn); + break; + case LIST_UPDATE_NOTEBOOK_JOBS: + unicastUpdateNotebookJobInfo(conn, messagereceived); + break; + case PING: + break; //do nothing + default: + break; + } + } catch (Exception e) { + LOG.error("Can't handle message", e); + } + } + + @Override + public void onClose(JobManagerSocket conn, int code, String reason) { + LOG.info("Closed connection to {} : {}. ({}) {}", conn.getRequest() + .getRemoteAddr(), conn.getRequest().getRemotePort(), code, reason); + connectedSockets.remove(conn); + } + + protected Message deserializeMessage(String msg) { + return gson.fromJson(msg, Message.class); + } + + protected String serializeMessage(Message m) { + return gson.toJson(m); + } + + private void addConnectionToJobManager(String jobPart, JobManagerSocket socket) { + synchronized (jobManagerSocketMap) { + // single note. + List socketList = jobManagerSocketMap.get(jobPart); + if (socketList == null) { + socketList = new LinkedList<>(); + jobManagerSocketMap.put(jobPart, socketList); + } + if (!socketList.contains(socket)) { + socketList.add(socket); + } + } + } + + private void removeConnectionToJobManager(String jobPart, JobManagerSocket socket) { + synchronized (jobManagerSocketMap) { + List socketList = jobManagerSocketMap.get(jobPart); + if (socketList != null) { + socketList.remove(socket); + } + } + } + + private void broadcast(String noteId, Message m) { + synchronized (jobManagerSocketMap) { + List socketLists = jobManagerSocketMap.get(noteId); + if (socketLists == null || socketLists.size() == 0) { + return; + } + LOG.debug("SEND >> " + m.op); + for (JobManagerSocket conn : socketLists) { + try { + conn.send(serializeMessage(m)); + } catch (IOException e) { + LOG.error("socket error", e); + } + } + } + } + + private void broadcastExcept(String noteId, Message m, JobManagerSocket exclude) { + synchronized (jobManagerSocketMap) { + List socketLists = jobManagerSocketMap.get(noteId); + if (socketLists == null || socketLists.size() == 0) { + return; + } + LOG.debug("SEND >> " + m.op); + for (JobManagerSocket conn : socketLists) { + if (exclude.equals(conn)) { + continue; + } + try { + conn.send(serializeMessage(m)); + } catch (IOException e) { + LOG.error("socket error", e); + } + } + } + } + + private void broadcastAll(Message m) { + for (JobManagerSocket conn : connectedSockets) { + try { + conn.send(serializeMessage(m)); + } catch (IOException e) { + LOG.error("socket error", e); + } + } + } + + private void unicast(Message m, JobManagerSocket conn) { + try { + conn.send(serializeMessage(m)); + } catch (IOException e) { + LOG.error("socket error", e); + } + } + + public void unicastNotebookJobInfo(JobManagerSocket conn) { + List> notebookJobs = generateNotebooksJobInfo(false); + Map response = new HashMap<>(); + + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", notebookJobs); + + unicast(new Message(OP.LIST_NOTEBOOK_JOBS).put("notebookJobs", response), conn); + } + + public void unicastUpdateNotebookJobInfo(JobManagerSocket conn, Message fromMessage) { + double lastUpdateUnixTimeRaw = (double) fromMessage.get("lastUpdateUnixTime"); + long lastUpdateUnixTime = new Double(lastUpdateUnixTimeRaw).longValue(); + List> notebookJobs; + notebookJobs = generateUpdateNotebooksJobInfo(false, lastUpdateUnixTime); + Map response = new HashMap<>(); + + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", notebookJobs); + + unicast(new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS) + .put("notebookRunningJobs", response), conn); + } + + + public List> generateNotebooksJobInfo(boolean needsReload) { + Notebook notebook = notebook(); + + ZeppelinConfiguration conf = notebook.getConf(); + String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); + boolean hideHomeScreenNotebookFromList = conf + .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); + + if (needsReload) { + try { + notebook.reloadAllNotes(); + } catch (IOException e) { + LOG.error("Fail to reload notes from repository"); + } + } + + List notes = notebook.getAllNotes(); + List> notesInfo = new LinkedList<>(); + for (Note note : notes) { + boolean isNotebookRunning = false; + Map info = new HashMap<>(); + + if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { + continue; + } + + String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; + info.put("notebookId", note.id()); + String notebookName = note.getName(); + if (notebookName != null) { + info.put("notebookName", note.getName()); + } else { + info.put("notebookName", note.id()); + } + + if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true + && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { + info.put("notebookType", "cron"); + } + else { + info.put("notebookType", "normal"); + } + + Date lastRunningDate = null; + long lastRunningUnixTime = 0; + + List> paragraphsInfo = new LinkedList<>(); + for (Paragraph paragraph : note.getParagraphs()) { + if (paragraph.getStatus().isRunning() == true) { + isNotebookRunning = true; + } + Map paragraphItem = new HashMap<>(); + // set paragraph id + paragraphItem.put("id", paragraph.getId()); + + // set paragraph name + String paragraphName = paragraph.getTitle(); + if (paragraphName != null) { + paragraphItem.put("name", paragraphName); + } else { + paragraphItem.put("name", paragraph.getId()); + } + + // set status for paragraph. + paragraphItem.put("status", paragraph.getStatus().toString()); + + // get last update time. + Date paragaraphDate = paragraph.getDateStarted(); + if (paragaraphDate == null) { + paragaraphDate = paragraph.getDateCreated(); + } + if (lastRunningDate == null) { + lastRunningDate = paragaraphDate; + } else { + if (lastRunningDate.after(paragaraphDate) == true) { + lastRunningDate = paragaraphDate; + } + } + + // convert date to unixtime(ms). + lastRunningUnixTime = lastRunningDate.getTime(); + + paragraphsInfo.add(paragraphItem); + } + + // Interpreter is set does not exist. + String interpreterGroupName = null; + if (note.getNoteReplLoader().getInterpreterSettings() != null + && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { + interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); + } + + // notebook json object root information. + info.put("interpreter", interpreterGroupName); + info.put("isRunningJob", isNotebookRunning); + info.put("unixTimeLastRun", lastRunningUnixTime); + info.put("paragraphs", paragraphsInfo); + notesInfo.add(info); + } + return notesInfo; + } + + + public List> generateUpdateNotebooksJobInfo( + boolean needsReload, long lastUpdateServerUnixTime) { + Notebook notebook = notebook(); + + ZeppelinConfiguration conf = notebook.getConf(); + String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); + boolean hideHomeScreenNotebookFromList = conf + .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); + + if (needsReload) { + try { + notebook.reloadAllNotes(); + } catch (IOException e) { + LOG.error("Fail to reload notes from repository"); + } + } + + List notes = notebook.getAllNotes(); + List> notesInfo = new LinkedList<>(); + for (Note note : notes) { + boolean isNotebookRunning = false; + boolean isUpdateNotebook = false; + + Map info = new HashMap<>(); + + if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { + continue; + } + + // set const keyword for cron type + String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; + info.put("notebookId", note.id()); + String notebookName = note.getName(); + if (notebookName != null) { + info.put("notebookName", note.getName()); + } else { + info.put("notebookName", note.id()); + } + + + if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true + && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { + info.put("notebookType", "cron"); + } + else { + info.put("notebookType", "normal"); + } + + Date lastRunningDate = null; + long lastRunningUnixTime = 0; + + List> paragraphsInfo = new LinkedList<>(); + for (Paragraph paragraph : note.getParagraphs()) { + + // check date for update time. + Date startedDate = paragraph.getDateStarted(); + Date createdDate = paragraph.getDateCreated(); + Date finishedDate = paragraph.getDateFinished(); + + if (startedDate != null && startedDate.getTime() > lastUpdateServerUnixTime) { + isUpdateNotebook = true; + } + if (createdDate != null && createdDate.getTime() > lastUpdateServerUnixTime) { + isUpdateNotebook = true; + } + if (finishedDate != null && finishedDate.getTime() > lastUpdateServerUnixTime) { + isUpdateNotebook = true; + } + + Map paragraphItem = new HashMap<>(); + + // set paragraph id + paragraphItem.put("id", paragraph.getId()); + + // set paragraph name + String paragraphName = paragraph.getTitle(); + if (paragraphName != null) { + paragraphItem.put("name", paragraphName); + } else { + paragraphItem.put("name", paragraph.getId()); + } + + // set status for paragraph + paragraphItem.put("status", paragraph.getStatus().toString()); + + Date paragaraphDate = startedDate; + if (paragaraphDate == null) { + paragaraphDate = createdDate; + } + + // set last update unixtime(ms). + if (lastRunningDate == null) { + lastRunningDate = paragaraphDate; + } else { + if (lastRunningDate.after(paragaraphDate) == true) { + lastRunningDate = paragaraphDate; + } + } + lastRunningUnixTime = lastRunningDate.getTime(); + if (paragraph.getStatus().isRunning() == true) { + isNotebookRunning = true; + isUpdateNotebook = true; + } + paragraphsInfo.add(paragraphItem); + } + + // Insert only data that has changed. + if (isUpdateNotebook != true) { + continue; + } + + // Interpreter is set does not exist. + String interpreterGroupName = null; + if (note.getNoteReplLoader().getInterpreterSettings() != null + && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { + interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); + } + + // set notebook root information. + info.put("interpreter", interpreterGroupName); + info.put("isRunningJob", isNotebookRunning); + info.put("unixTimeLastRun", lastRunningUnixTime); + info.put("paragraphs", paragraphsInfo); + notesInfo.add(info); + } + if (notesInfo.size() > 0) { + LOG.info("update count {}", notesInfo.size()); + } + return notesInfo; + } + + @Override + public ParagraphJobListener getParagraphJobListener(Note note) { + return null; + } +} + diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocket.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocket.java new file mode 100644 index 00000000000..35201897e9b --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocket.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.socket; + +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketAdapter; + +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; + +/** + * Notebook websocket + */ +public class JobManagerSocket extends WebSocketAdapter { + + private Session connection; + private JobManagerSocketListener listener; + private HttpServletRequest request; + private String protocol; + + public JobManagerSocket(HttpServletRequest req, String protocol, + JobManagerSocketListener listener) { + this.listener = listener; + this.request = req; + this.protocol = protocol; + } + + @Override + public void onWebSocketClose(int closeCode, String message) { + listener.onClose(this, closeCode, message); + } + + @Override + public void onWebSocketConnect(Session connection) { + this.connection = connection; + listener.onOpen(this); + } + + @Override + public void onWebSocketText(String message) { + listener.onMessage(this, message); + } + + + public HttpServletRequest getRequest() { + return request; + } + + public String getProtocol() { + return protocol; + } + + public void send(String serializeMessage) throws IOException { + connection.getRemote().sendString(serializeMessage); + } + +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocketListener.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocketListener.java new file mode 100644 index 00000000000..5f44eebb755 --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocketListener.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.socket; + +/** + * NoteboookSocket listener + */ +public interface JobManagerSocketListener { + void onClose(JobManagerSocket socket, int code, String message); + void onOpen(JobManagerSocket socket); + void onMessage(JobManagerSocket socket, String message); +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerWebSocketCreator.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerWebSocketCreator.java new file mode 100644 index 00000000000..d14a7ea49fa --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerWebSocketCreator.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.socket; + +import org.eclipse.jetty.websocket.servlet.ServletUpgradeRequest; +import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse; +import org.eclipse.jetty.websocket.servlet.WebSocketCreator; + +/** + * Responsible to create the WebSockets for the NotebookServer. + */ +public class JobManagerWebSocketCreator implements WebSocketCreator { + private JobManagerServer jobManagerServer; + + public JobManagerWebSocketCreator(JobManagerServer jobManagerServer) { + this.jobManagerServer = jobManagerServer; + } + public Object createWebSocket(ServletUpgradeRequest request, ServletUpgradeResponse response) { + return new JobManagerSocket(request.getHttpServletRequest(), "", jobManagerServer); + } + +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 56727b7ab29..521402aebf1 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -132,18 +132,12 @@ public void onMessage(NotebookSocket conn, String msg) { userAndRoles.addAll(roles); } } - + LOG.info("lcs notebook received"); /** Lets be elegant here */ switch (messagereceived.op) { case LIST_NOTES: unicastNoteList(conn); break; - case LIST_NOTEBOOK_JOBS: - unicastNotebookJobInfo(conn); - break; - case LIST_UPDATE_NOTEBOOK_JOBS: - unicastUpdateNotebookJobInfo(conn, messagereceived); - break; case RELOAD_NOTES_FROM_REPO: broadcastReloadedNoteList(); break; @@ -356,237 +350,6 @@ private void unicast(Message m, NotebookSocket conn) { } } - public List> generateNotebooksJobInfo(boolean needsReload) { - Notebook notebook = notebook(); - - ZeppelinConfiguration conf = notebook.getConf(); - String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); - boolean hideHomeScreenNotebookFromList = conf - .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); - - if (needsReload) { - try { - notebook.reloadAllNotes(); - } catch (IOException e) { - LOG.error("Fail to reload notes from repository"); - } - } - - List notes = notebook.getAllNotes(); - List> notesInfo = new LinkedList<>(); - for (Note note : notes) { - boolean isNotebookRunning = false; - Map info = new HashMap<>(); - - if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { - continue; - } - - String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; - info.put("notebookId", note.id()); - String notebookName = note.getName(); - if (notebookName != null) { - info.put("notebookName", note.getName()); - } else { - info.put("notebookName", note.id()); - } - - if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true - && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { - info.put("notebookType", "cron"); - } - else { - info.put("notebookType", "normal"); - } - - Date lastRunningDate = null; - long lastRunningUnixTime = 0; - - List> paragraphsInfo = new LinkedList<>(); - for (Paragraph paragraph : note.getParagraphs()) { - if (paragraph.getStatus().isRunning() == true) { - isNotebookRunning = true; - } - Map paragraphItem = new HashMap<>(); - // set paragraph id - paragraphItem.put("id", paragraph.getId()); - - // set paragraph name - String paragraphName = paragraph.getTitle(); - if (paragraphName != null) { - paragraphItem.put("name", paragraphName); - } else { - paragraphItem.put("name", paragraph.getId()); - } - - // set status for paragraph. - paragraphItem.put("status", paragraph.getStatus().toString()); - - // get last update time. - Date paragaraphDate = paragraph.getDateStarted(); - if (paragaraphDate == null) { - paragaraphDate = paragraph.getDateCreated(); - } - if (lastRunningDate == null) { - lastRunningDate = paragaraphDate; - } else { - if (lastRunningDate.after(paragaraphDate) == true) { - lastRunningDate = paragaraphDate; - } - } - - // convert date to unixtime(ms). - lastRunningUnixTime = lastRunningDate.getTime(); - - paragraphsInfo.add(paragraphItem); - } - - // Interpreter is set does not exist. - String interpreterGroupName = null; - if (note.getNoteReplLoader().getInterpreterSettings() != null - && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { - interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); - } - - // notebook json object root information. - info.put("interpreter", interpreterGroupName); - info.put("isRunningJob", isNotebookRunning); - info.put("unixTimeLastRun", lastRunningUnixTime); - info.put("paragraphs", paragraphsInfo); - notesInfo.add(info); - } - - return notesInfo; - } - - - public List> generateUpdateNotebooksJobInfo( - boolean needsReload, long lastUpdateServerUnixTime) { - Notebook notebook = notebook(); - - ZeppelinConfiguration conf = notebook.getConf(); - String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); - boolean hideHomeScreenNotebookFromList = conf - .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); - - if (needsReload) { - try { - notebook.reloadAllNotes(); - } catch (IOException e) { - LOG.error("Fail to reload notes from repository"); - } - } - - List notes = notebook.getAllNotes(); - List> notesInfo = new LinkedList<>(); - for (Note note : notes) { - boolean isNotebookRunning = false; - boolean isUpdateNotebook = false; - - Map info = new HashMap<>(); - - if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { - continue; - } - - // set const keyword for cron type - String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; - info.put("notebookId", note.id()); - String notebookName = note.getName(); - if (notebookName != null) { - info.put("notebookName", note.getName()); - } else { - info.put("notebookName", note.id()); - } - - if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true - && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { - info.put("notebookType", "cron"); - } - else { - info.put("notebookType", "normal"); - } - - Date lastRunningDate = null; - long lastRunningUnixTime = 0; - - List> paragraphsInfo = new LinkedList<>(); - for (Paragraph paragraph : note.getParagraphs()) { - - // check date for update time. - Date startedDate = paragraph.getDateStarted(); - Date createdDate = paragraph.getDateCreated(); - Date finishedDate = paragraph.getDateFinished(); - - if (startedDate != null && startedDate.getTime() > lastUpdateServerUnixTime) { - isUpdateNotebook = true; - } - if (createdDate != null && createdDate.getTime() > lastUpdateServerUnixTime) { - isUpdateNotebook = true; - } - if (finishedDate != null && finishedDate.getTime() > lastUpdateServerUnixTime) { - isUpdateNotebook = true; - } - - Map paragraphItem = new HashMap<>(); - - // set paragraph id - paragraphItem.put("id", paragraph.getId()); - - // set paragraph name - String paragraphName = paragraph.getTitle(); - if (paragraphName != null) { - paragraphItem.put("name", paragraphName); - } else { - paragraphItem.put("name", paragraph.getId()); - } - - // set status for paragraph - paragraphItem.put("status", paragraph.getStatus().toString()); - - Date paragaraphDate = startedDate; - if (paragaraphDate == null) { - paragaraphDate = createdDate; - } - - // set last update unixtime(ms). - if (lastRunningDate == null) { - lastRunningDate = paragaraphDate; - } else { - if (lastRunningDate.after(paragaraphDate) == true) { - lastRunningDate = paragaraphDate; - } - } - lastRunningUnixTime = lastRunningDate.getTime(); - if (paragraph.getStatus().isRunning() == true) { - isNotebookRunning = true; - isUpdateNotebook = true; - } - paragraphsInfo.add(paragraphItem); - } - - // Insert only data that has changed. - if (isUpdateNotebook != true) { - continue; - } - - // Interpreter is set does not exist. - String interpreterGroupName = null; - if (note.getNoteReplLoader().getInterpreterSettings() != null - && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { - interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); - } - - // set notebook root information. - info.put("interpreter", interpreterGroupName); - info.put("isRunningJob", isNotebookRunning); - info.put("unixTimeLastRun", lastRunningUnixTime); - info.put("paragraphs", paragraphsInfo); - notesInfo.add(info); - } - - return notesInfo; - } public List> generateNotebooksInfo(boolean needsReload) { Notebook notebook = notebook(); @@ -635,30 +398,6 @@ public void unicastNoteList(NotebookSocket conn) { unicast(new Message(OP.NOTES_INFO).put("notes", notesInfo), conn); } - public void unicastNotebookJobInfo(NotebookSocket conn) { - List> notebookJobs = generateNotebooksJobInfo(false); - Map response = new HashMap<>(); - - response.put("lastResponseUnixTime", System.currentTimeMillis()); - response.put("jobs", notebookJobs); - - unicast(new Message(OP.LIST_NOTEBOOK_JOBS).put("notebookJobs", response), conn); - } - - public void unicastUpdateNotebookJobInfo(NotebookSocket conn, Message fromMessage) { - double lastUpdateUnixTimeRaw = (double) fromMessage.get("lastUpdateUnixTime"); - long lastUpdateUnixTime = new Double(lastUpdateUnixTimeRaw).longValue(); - List> notebookJobs; - notebookJobs = generateUpdateNotebooksJobInfo(false, lastUpdateUnixTime); - Map response = new HashMap<>(); - - response.put("lastResponseUnixTime", System.currentTimeMillis()); - response.put("jobs", notebookJobs); - - unicast(new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS) - .put("notebookRunningJobs", response), conn); - } - public void broadcastReloadedNoteList() { List> notesInfo = generateNotebooksInfo(true); broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo)); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 0d25e45c88c..13f17d6417f 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -85,7 +85,9 @@ angular.module('zeppelinWebApp') var newItem = angular.copy(changedItem); jobInfomations.push(newItem); indexStore[changedItem.notebookId] = newItem; + console.log('111111111111111111111111'); } else { + console.log('222222222222222222222222'); var changeOriginTarget = indexStore[changedItem.notebookId]; changeOriginTarget.isRunningJob = changedItem.isRunningJob; changeOriginTarget.notebookName = changedItem.notebookName; @@ -108,6 +110,7 @@ angular.module('zeppelinWebApp') $scope.doFilterInputTyping = function (keyEvent, jobInfomations, filterConfig) { var returnKey = 13; + console.log('key ', keyEvent.which); $timeout.cancel($scope.dofilterTimeoutObject); $scope.dofilterTimeoutObject = $timeout(function(){$scope.doFiltering(jobInfomations, filterConfig);}, 1000); if (keyEvent.which === returnKey) { diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.html b/zeppelin-web/src/app/jobmanager/jobmanager.html index 31d1f1c3d84..443f7a8ff8d 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.html +++ b/zeppelin-web/src/app/jobmanager/jobmanager.html @@ -43,7 +43,7 @@

    + ng-keyup="doFilterInputTyping($event, jobInfomations, filterConfig)"/>

    - +
    @@ -39,7 +39,7 @@ - +
    diff --git a/zeppelin-web/src/app/notebook/notebook.controller.js b/zeppelin-web/src/app/notebook/notebook.controller.js index 4149311c3bd..3011e7f29cf 100644 --- a/zeppelin-web/src/app/notebook/notebook.controller.js +++ b/zeppelin-web/src/app/notebook/notebook.controller.js @@ -43,7 +43,7 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', var connectedOnce = false; - $scope.$on('setConnectedStatus', function(event, param) { + $scope.$on('setNotebookWSConnectedStatus', function(event, param) { if(connectedOnce && param){ initNotebook(); } @@ -694,7 +694,7 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', console.log('Error %o %o', status, data.message); BootstrapDialog.show({ closable: true, - title: 'Insufficient privileges', + title: 'Insufficient privileges', message: data.message, buttons: [{ label: 'Login', diff --git a/zeppelin-web/src/components/baseUrl/baseUrl.service.js b/zeppelin-web/src/components/baseUrl/baseUrl.service.js index f06eef3c4d5..e836212371d 100644 --- a/zeppelin-web/src/components/baseUrl/baseUrl.service.js +++ b/zeppelin-web/src/components/baseUrl/baseUrl.service.js @@ -35,6 +35,11 @@ angular.module('zeppelinWebApp').service('baseUrlSrv', function() { return wsProtocol + '//' + location.hostname + ':' + this.getPort() + skipTrailingSlash(location.pathname) + '/ws'; }; + this.getJobManagerWebsocketUrl = function() { + var wsProtocol = location.protocol === 'https:' ? 'wss:' : 'ws:'; + return wsProtocol + '//' + location.hostname + ':' + this.getPort() + skipTrailingSlash(location.pathname) + '/ws/job'; + }; + this.getRestApiBase = function() { return location.protocol + '//' + location.hostname + ':' + this.getPort() + skipTrailingSlash(location.pathname) + '/api'; }; diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index 50c769f1c3c..22ce13a1d77 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -42,7 +42,7 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco notebookListDataFactory.setNotes(notes); }); - $scope.$on('setConnectedStatus', function(event, param) { + $scope.$on('setNotebookWSConnectedStatus', function(event, param) { vm.connected = param; }); diff --git a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js index 9f09858113d..2ed254798a8 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js +++ b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js @@ -14,20 +14,21 @@ 'use strict'; angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, $websocket, $location, baseUrlSrv) { - var websocketCalls = {}; + var websocketCalls = { notebookServer : {}, jobManagerServer : {} }; - websocketCalls.ws = $websocket(baseUrlSrv.getWebsocketUrl()); - websocketCalls.ws.reconnectIfNotNormalClose = true; + // create socket for notebook server + websocketCalls.notebookServer.ws = $websocket(baseUrlSrv.getWebsocketUrl()); + websocketCalls.notebookServer.ws.reconnectIfNotNormalClose = true; - websocketCalls.ws.onOpen(function() { - console.log('Websocket created'); - $rootScope.$broadcast('setConnectedStatus', true); + websocketCalls.notebookServer.ws.onOpen(function() { + console.log('Websocket Notebook Server created'); + $rootScope.$broadcast('setNotebookWSConnectedStatus', true); setInterval(function(){ - websocketCalls.sendNewEvent({op: 'PING'}); + websocketCalls.notebookServer.sendNewEvent({op: 'PING'}); }, 10000); }); - websocketCalls.sendNewEvent = function(data) { + websocketCalls.notebookServer.sendNewEvent = function(data) { if ($rootScope.ticket !== undefined) { data.principal = $rootScope.ticket.principal; data.ticket = $rootScope.ticket.ticket; @@ -37,20 +38,20 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, data.ticket = ''; data.roles = ''; } - console.log('Send >> %o, %o, %o, %o, %o', data.op, data.principal, data.ticket, data.roles, data); - websocketCalls.ws.send(JSON.stringify(data)); + console.log('Send Notebook Server >> %o, %o, %o, %o, %o', data.op, data.principal, data.ticket, data.roles, data); + websocketCalls.notebookServer.ws.send(JSON.stringify(data)); }; - websocketCalls.isConnected = function() { - return (websocketCalls.ws.socket.readyState === 1); + websocketCalls.notebookServer.isConnected = function() { + return (websocketCalls.notebookServer.ws.socket.readyState === 1); }; - websocketCalls.ws.onMessage(function(event) { + websocketCalls.notebookServer.ws.onMessage(function(event) { var payload; if (event.data) { payload = angular.fromJson(event.data); } - console.log('Receive << %o, %o', payload.op, payload); + console.log('Receive Notebook Server << %o, %o', payload.op, payload); var op = payload.op; var data = payload.data; if (op === 'NOTE') { @@ -59,10 +60,6 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, $location.path('notebook/' + data.note.id); } else if (op === 'NOTES_INFO') { $rootScope.$broadcast('setNoteMenu', data.notes); - } else if (op === 'LIST_NOTEBOOK_JOBS') { - $rootScope.$broadcast('setNotebookJobs', data.notebookJobs); - } else if (op === 'LIST_UPDATE_NOTEBOOK_JOBS') { - $rootScope.$broadcast('setUpdateNotebookJobs', data.notebookRunningJobs); } else if (op === 'AUTH_INFO') { BootstrapDialog.show({ closable: true, @@ -100,14 +97,69 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, } }); - websocketCalls.ws.onError(function(event) { - console.log('error message: ', event); - $rootScope.$broadcast('setConnectedStatus', false); + websocketCalls.notebookServer.ws.onError(function(event) { + console.log('[notebookWS] Notebook Server message: ', event); + $rootScope.$broadcast('setNotebookWSConnectedStatus', false); + }); + + websocketCalls.notebookServer.ws.onClose(function(event) { + console.log('[notebookWS] close message: ', event); + $rootScope.$broadcast('setNotebookWSConnectedStatus', false); + }); + + // create socket for job manager server + websocketCalls.jobManagerServer.ws = $websocket(baseUrlSrv.getJobManagerWebsocketUrl()); + websocketCalls.jobManagerServer.ws.reconnectIfNotNormalClose = true; + + websocketCalls.jobManagerServer.ws.onOpen(function() { + console.log('Websocket Job Manager Server created'); + $rootScope.$broadcast('setJobManagerWSConnectedStatus', true); + setInterval(function(){ + websocketCalls.jobManagerServer.sendNewEvent({op: 'PING'}); + }, 10000); + }); + + websocketCalls.jobManagerServer.sendNewEvent = function(data) { + if ($rootScope.ticket !== undefined) { + data.principal = $rootScope.ticket.principal; + data.ticket = $rootScope.ticket.ticket; + data.roles = $rootScope.ticket.roles; + } else { + data.principal = ''; + data.ticket = ''; + data.roles = ''; + } + console.log('Send Job Manager Server >> %o, %o, %o, %o, %o', data.op, data.principal, data.ticket, data.roles, data); + websocketCalls.jobManagerServer.ws.send(JSON.stringify(data)); + }; + + websocketCalls.jobManagerServer.isConnected = function() { + return (websocketCalls.notebookServer.ws.socket.readyState === 1); + }; + + websocketCalls.jobManagerServer.ws.onMessage(function(event) { + var payload; + if (event.data) { + payload = angular.fromJson(event.data); + } + console.log('Receive Job Manager Server<< %o, %o', payload.op, payload); + var op = payload.op; + var data = payload.data; + if (op === 'LIST_NOTEBOOK_JOBS') { + $rootScope.$broadcast('setNotebookJobs', data.notebookJobs); + } else if (op === 'LIST_UPDATE_NOTEBOOK_JOBS') { + $rootScope.$broadcast('setUpdateNotebookJobs', data.notebookRunningJobs); + } + }); + + websocketCalls.jobManagerServer.ws.onError(function(event) { + console.log('[jobManagerWS] error message: ', event); + $rootScope.$broadcast('setJobManagerWSConnectedStatus', false); }); - websocketCalls.ws.onClose(function(event) { - console.log('close message: ', event); - $rootScope.$broadcast('setConnectedStatus', false); + websocketCalls.jobManagerServer.ws.onClose(function(event) { + console.log('[jobManagerWS] close message: ', event); + $rootScope.$broadcast('setJobManagerWSConnectedStatus', false); }); return websocketCalls; diff --git a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js index 35bffd22188..aa857db3036 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js +++ b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js @@ -18,57 +18,47 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, return { getHomeNotebook: function() { - websocketEvents.sendNewEvent({op: 'GET_HOME_NOTE'}); + websocketEvents.notebookServer.sendNewEvent({op: 'GET_HOME_NOTE'}); }, createNotebook: function(noteName) { - websocketEvents.sendNewEvent({op: 'NEW_NOTE',data: {name: noteName}}); + websocketEvents.notebookServer.sendNewEvent({op: 'NEW_NOTE',data: {name: noteName}}); }, deleteNotebook: function(noteId) { - websocketEvents.sendNewEvent({op: 'DEL_NOTE', data: {id: noteId}}); + websocketEvents.notebookServer.sendNewEvent({op: 'DEL_NOTE', data: {id: noteId}}); }, cloneNotebook: function(noteIdToClone, newNoteName ) { - websocketEvents.sendNewEvent({op: 'CLONE_NOTE', data: {id: noteIdToClone, name: newNoteName}}); + websocketEvents.notebookServer.sendNewEvent({op: 'CLONE_NOTE', data: {id: noteIdToClone, name: newNoteName}}); }, getNotebookList: function() { - websocketEvents.sendNewEvent({op: 'LIST_NOTES'}); - }, - - getNotebookJobsList: function() { - websocketEvents.sendNewEvent({op: 'LIST_NOTEBOOK_JOBS'}); - }, - - getUpdateNotebookJobsList: function(lastUpdateServerUnixTime) { - websocketEvents.sendNewEvent( - {op: 'LIST_UPDATE_NOTEBOOK_JOBS', data : {lastUpdateUnixTime : lastUpdateServerUnixTime*1}} - ); + websocketEvents.notebookServer.sendNewEvent({op: 'LIST_NOTES'}); }, reloadAllNotesFromRepo: function() { - websocketEvents.sendNewEvent({op: 'RELOAD_NOTES_FROM_REPO'}); + websocketEvents.notebookServer.sendNewEvent({op: 'RELOAD_NOTES_FROM_REPO'}); }, getNotebook: function(noteId) { - websocketEvents.sendNewEvent({op: 'GET_NOTE', data: {id: noteId}}); + websocketEvents.notebookServer.sendNewEvent({op: 'GET_NOTE', data: {id: noteId}}); }, updateNotebook: function(noteId, noteName, noteConfig) { - websocketEvents.sendNewEvent({op: 'NOTE_UPDATE', data: {id: noteId, name: noteName, config : noteConfig}}); + websocketEvents.notebookServer.sendNewEvent({op: 'NOTE_UPDATE', data: {id: noteId, name: noteName, config : noteConfig}}); }, moveParagraph: function(paragraphId, newIndex) { - websocketEvents.sendNewEvent({ op: 'MOVE_PARAGRAPH', data : {id: paragraphId, index: newIndex}}); + websocketEvents.notebookServer.sendNewEvent({ op: 'MOVE_PARAGRAPH', data : {id: paragraphId, index: newIndex}}); }, insertParagraph: function(newIndex) { - websocketEvents.sendNewEvent({ op: 'INSERT_PARAGRAPH', data : {index: newIndex}}); + websocketEvents.notebookServer.sendNewEvent({ op: 'INSERT_PARAGRAPH', data : {index: newIndex}}); }, updateAngularObject: function(noteId, paragraphId, name, value, interpreterGroupId) { - websocketEvents.sendNewEvent({ + websocketEvents.notebookServer.sendNewEvent({ op: 'ANGULAR_OBJECT_UPDATED', data: { noteId: noteId, @@ -81,7 +71,7 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, }, clientBindAngularObject: function(noteId, name, value, paragraphId) { - websocketEvents.sendNewEvent({ + websocketEvents.notebookServer.sendNewEvent({ op: 'ANGULAR_OBJECT_CLIENT_BIND', data: { noteId: noteId, @@ -93,7 +83,7 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, }, clientUnbindAngularObject: function(noteId, name, paragraphId) { - websocketEvents.sendNewEvent({ + websocketEvents.notebookServer.sendNewEvent({ op: 'ANGULAR_OBJECT_CLIENT_UNBIND', data: { noteId: noteId, @@ -104,11 +94,11 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, }, cancelParagraphRun: function(paragraphId) { - websocketEvents.sendNewEvent({op: 'CANCEL_PARAGRAPH', data: {id: paragraphId}}); + websocketEvents.notebookServer.sendNewEvent({op: 'CANCEL_PARAGRAPH', data: {id: paragraphId}}); }, runParagraph: function(paragraphId, paragraphTitle, paragraphData, paragraphConfig, paragraphParams) { - websocketEvents.sendNewEvent({ + websocketEvents.notebookServer.sendNewEvent({ op: 'RUN_PARAGRAPH', data: { id: paragraphId, @@ -121,15 +111,15 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, }, removeParagraph: function(paragraphId) { - websocketEvents.sendNewEvent({op: 'PARAGRAPH_REMOVE', data: {id: paragraphId}}); + websocketEvents.notebookServer.sendNewEvent({op: 'PARAGRAPH_REMOVE', data: {id: paragraphId}}); }, clearParagraphOutput: function(paragraphId) { - websocketEvents.sendNewEvent({op: 'PARAGRAPH_CLEAR_OUTPUT', data: {id: paragraphId}}); + websocketEvents.notebookServer.sendNewEvent({op: 'PARAGRAPH_CLEAR_OUTPUT', data: {id: paragraphId}}); }, completion: function(paragraphId, buf, cursor) { - websocketEvents.sendNewEvent({ + websocketEvents.notebookServer.sendNewEvent({ op : 'COMPLETION', data : { id : paragraphId, @@ -140,7 +130,7 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, }, commitParagraph: function(paragraphId, paragraphTitle, paragraphData, paragraphConfig, paragraphParams) { - websocketEvents.sendNewEvent({ + websocketEvents.notebookServer.sendNewEvent({ op: 'COMMIT_PARAGRAPH', data: { id: paragraphId, @@ -153,7 +143,7 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, }, importNotebook: function(notebook) { - websocketEvents.sendNewEvent({ + websocketEvents.notebookServer.sendNewEvent({ op: 'IMPORT_NOTE', data: { notebook: notebook @@ -162,7 +152,7 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, }, checkpointNotebook: function(noteId, commitMessage) { - websocketEvents.sendNewEvent({ + websocketEvents.notebookServer.sendNewEvent({ op: 'CHECKPOINT_NOTEBOOK', data: { noteId: noteId, @@ -172,7 +162,17 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, }, isConnected: function(){ - return websocketEvents.isConnected(); + return websocketEvents.notebookServer.isConnected(); + }, + + getNotebookJobsList: function() { + websocketEvents.jobManagerServer.sendNewEvent({op: 'LIST_NOTEBOOK_JOBS'}); + }, + + getUpdateNotebookJobsList: function(lastUpdateServerUnixTime) { + websocketEvents.jobManagerServer.sendNewEvent( + {op: 'LIST_UPDATE_NOTEBOOK_JOBS', data : {lastUpdateUnixTime : lastUpdateServerUnixTime*1}} + ); } }; From a9490151216a84befc1d3fe2f6ed84683d360d1c Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Wed, 1 Jun 2016 22:47:25 +0900 Subject: [PATCH 15/25] refactor job manager --- .../apache/zeppelin/rest/NotebookRestApi.java | 32 +- .../zeppelin/server/ZeppelinServer.java | 24 +- .../apache/zeppelin/socket/AppMainServer.java | 250 +++ .../zeppelin/socket/JobManagerServer.java | 509 ------ .../zeppelin/socket/JobManagerSocket.java | 71 - .../socket/JobManagerWebSocketCreator.java | 36 - .../zeppelin/socket/JobMangerServer.java | 1439 +++++++++++++++++ .../zeppelin/socket/NotebookServer.java | 188 +-- ...{NotebookSocket.java => WebAppSocket.java} | 8 +- ...tCreator.java => WebAppSocketCreator.java} | 12 +- ...etListener.java => WebSocketListener.java} | 10 +- ...cketListener.java => WebSocketServer.java} | 8 +- ...ServerTest.java => AppMainServerTest.java} | 34 +- .../app/jobmanager/jobmanager.controller.js | 4 +- 14 files changed, 1819 insertions(+), 806 deletions(-) create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/socket/AppMainServer.java delete mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerServer.java delete mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocket.java delete mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerWebSocketCreator.java create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java rename zeppelin-server/src/main/java/org/apache/zeppelin/socket/{NotebookSocket.java => WebAppSocket.java} (90%) rename zeppelin-server/src/main/java/org/apache/zeppelin/socket/{NotebookWebSocketCreator.java => WebAppSocketCreator.java} (75%) rename zeppelin-server/src/main/java/org/apache/zeppelin/socket/{NotebookSocketListener.java => WebSocketListener.java} (78%) rename zeppelin-server/src/main/java/org/apache/zeppelin/socket/{JobManagerSocketListener.java => WebSocketServer.java} (78%) rename zeppelin-server/src/test/java/org/apache/zeppelin/socket/{NotebookServerTest.java => AppMainServerTest.java} (92%) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index f3e63a98b67..66be56aacf9 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -45,7 +45,7 @@ import org.apache.zeppelin.rest.message.RunParagraphWithParametersRequest; import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.server.JsonResponse; -import org.apache.zeppelin.socket.NotebookServer; +import org.apache.zeppelin.socket.AppMainServer; import org.apache.zeppelin.utils.SecurityUtils; import org.quartz.CronExpression; import org.slf4j.Logger; @@ -54,9 +54,7 @@ import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; -import com.google.gson.GsonBuilder; -import com.google.gson.stream.JsonReader; -import java.io.StringReader; + /** * Rest api endpoint for the noteBook. */ @@ -66,15 +64,15 @@ public class NotebookRestApi { private static final Logger LOG = LoggerFactory.getLogger(NotebookRestApi.class); Gson gson = new Gson(); private Notebook notebook; - private NotebookServer notebookServer; + private AppMainServer appMainServer; private SearchService notebookIndex; private NotebookAuthorization notebookAuthorization; public NotebookRestApi() {} - public NotebookRestApi(Notebook notebook, NotebookServer notebookServer, SearchService search) { + public NotebookRestApi(Notebook notebook, AppMainServer appMainServer, SearchService search) { this.notebook = notebook; - this.notebookServer = notebookServer; + this.appMainServer = appMainServer; this.notebookIndex = search; this.notebookAuthorization = notebook.getNotebookAuthorization(); } @@ -157,7 +155,7 @@ public Response putNotePermissions(@PathParam("noteId") String noteId, String re notebookAuthorization.getReaders(noteId), notebookAuthorization.getWriters(noteId)); note.persist(); - notebookServer.broadcastNote(note); + appMainServer.broadcastNote(note); return new JsonResponse<>(Status.OK).build(); } @@ -219,7 +217,7 @@ public Response bind(@PathParam("noteId") String noteId) { @GET @Path("/") public Response getNotebookList() throws IOException { - List> notesInfo = notebookServer.generateNotebooksInfo(false); + List> notesInfo = appMainServer.generateNotebooksInfo(false); return new JsonResponse<>(Status.OK, "", notesInfo ).build(); } @@ -290,8 +288,8 @@ public Response createNote(String message) throws IOException { } note.setName(noteName); note.persist(); - notebookServer.broadcastNote(note); - notebookServer.broadcastNoteList(); + appMainServer.broadcastNote(note); + appMainServer.broadcastNoteList(); return new JsonResponse<>(Status.CREATED, "", note.getId() ).build(); } @@ -311,7 +309,7 @@ public Response deleteNote(@PathParam("notebookId") String notebookId) throws IO notebook.removeNote(notebookId); } } - notebookServer.broadcastNoteList(); + appMainServer.broadcastNoteList(); return new JsonResponse<>(Status.OK, "").build(); } @@ -330,8 +328,8 @@ public Response cloneNote(@PathParam("notebookId") String notebookId, String mes NewNotebookRequest.class); String newNoteName = request.getName(); Note newNote = notebook.cloneNote(notebookId, newNoteName); - notebookServer.broadcastNote(newNote); - notebookServer.broadcastNoteList(); + appMainServer.broadcastNote(newNote); + appMainServer.broadcastNoteList(); return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build(); } @@ -365,7 +363,7 @@ public Response insertParagraph(@PathParam("notebookId") String notebookId, Stri p.setText(request.getText()); note.persist(); - notebookServer.broadcastNote(note); + appMainServer.broadcastNote(note); return new JsonResponse(Status.CREATED, "", p.getId()).build(); } @@ -421,7 +419,7 @@ public Response moveParagraph(@PathParam("notebookId") String notebookId, note.moveParagraph(paragraphId, Integer.parseInt(newIndex), true); note.persist(); - notebookServer.broadcastNote(note); + appMainServer.broadcastNote(note); return new JsonResponse(Status.OK, "").build(); } catch (IndexOutOfBoundsException e) { LOG.error("Exception in NotebookRestApi while moveParagraph ", e); @@ -453,7 +451,7 @@ public Response deleteParagraph(@PathParam("notebookId") String notebookId, note.removeParagraph(paragraphId); note.persist(); - notebookServer.broadcastNote(note); + appMainServer.broadcastNote(note); return new JsonResponse(Status.OK, "").build(); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 8b075592914..9c24bbc1ac7 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -30,8 +30,7 @@ import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.search.LuceneSearch; import org.apache.zeppelin.search.SearchService; -import org.apache.zeppelin.socket.JobManagerServer; -import org.apache.zeppelin.socket.NotebookServer; +import org.apache.zeppelin.socket.AppMainServer; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.server.*; import org.eclipse.jetty.server.handler.ContextHandlerCollection; @@ -61,8 +60,7 @@ public class ZeppelinServer extends Application { public static Notebook notebook; public static Server jettyWebServer; - public static NotebookServer notebookWsServer; - public static JobManagerServer jobManagerWsServer; + public static AppMainServer notebookWsServer; private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; @@ -103,9 +101,6 @@ public static void main(String[] args) throws InterruptedException { // REST api setupRestApiContextHandler(webApp, conf); - // Job Manager server - setupJobManagerServer(webApp, conf); - // Notebook server setupNotebookServer(webApp, conf); @@ -196,7 +191,7 @@ private static Server setupJettyServer(ZeppelinConfiguration conf) { private static void setupNotebookServer(WebAppContext webapp, ZeppelinConfiguration conf) { - notebookWsServer = new NotebookServer(); + notebookWsServer = new AppMainServer(); String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); final ServletHolder servletHolder = new ServletHolder(notebookWsServer); servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); @@ -207,19 +202,6 @@ private static void setupNotebookServer(WebAppContext webapp, webapp.addServlet(servletHolder, "/ws/*"); } - private static void setupJobManagerServer(WebAppContext webapp, - ZeppelinConfiguration conf) { - jobManagerWsServer = new JobManagerServer(); - String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); - final ServletHolder servletHolder = new ServletHolder(jobManagerWsServer); - servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); - - final ServletContextHandler cxfContext = new ServletContextHandler( - ServletContextHandler.SESSIONS); - - webapp.addServlet(servletHolder, "/ws/job/*"); - } - private static SslContextFactory getSslContextFactory(ZeppelinConfiguration conf) { SslContextFactory sslContextFactory = new SslContextFactory(); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/AppMainServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/AppMainServer.java new file mode 100644 index 00000000000..eda45ab416d --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/AppMainServer.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.socket; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; + +import javax.servlet.http.HttpServletRequest; + +import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.notebook.socket.Message; +import org.apache.zeppelin.ticket.TicketContainer; +import org.apache.zeppelin.utils.SecurityUtils; +import org.eclipse.jetty.websocket.servlet.WebSocketServlet; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Zeppelin websocket service. + */ +public class AppMainServer extends WebSocketServlet implements + WebSocketListener, WebSocketServer { + private static final Logger LOG = LoggerFactory.getLogger(AppMainServer.class); + Gson gson = new GsonBuilder() + .setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); + final Map> userWebSocketMap = new HashMap<>(); + final Queue connectedSockets = new ConcurrentLinkedQueue<>(); + + @Override + public void configure(WebSocketServletFactory factory) { + factory.setCreator(new WebAppSocketCreator(this)); + } + + public boolean checkOrigin(HttpServletRequest request, String origin) { + try { + return SecurityUtils.isValidOrigin(origin, ZeppelinConfiguration.create()); + } catch (UnknownHostException e) { + LOG.error(e.toString(), e); + } catch (URISyntaxException e) { + LOG.error(e.toString(), e); + } + return false; + } + + public WebAppSocket doWebSocketConnect(HttpServletRequest req, String protocol) { + return new WebAppSocket(req, protocol, this); + } + + @Override + public void onOpen(WebAppSocket conn) { + LOG.info("New connection from {} : {}", conn.getRequest().getRemoteAddr(), + conn.getRequest().getRemotePort()); + connectedSockets.add(conn); + } + + @Override + public void onMessage(WebAppSocket conn, String msg) { + try { + Message messagereceived = deserializeMessage(msg); + LOG.debug("RECEIVE << " + messagereceived.op); + LOG.debug("RECEIVE PRINCIPAL << " + messagereceived.principal); + LOG.debug("RECEIVE TICKET << " + messagereceived.ticket); + LOG.debug("RECEIVE ROLES << " + messagereceived.roles); + + if (LOG.isTraceEnabled()) { + LOG.trace("RECEIVE MSG = " + messagereceived); + } + + String ticket = TicketContainer.instance.getTicket(messagereceived.principal); + if (ticket != null && !ticket.equals(messagereceived.ticket)) + throw new Exception("Invalid ticket " + messagereceived.ticket + " != " + ticket); + + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + boolean allowAnonymous = conf. + getBoolean(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED); + if (!allowAnonymous && messagereceived.principal.equals("anonymous")) { + throw new Exception("Anonymous access not allowed "); + } + + HashSet userAndRoles = new HashSet(); + userAndRoles.add(messagereceived.principal); + if (!messagereceived.roles.equals("")) { + HashSet roles = gson.fromJson(messagereceived.roles, + new TypeToken>(){}.getType()); + if (roles != null) { + userAndRoles.addAll(roles); + } + } + LOG.info("lcs main server received"); + /** Lets be elegant here */ + switch (messagereceived.op) { + case PING: + break; //do nothing + default: + break; + } + } catch (Exception e) { + LOG.error("Can't handle message", e); + } + } + + @Override + public void onClose(WebAppSocket conn, int code, String reason) { + LOG.info("Closed connection to {} : {}. ({}) {}", conn.getRequest() + .getRemoteAddr(), conn.getRequest().getRemotePort(), code, reason); + removeConnectionFromAllKey(conn); + connectedSockets.remove(conn); + } + + protected Message deserializeMessage(String msg) { + return gson.fromJson(msg, Message.class); + } + + protected String serializeMessage(Message m) { + return gson.toJson(m); + } + + protected void addConnectionToKey(String key, WebAppSocket socket) { + synchronized (userWebSocketMap) { + removeConnectionFromAllKey(socket); // make sure a socket relates only a + // single key. + List socketList = userWebSocketMap.get(key); + if (socketList == null) { + socketList = new LinkedList<>(); + userWebSocketMap.put(key, socketList); + } + if (!socketList.contains(socket)) { + socketList.add(socket); + } + } + } + + protected void removeConnectionFromKey(String key, WebAppSocket socket) { + synchronized (userWebSocketMap) { + List socketList = userWebSocketMap.get(key); + if (socketList != null) { + socketList.remove(socket); + } + } + } + + protected void removeKey(String key) { + synchronized (userWebSocketMap) { + List socketList = userWebSocketMap.remove(key); + } + } + + protected void removeConnectionFromAllKey(WebAppSocket socket) { + synchronized (userWebSocketMap) { + Set keys = userWebSocketMap.keySet(); + for (String keyValue : keys) { + removeConnectionFromKey(keyValue, socket); + } + } + } + + protected String getOpenKey(WebAppSocket socket) { + String key = null; + synchronized (userWebSocketMap) { + Set keys = userWebSocketMap.keySet(); + for (String keyValue : keys) { + List sockets = userWebSocketMap.get(keyValue); + if (sockets.contains(socket)) { + key = keyValue; + } + } + } + + return key; + } + + private void broadcast(String key, Message m) { + synchronized (userWebSocketMap) { + List socketLists = userWebSocketMap.get(key); + if (socketLists == null || socketLists.size() == 0) { + return; + } + LOG.debug("SEND >> " + m.op); + for (WebAppSocket conn : socketLists) { + try { + conn.send(serializeMessage(m)); + } catch (IOException e) { + LOG.error("socket error", e); + } + } + } + } + + private void broadcastExcept(String key, Message m, WebAppSocket exclude) { + synchronized (userWebSocketMap) { + List socketLists = userWebSocketMap.get(key); + if (socketLists == null || socketLists.size() == 0) { + return; + } + LOG.debug("SEND >> " + m.op); + for (WebAppSocket conn : socketLists) { + if (exclude.equals(conn)) { + continue; + } + try { + conn.send(serializeMessage(m)); + } catch (IOException e) { + LOG.error("socket error", e); + } + } + } + } + + private void broadcastAll(Message m) { + for (WebAppSocket conn : connectedSockets) { + try { + conn.send(serializeMessage(m)); + } catch (IOException e) { + LOG.error("socket error", e); + } + } + } + + private void unicast(Message m, WebAppSocket conn) { + try { + conn.send(serializeMessage(m)); + } catch (IOException e) { + LOG.error("socket error", e); + } + } +} + diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerServer.java deleted file mode 100644 index 52c49a0e4da..00000000000 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerServer.java +++ /dev/null @@ -1,509 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.zeppelin.socket; - -import com.google.common.base.Strings; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.reflect.TypeToken; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import org.apache.zeppelin.display.AngularObject; -import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.display.AngularObjectRegistryListener; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterOutput; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterSetting; -import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; -import org.apache.zeppelin.notebook.*; -import org.apache.zeppelin.notebook.socket.Message; -import org.apache.zeppelin.notebook.socket.Message.OP; -import org.apache.zeppelin.scheduler.Job; -import org.apache.zeppelin.scheduler.Job.Status; -import org.apache.zeppelin.server.ZeppelinServer; -import org.apache.zeppelin.ticket.TicketContainer; -import org.apache.zeppelin.user.AuthenticationInfo; -import org.apache.zeppelin.utils.SecurityUtils; -import org.eclipse.jetty.websocket.servlet.WebSocketServlet; -import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; -import org.quartz.SchedulerException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.servlet.http.HttpServletRequest; -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.UnknownHostException; -import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; - -/** - * Zeppelin websocket service. - */ -public class JobManagerServer extends WebSocketServlet implements - JobManagerSocketListener, JobListenerFactory { - private static final Logger LOG = LoggerFactory.getLogger(JobManagerServer.class); - Gson gson = new GsonBuilder() - .setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); - final Map> jobManagerSocketMap = new HashMap<>(); - final Queue connectedSockets = new ConcurrentLinkedQueue<>(); - - private Notebook notebook() { - return ZeppelinServer.notebook; - } - - @Override - public void configure(WebSocketServletFactory factory) { - factory.setCreator(new JobManagerWebSocketCreator(this)); - } - - public boolean checkOrigin(HttpServletRequest request, String origin) { - try { - return SecurityUtils.isValidOrigin(origin, ZeppelinConfiguration.create()); - } catch (UnknownHostException e) { - LOG.error(e.toString(), e); - } catch (URISyntaxException e) { - LOG.error(e.toString(), e); - } - return false; - } - - public JobManagerSocket doWebSocketConnect(HttpServletRequest req, String protocol) { - return new JobManagerSocket(req, protocol, this); - } - - @Override - public void onOpen(JobManagerSocket conn) { - LOG.info("New connection from {} : {}", conn.getRequest().getRemoteAddr(), - conn.getRequest().getRemotePort()); - connectedSockets.add(conn); - } - - @Override - public void onMessage(JobManagerSocket conn, String msg) { - Notebook notebook = notebook(); - try { - Message messagereceived = deserializeMessage(msg); - LOG.debug("RECEIVE << " + messagereceived.op); - LOG.debug("RECEIVE PRINCIPAL << " + messagereceived.principal); - LOG.debug("RECEIVE TICKET << " + messagereceived.ticket); - LOG.debug("RECEIVE ROLES << " + messagereceived.roles); - - if (LOG.isTraceEnabled()) { - LOG.trace("RECEIVE MSG = " + messagereceived); - } - - String ticket = TicketContainer.instance.getTicket(messagereceived.principal); - if (ticket != null && !ticket.equals(messagereceived.ticket)) - throw new Exception("Invalid ticket " + messagereceived.ticket + " != " + ticket); - - ZeppelinConfiguration conf = ZeppelinConfiguration.create(); - boolean allowAnonymous = conf. - getBoolean(ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED); - if (!allowAnonymous && messagereceived.principal.equals("anonymous")) { - throw new Exception("Anonymous access not allowed "); - } - - HashSet userAndRoles = new HashSet(); - userAndRoles.add(messagereceived.principal); - if (!messagereceived.roles.equals("")) { - HashSet roles = gson.fromJson(messagereceived.roles, - new TypeToken>(){}.getType()); - if (roles != null) { - userAndRoles.addAll(roles); - } - } - - LOG.info("lcs job received"); - /** Lets be elegant here */ - switch (messagereceived.op) { - case LIST_NOTEBOOK_JOBS: - unicastNotebookJobInfo(conn); - break; - case LIST_UPDATE_NOTEBOOK_JOBS: - unicastUpdateNotebookJobInfo(conn, messagereceived); - break; - case PING: - break; //do nothing - default: - break; - } - } catch (Exception e) { - LOG.error("Can't handle message", e); - } - } - - @Override - public void onClose(JobManagerSocket conn, int code, String reason) { - LOG.info("Closed connection to {} : {}. ({}) {}", conn.getRequest() - .getRemoteAddr(), conn.getRequest().getRemotePort(), code, reason); - connectedSockets.remove(conn); - } - - protected Message deserializeMessage(String msg) { - return gson.fromJson(msg, Message.class); - } - - protected String serializeMessage(Message m) { - return gson.toJson(m); - } - - private void addConnectionToJobManager(String jobPart, JobManagerSocket socket) { - synchronized (jobManagerSocketMap) { - // single note. - List socketList = jobManagerSocketMap.get(jobPart); - if (socketList == null) { - socketList = new LinkedList<>(); - jobManagerSocketMap.put(jobPart, socketList); - } - if (!socketList.contains(socket)) { - socketList.add(socket); - } - } - } - - private void removeConnectionToJobManager(String jobPart, JobManagerSocket socket) { - synchronized (jobManagerSocketMap) { - List socketList = jobManagerSocketMap.get(jobPart); - if (socketList != null) { - socketList.remove(socket); - } - } - } - - private void broadcast(String noteId, Message m) { - synchronized (jobManagerSocketMap) { - List socketLists = jobManagerSocketMap.get(noteId); - if (socketLists == null || socketLists.size() == 0) { - return; - } - LOG.debug("SEND >> " + m.op); - for (JobManagerSocket conn : socketLists) { - try { - conn.send(serializeMessage(m)); - } catch (IOException e) { - LOG.error("socket error", e); - } - } - } - } - - private void broadcastExcept(String noteId, Message m, JobManagerSocket exclude) { - synchronized (jobManagerSocketMap) { - List socketLists = jobManagerSocketMap.get(noteId); - if (socketLists == null || socketLists.size() == 0) { - return; - } - LOG.debug("SEND >> " + m.op); - for (JobManagerSocket conn : socketLists) { - if (exclude.equals(conn)) { - continue; - } - try { - conn.send(serializeMessage(m)); - } catch (IOException e) { - LOG.error("socket error", e); - } - } - } - } - - private void broadcastAll(Message m) { - for (JobManagerSocket conn : connectedSockets) { - try { - conn.send(serializeMessage(m)); - } catch (IOException e) { - LOG.error("socket error", e); - } - } - } - - private void unicast(Message m, JobManagerSocket conn) { - try { - conn.send(serializeMessage(m)); - } catch (IOException e) { - LOG.error("socket error", e); - } - } - - public void unicastNotebookJobInfo(JobManagerSocket conn) { - List> notebookJobs = generateNotebooksJobInfo(false); - Map response = new HashMap<>(); - - response.put("lastResponseUnixTime", System.currentTimeMillis()); - response.put("jobs", notebookJobs); - - unicast(new Message(OP.LIST_NOTEBOOK_JOBS).put("notebookJobs", response), conn); - } - - public void unicastUpdateNotebookJobInfo(JobManagerSocket conn, Message fromMessage) { - double lastUpdateUnixTimeRaw = (double) fromMessage.get("lastUpdateUnixTime"); - long lastUpdateUnixTime = new Double(lastUpdateUnixTimeRaw).longValue(); - List> notebookJobs; - notebookJobs = generateUpdateNotebooksJobInfo(false, lastUpdateUnixTime); - Map response = new HashMap<>(); - - response.put("lastResponseUnixTime", System.currentTimeMillis()); - response.put("jobs", notebookJobs); - - unicast(new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS) - .put("notebookRunningJobs", response), conn); - } - - - public List> generateNotebooksJobInfo(boolean needsReload) { - Notebook notebook = notebook(); - - ZeppelinConfiguration conf = notebook.getConf(); - String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); - boolean hideHomeScreenNotebookFromList = conf - .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); - - if (needsReload) { - try { - notebook.reloadAllNotes(); - } catch (IOException e) { - LOG.error("Fail to reload notes from repository"); - } - } - - List notes = notebook.getAllNotes(); - List> notesInfo = new LinkedList<>(); - for (Note note : notes) { - boolean isNotebookRunning = false; - Map info = new HashMap<>(); - - if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { - continue; - } - - String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; - info.put("notebookId", note.id()); - String notebookName = note.getName(); - if (notebookName != null) { - info.put("notebookName", note.getName()); - } else { - info.put("notebookName", note.id()); - } - - if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true - && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { - info.put("notebookType", "cron"); - } - else { - info.put("notebookType", "normal"); - } - - Date lastRunningDate = null; - long lastRunningUnixTime = 0; - - List> paragraphsInfo = new LinkedList<>(); - for (Paragraph paragraph : note.getParagraphs()) { - if (paragraph.getStatus().isRunning() == true) { - isNotebookRunning = true; - } - Map paragraphItem = new HashMap<>(); - // set paragraph id - paragraphItem.put("id", paragraph.getId()); - - // set paragraph name - String paragraphName = paragraph.getTitle(); - if (paragraphName != null) { - paragraphItem.put("name", paragraphName); - } else { - paragraphItem.put("name", paragraph.getId()); - } - - // set status for paragraph. - paragraphItem.put("status", paragraph.getStatus().toString()); - - // get last update time. - Date paragaraphDate = paragraph.getDateStarted(); - if (paragaraphDate == null) { - paragaraphDate = paragraph.getDateCreated(); - } - if (lastRunningDate == null) { - lastRunningDate = paragaraphDate; - } else { - if (lastRunningDate.after(paragaraphDate) == true) { - lastRunningDate = paragaraphDate; - } - } - - // convert date to unixtime(ms). - lastRunningUnixTime = lastRunningDate.getTime(); - - paragraphsInfo.add(paragraphItem); - } - - // Interpreter is set does not exist. - String interpreterGroupName = null; - if (note.getNoteReplLoader().getInterpreterSettings() != null - && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { - interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); - } - - // notebook json object root information. - info.put("interpreter", interpreterGroupName); - info.put("isRunningJob", isNotebookRunning); - info.put("unixTimeLastRun", lastRunningUnixTime); - info.put("paragraphs", paragraphsInfo); - notesInfo.add(info); - } - return notesInfo; - } - - - public List> generateUpdateNotebooksJobInfo( - boolean needsReload, long lastUpdateServerUnixTime) { - Notebook notebook = notebook(); - - ZeppelinConfiguration conf = notebook.getConf(); - String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); - boolean hideHomeScreenNotebookFromList = conf - .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); - - if (needsReload) { - try { - notebook.reloadAllNotes(); - } catch (IOException e) { - LOG.error("Fail to reload notes from repository"); - } - } - - List notes = notebook.getAllNotes(); - List> notesInfo = new LinkedList<>(); - for (Note note : notes) { - boolean isNotebookRunning = false; - boolean isUpdateNotebook = false; - - Map info = new HashMap<>(); - - if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { - continue; - } - - // set const keyword for cron type - String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; - info.put("notebookId", note.id()); - String notebookName = note.getName(); - if (notebookName != null) { - info.put("notebookName", note.getName()); - } else { - info.put("notebookName", note.id()); - } - - - if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true - && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { - info.put("notebookType", "cron"); - } - else { - info.put("notebookType", "normal"); - } - - Date lastRunningDate = null; - long lastRunningUnixTime = 0; - - List> paragraphsInfo = new LinkedList<>(); - for (Paragraph paragraph : note.getParagraphs()) { - - // check date for update time. - Date startedDate = paragraph.getDateStarted(); - Date createdDate = paragraph.getDateCreated(); - Date finishedDate = paragraph.getDateFinished(); - - if (startedDate != null && startedDate.getTime() > lastUpdateServerUnixTime) { - isUpdateNotebook = true; - } - if (createdDate != null && createdDate.getTime() > lastUpdateServerUnixTime) { - isUpdateNotebook = true; - } - if (finishedDate != null && finishedDate.getTime() > lastUpdateServerUnixTime) { - isUpdateNotebook = true; - } - - Map paragraphItem = new HashMap<>(); - - // set paragraph id - paragraphItem.put("id", paragraph.getId()); - - // set paragraph name - String paragraphName = paragraph.getTitle(); - if (paragraphName != null) { - paragraphItem.put("name", paragraphName); - } else { - paragraphItem.put("name", paragraph.getId()); - } - - // set status for paragraph - paragraphItem.put("status", paragraph.getStatus().toString()); - - Date paragaraphDate = startedDate; - if (paragaraphDate == null) { - paragaraphDate = createdDate; - } - - // set last update unixtime(ms). - if (lastRunningDate == null) { - lastRunningDate = paragaraphDate; - } else { - if (lastRunningDate.after(paragaraphDate) == true) { - lastRunningDate = paragaraphDate; - } - } - lastRunningUnixTime = lastRunningDate.getTime(); - if (paragraph.getStatus().isRunning() == true) { - isNotebookRunning = true; - isUpdateNotebook = true; - } - paragraphsInfo.add(paragraphItem); - } - - // Insert only data that has changed. - if (isUpdateNotebook != true) { - continue; - } - - // Interpreter is set does not exist. - String interpreterGroupName = null; - if (note.getNoteReplLoader().getInterpreterSettings() != null - && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { - interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); - } - - // set notebook root information. - info.put("interpreter", interpreterGroupName); - info.put("isRunningJob", isNotebookRunning); - info.put("unixTimeLastRun", lastRunningUnixTime); - info.put("paragraphs", paragraphsInfo); - notesInfo.add(info); - } - if (notesInfo.size() > 0) { - LOG.info("update count {}", notesInfo.size()); - } - return notesInfo; - } - - @Override - public ParagraphJobListener getParagraphJobListener(Note note) { - return null; - } -} - diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocket.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocket.java deleted file mode 100644 index 35201897e9b..00000000000 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocket.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.zeppelin.socket; - -import org.eclipse.jetty.websocket.api.Session; -import org.eclipse.jetty.websocket.api.WebSocketAdapter; - -import javax.servlet.http.HttpServletRequest; -import java.io.IOException; - -/** - * Notebook websocket - */ -public class JobManagerSocket extends WebSocketAdapter { - - private Session connection; - private JobManagerSocketListener listener; - private HttpServletRequest request; - private String protocol; - - public JobManagerSocket(HttpServletRequest req, String protocol, - JobManagerSocketListener listener) { - this.listener = listener; - this.request = req; - this.protocol = protocol; - } - - @Override - public void onWebSocketClose(int closeCode, String message) { - listener.onClose(this, closeCode, message); - } - - @Override - public void onWebSocketConnect(Session connection) { - this.connection = connection; - listener.onOpen(this); - } - - @Override - public void onWebSocketText(String message) { - listener.onMessage(this, message); - } - - - public HttpServletRequest getRequest() { - return request; - } - - public String getProtocol() { - return protocol; - } - - public void send(String serializeMessage) throws IOException { - connection.getRemote().sendString(serializeMessage); - } - -} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerWebSocketCreator.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerWebSocketCreator.java deleted file mode 100644 index d14a7ea49fa..00000000000 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerWebSocketCreator.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.zeppelin.socket; - -import org.eclipse.jetty.websocket.servlet.ServletUpgradeRequest; -import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse; -import org.eclipse.jetty.websocket.servlet.WebSocketCreator; - -/** - * Responsible to create the WebSockets for the NotebookServer. - */ -public class JobManagerWebSocketCreator implements WebSocketCreator { - private JobManagerServer jobManagerServer; - - public JobManagerWebSocketCreator(JobManagerServer jobManagerServer) { - this.jobManagerServer = jobManagerServer; - } - public Object createWebSocket(ServletUpgradeRequest request, ServletUpgradeResponse response) { - return new JobManagerSocket(request.getHttpServletRequest(), "", jobManagerServer); - } - -} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java new file mode 100644 index 00000000000..015946dde4d --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java @@ -0,0 +1,1439 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.socket; + +import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import org.apache.zeppelin.notebook.*; +import org.apache.zeppelin.notebook.socket.Message; +import org.apache.zeppelin.notebook.socket.Message.OP; +import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.server.ZeppelinServer; +import org.apache.zeppelin.ticket.TicketContainer; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.apache.zeppelin.utils.SecurityUtils; +import org.eclipse.jetty.websocket.servlet.WebSocketServlet; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.quartz.SchedulerException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * Zeppelin websocket service. + */ +public class JobMangerServer extends AppMainServer implements WebSocketServer { + private static final Logger LOG = LoggerFactory.getLogger(JobMangerServer.class); + Gson gson = new GsonBuilder() + .setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); + final Map> noteSocketMap = new HashMap<>(); + final Queue connectedSockets = new ConcurrentLinkedQueue<>(); + + private Notebook notebook() { + return ZeppelinServer.notebook; + } + + @Override + public void configure(WebSocketServletFactory factory) { + factory.setCreator(new WebAppSocketCreator(this)); + } + + public boolean checkOrigin(HttpServletRequest request, String origin) { + try { + return SecurityUtils.isValidOrigin(origin, ZeppelinConfiguration.create()); + } catch (UnknownHostException e) { + LOG.error(e.toString(), e); + } catch (URISyntaxException e) { + LOG.error(e.toString(), e); + } + return false; + } + + public WebAppSocket doWebSocketConnect(HttpServletRequest req, String protocol) { + return new WebAppSocket(req, protocol, this); + } + + @Override + public void onOpen(WebAppSocket conn) { + LOG.info("New connection from {} : {}", conn.getRequest().getRemoteAddr(), + conn.getRequest().getRemotePort()); + connectedSockets.add(conn); + } + + @Override + public void onMessage(WebAppSocket conn, String msg) { + Notebook notebook = notebook(); + try { + Message messagereceived = deserializeMessage(msg); + LOG.debug("RECEIVE << " + messagereceived.op); + LOG.debug("RECEIVE PRINCIPAL << " + messagereceived.principal); + LOG.debug("RECEIVE TICKET << " + messagereceived.ticket); + LOG.debug("RECEIVE ROLES << " + messagereceived.roles); + + if (LOG.isTraceEnabled()) { + LOG.trace("RECEIVE MSG = " + messagereceived); + } + + String ticket = TicketContainer.instance.getTicket(messagereceived.principal); + if (ticket != null && !ticket.equals(messagereceived.ticket)) + throw new Exception("Invalid ticket " + messagereceived.ticket + " != " + ticket); + + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + boolean allowAnonymous = conf. + getBoolean(ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED); + if (!allowAnonymous && messagereceived.principal.equals("anonymous")) { + throw new Exception("Anonymous access not allowed "); + } + + HashSet userAndRoles = new HashSet(); + userAndRoles.add(messagereceived.principal); + if (!messagereceived.roles.equals("")) { + HashSet roles = gson.fromJson(messagereceived.roles, + new TypeToken>(){}.getType()); + if (roles != null) { + userAndRoles.addAll(roles); + } + } + LOG.info("lcs notebook received"); + /** Lets be elegant here */ + switch (messagereceived.op) { + case LIST_NOTEBOOK_JOBS: + unicastNotebookJobInfo(conn); + break; + case LIST_UPDATE_NOTEBOOK_JOBS: + unicastUpdateNotebookJobInfo(conn, messagereceived); + break; + default: + break; + } + } catch (Exception e) { + LOG.error("Can't handle message", e); + } + } + + @Override + public void onClose(WebAppSocket conn, int code, String reason) { + LOG.info("Closed connection to {} : {}. ({}) {}", conn.getRequest() + .getRemoteAddr(), conn.getRequest().getRemotePort(), code, reason); + removeConnectionFromAllNote(conn); + connectedSockets.remove(conn); + } + + protected Message deserializeMessage(String msg) { + return gson.fromJson(msg, Message.class); + } + + protected String serializeMessage(Message m) { + return gson.toJson(m); + } + + + public void unicastNotebookJobInfo(WebAppSocket conn) { + List> notebookJobs = generateNotebooksJobInfo(false); + Map response = new HashMap<>(); + + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", notebookJobs); + + unicast(new Message(OP.LIST_NOTEBOOK_JOBS).put("notebookJobs", response), conn); + } + + public void unicastUpdateNotebookJobInfo(WebAppSocket conn, Message fromMessage) { + double lastUpdateUnixTimeRaw = (double) fromMessage.get("lastUpdateUnixTime"); + long lastUpdateUnixTime = new Double(lastUpdateUnixTimeRaw).longValue(); + List> notebookJobs; + notebookJobs = generateUpdateNotebooksJobInfo(false, lastUpdateUnixTime); + Map response = new HashMap<>(); + + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", notebookJobs); + + unicast(new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS) + .put("notebookRunningJobs", response), conn); + } + + + public List> generateNotebooksJobInfo(boolean needsReload) { + Notebook notebook = notebook(); + + ZeppelinConfiguration conf = notebook.getConf(); + String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); + boolean hideHomeScreenNotebookFromList = conf + .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); + + if (needsReload) { + try { + notebook.reloadAllNotes(); + } catch (IOException e) { + LOG.error("Fail to reload notes from repository"); + } + } + + List notes = notebook.getAllNotes(); + List> notesInfo = new LinkedList<>(); + for (Note note : notes) { + boolean isNotebookRunning = false; + Map info = new HashMap<>(); + + if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { + continue; + } + + String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; + info.put("notebookId", note.id()); + String notebookName = note.getName(); + if (notebookName != null) { + info.put("notebookName", note.getName()); + } else { + info.put("notebookName", note.id()); + } + + if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true + && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { + info.put("notebookType", "cron"); + } + else { + info.put("notebookType", "normal"); + } + + Date lastRunningDate = null; + long lastRunningUnixTime = 0; + + List> paragraphsInfo = new LinkedList<>(); + for (Paragraph paragraph : note.getParagraphs()) { + if (paragraph.getStatus().isRunning() == true) { + isNotebookRunning = true; + } + Map paragraphItem = new HashMap<>(); + // set paragraph id + paragraphItem.put("id", paragraph.getId()); + + // set paragraph name + String paragraphName = paragraph.getTitle(); + if (paragraphName != null) { + paragraphItem.put("name", paragraphName); + } else { + paragraphItem.put("name", paragraph.getId()); + } + + // set status for paragraph. + paragraphItem.put("status", paragraph.getStatus().toString()); + + // get last update time. + Date paragaraphDate = paragraph.getDateStarted(); + if (paragaraphDate == null) { + paragaraphDate = paragraph.getDateCreated(); + } + if (lastRunningDate == null) { + lastRunningDate = paragaraphDate; + } else { + if (lastRunningDate.after(paragaraphDate) == true) { + lastRunningDate = paragaraphDate; + } + } + + // convert date to unixtime(ms). + lastRunningUnixTime = lastRunningDate.getTime(); + + paragraphsInfo.add(paragraphItem); + } + + // Interpreter is set does not exist. + String interpreterGroupName = null; + if (note.getNoteReplLoader().getInterpreterSettings() != null + && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { + interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); + } + + // notebook json object root information. + info.put("interpreter", interpreterGroupName); + info.put("isRunningJob", isNotebookRunning); + info.put("unixTimeLastRun", lastRunningUnixTime); + info.put("paragraphs", paragraphsInfo); + notesInfo.add(info); + } + return notesInfo; + } + + + public List> generateUpdateNotebooksJobInfo( + boolean needsReload, long lastUpdateServerUnixTime) { + Notebook notebook = notebook(); + + ZeppelinConfiguration conf = notebook.getConf(); + String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); + boolean hideHomeScreenNotebookFromList = conf + .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); + + if (needsReload) { + try { + notebook.reloadAllNotes(); + } catch (IOException e) { + LOG.error("Fail to reload notes from repository"); + } + } + + List notes = notebook.getAllNotes(); + List> notesInfo = new LinkedList<>(); + for (Note note : notes) { + boolean isNotebookRunning = false; + boolean isUpdateNotebook = false; + + Map info = new HashMap<>(); + + if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { + continue; + } + + // set const keyword for cron type + String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; + info.put("notebookId", note.id()); + String notebookName = note.getName(); + if (notebookName != null) { + info.put("notebookName", note.getName()); + } else { + info.put("notebookName", note.id()); + } + + + if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true + && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { + info.put("notebookType", "cron"); + } + else { + info.put("notebookType", "normal"); + } + + Date lastRunningDate = null; + long lastRunningUnixTime = 0; + + List> paragraphsInfo = new LinkedList<>(); + for (Paragraph paragraph : note.getParagraphs()) { + + // check date for update time. + Date startedDate = paragraph.getDateStarted(); + Date createdDate = paragraph.getDateCreated(); + Date finishedDate = paragraph.getDateFinished(); + + if (startedDate != null && startedDate.getTime() > lastUpdateServerUnixTime) { + isUpdateNotebook = true; + } + if (createdDate != null && createdDate.getTime() > lastUpdateServerUnixTime) { + isUpdateNotebook = true; + } + if (finishedDate != null && finishedDate.getTime() > lastUpdateServerUnixTime) { + isUpdateNotebook = true; + } + + Map paragraphItem = new HashMap<>(); + + // set paragraph id + paragraphItem.put("id", paragraph.getId()); + + // set paragraph name + String paragraphName = paragraph.getTitle(); + if (paragraphName != null) { + paragraphItem.put("name", paragraphName); + } else { + paragraphItem.put("name", paragraph.getId()); + } + + // set status for paragraph + paragraphItem.put("status", paragraph.getStatus().toString()); + + Date paragaraphDate = startedDate; + if (paragaraphDate == null) { + paragaraphDate = createdDate; + } + + // set last update unixtime(ms). + if (lastRunningDate == null) { + lastRunningDate = paragaraphDate; + } else { + if (lastRunningDate.after(paragaraphDate) == true) { + lastRunningDate = paragaraphDate; + } + } + lastRunningUnixTime = lastRunningDate.getTime(); + if (paragraph.getStatus().isRunning() == true) { + isNotebookRunning = true; + isUpdateNotebook = true; + } + paragraphsInfo.add(paragraphItem); + } + + // Insert only data that has changed. + if (isUpdateNotebook != true) { + continue; + } + + // Interpreter is set does not exist. + String interpreterGroupName = null; + if (note.getNoteReplLoader().getInterpreterSettings() != null + && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { + interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); + } + + // set notebook root information. + info.put("interpreter", interpreterGroupName); + info.put("isRunningJob", isNotebookRunning); + info.put("unixTimeLastRun", lastRunningUnixTime); + info.put("paragraphs", paragraphsInfo); + notesInfo.add(info); + } + if (notesInfo.size() > 0) { + LOG.info("update count {}", notesInfo.size()); + } + return notesInfo; + } + + private void addConnectionToNote(String noteId, WebAppSocket socket) { + synchronized (noteSocketMap) { + removeConnectionFromAllNote(socket); // make sure a socket relates only a + // single note. + List socketList = noteSocketMap.get(noteId); + if (socketList == null) { + socketList = new LinkedList<>(); + noteSocketMap.put(noteId, socketList); + } + if (!socketList.contains(socket)) { + socketList.add(socket); + } + } + } + + private void removeConnectionFromNote(String noteId, WebAppSocket socket) { + synchronized (noteSocketMap) { + List socketList = noteSocketMap.get(noteId); + if (socketList != null) { + socketList.remove(socket); + } + } + } + + private void removeNote(String noteId) { + synchronized (noteSocketMap) { + List socketList = noteSocketMap.remove(noteId); + } + } + + private void removeConnectionFromAllNote(WebAppSocket socket) { + synchronized (noteSocketMap) { + Set keys = noteSocketMap.keySet(); + for (String noteId : keys) { + removeConnectionFromNote(noteId, socket); + } + } + } + + private String getOpenNoteId(WebAppSocket socket) { + String id = null; + synchronized (noteSocketMap) { + Set keys = noteSocketMap.keySet(); + for (String noteId : keys) { + List sockets = noteSocketMap.get(noteId); + if (sockets.contains(socket)) { + id = noteId; + } + } + } + + return id; + } + + private void broadcastToNoteBindedInterpreter(String interpreterGroupId, + Message m) { + Notebook notebook = notebook(); + List notes = notebook.getAllNotes(); + for (Note note : notes) { + List ids = note.getNoteReplLoader().getInterpreters(); + for (String id : ids) { + if (id.equals(interpreterGroupId)) { + broadcast(note.id(), m); + } + } + } + } + + private void broadcast(String noteId, Message m) { + synchronized (noteSocketMap) { + List socketLists = noteSocketMap.get(noteId); + if (socketLists == null || socketLists.size() == 0) { + return; + } + LOG.debug("SEND >> " + m.op); + for (WebAppSocket conn : socketLists) { + try { + conn.send(serializeMessage(m)); + } catch (IOException e) { + LOG.error("socket error", e); + } + } + } + } + + private void broadcastExcept(String noteId, Message m, WebAppSocket exclude) { + synchronized (noteSocketMap) { + List socketLists = noteSocketMap.get(noteId); + if (socketLists == null || socketLists.size() == 0) { + return; + } + LOG.debug("SEND >> " + m.op); + for (WebAppSocket conn : socketLists) { + if (exclude.equals(conn)) { + continue; + } + try { + conn.send(serializeMessage(m)); + } catch (IOException e) { + LOG.error("socket error", e); + } + } + } + } + + private void broadcastAll(Message m) { + for (WebAppSocket conn : connectedSockets) { + try { + conn.send(serializeMessage(m)); + } catch (IOException e) { + LOG.error("socket error", e); + } + } + } + + private void unicast(Message m, WebAppSocket conn) { + try { + conn.send(serializeMessage(m)); + } catch (IOException e) { + LOG.error("socket error", e); + } + } + + + public List> generateNotebooksInfo(boolean needsReload) { + Notebook notebook = notebook(); + + ZeppelinConfiguration conf = notebook.getConf(); + String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); + boolean hideHomeScreenNotebookFromList = conf + .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); + + if (needsReload) { + try { + notebook.reloadAllNotes(); + } catch (IOException e) { + LOG.error("Fail to reload notes from repository", e); + } + } + + List notes = notebook.getAllNotes(); + List> notesInfo = new LinkedList<>(); + for (Note note : notes) { + Map info = new HashMap<>(); + + if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { + continue; + } + + info.put("id", note.id()); + info.put("name", note.getName()); + notesInfo.add(info); + } + + return notesInfo; + } + + public void broadcastNote(Note note) { + broadcast(note.id(), new Message(OP.NOTE).put("note", note)); + } + + public void broadcastNoteList() { + List> notesInfo = generateNotebooksInfo(false); + broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo)); + } + + public void unicastNoteList(WebAppSocket conn) { + List> notesInfo = generateNotebooksInfo(false); + unicast(new Message(OP.NOTES_INFO).put("notes", notesInfo), conn); + } + + public void broadcastReloadedNoteList() { + List> notesInfo = generateNotebooksInfo(true); + broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo)); + } + + void permissionError(WebAppSocket conn, String op, Set userAndRoles, + Set allowed) throws IOException { + LOG.info("Cannot {}. Connection readers {}. Allowed readers {}", + op, userAndRoles, allowed); + + String userName = userAndRoles.iterator().next(); + + conn.send(serializeMessage(new Message(OP.AUTH_INFO).put("info", + "Insufficient privileges to " + op + " notebook.\n\n" + + "Allowed users or roles: " + allowed.toString() + "\n\n" + + "But the user " + userName + " belongs to: " + userAndRoles.toString()))); + } + + private void sendNote(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, + Message fromMessage) throws IOException { + + LOG.info("New operation from {} : {} : {} : {} : {}", conn.getRequest().getRemoteAddr(), + conn.getRequest().getRemotePort(), + fromMessage.principal, fromMessage.op, fromMessage.get("id") + ); + + String noteId = (String) fromMessage.get("id"); + if (noteId == null) { + return; + } + + Note note = notebook.getNote(noteId); + NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); + if (note != null) { + if (!notebookAuthorization.isReader(noteId, userAndRoles)) { + permissionError(conn, "read", userAndRoles, notebookAuthorization.getReaders(noteId)); + return; + } + addConnectionToNote(note.id(), conn); + conn.send(serializeMessage(new Message(OP.NOTE).put("note", note))); + sendAllAngularObjects(note, conn); + } + } + + private void sendHomeNote(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook) throws IOException { + String noteId = notebook.getConf().getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); + + Note note = null; + if (noteId != null) { + note = notebook.getNote(noteId); + } + + if (note != null) { + NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); + if (!notebookAuthorization.isReader(noteId, userAndRoles)) { + permissionError(conn, "read", userAndRoles, notebookAuthorization.getReaders(noteId)); + return; + } + addConnectionToNote(note.id(), conn); + conn.send(serializeMessage(new Message(OP.NOTE).put("note", note))); + sendAllAngularObjects(note, conn); + } else { + removeConnectionFromAllNote(conn); + conn.send(serializeMessage(new Message(OP.NOTE).put("note", null))); + } + } + + private void updateNote(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook, Message fromMessage) + throws SchedulerException, IOException { + String noteId = (String) fromMessage.get("id"); + String name = (String) fromMessage.get("name"); + Map config = (Map) fromMessage + .get("config"); + if (noteId == null) { + return; + } + if (config == null) { + return; + } + + NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); + if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { + permissionError(conn, "update", userAndRoles, notebookAuthorization.getWriters(noteId)); + return; + } + + Note note = notebook.getNote(noteId); + if (note != null) { + boolean cronUpdated = isCronUpdated(config, note.getConfig()); + note.setName(name); + note.setConfig(config); + if (cronUpdated) { + notebook.refreshCron(note.id()); + } + + note.persist(); + broadcastNote(note); + broadcastNoteList(); + } + } + + private boolean isCronUpdated(Map configA, + Map configB) { + boolean cronUpdated = false; + if (configA.get("cron") != null && configB.get("cron") != null + && configA.get("cron").equals(configB.get("cron"))) { + cronUpdated = true; + } else if (configA.get("cron") == null && configB.get("cron") == null) { + cronUpdated = false; + } else if (configA.get("cron") != null || configB.get("cron") != null) { + cronUpdated = true; + } + + return cronUpdated; + } + private void createNote(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook, Message message) + throws IOException { + Note note = notebook.createNote(); + note.addParagraph(); // it's an empty note. so add one paragraph + if (message != null) { + String noteName = (String) message.get("name"); + if (noteName == null || noteName.isEmpty()){ + noteName = "Note " + note.getId(); + } + note.setName(noteName); + } + + note.persist(); + addConnectionToNote(note.id(), (WebAppSocket) conn); + conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note))); + broadcastNoteList(); + } + + private void removeNote(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook, Message fromMessage) + throws IOException { + String noteId = (String) fromMessage.get("id"); + if (noteId == null) { + return; + } + + Note note = notebook.getNote(noteId); + NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); + if (!notebookAuthorization.isOwner(noteId, userAndRoles)) { + permissionError(conn, "remove", userAndRoles, notebookAuthorization.getOwners(noteId)); + return; + } + + notebook.removeNote(noteId); + removeNote(noteId); + broadcastNoteList(); + } + + private void updateParagraph(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook, Message fromMessage) throws IOException { + String paragraphId = (String) fromMessage.get("id"); + if (paragraphId == null) { + return; + } + + Map params = (Map) fromMessage + .get("params"); + Map config = (Map) fromMessage + .get("config"); + String noteId = getOpenNoteId(conn); + final Note note = notebook.getNote(noteId); + NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); + if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { + permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); + return; + } + + Paragraph p = note.getParagraph(paragraphId); + p.settings.setParams(params); + p.setConfig(config); + p.setTitle((String) fromMessage.get("title")); + p.setText((String) fromMessage.get("paragraph")); + note.persist(); + broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p)); + } + + private void cloneNote(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook, Message fromMessage) + throws IOException, CloneNotSupportedException { + String noteId = getOpenNoteId(conn); + String name = (String) fromMessage.get("name"); + Note newNote = notebook.cloneNote(noteId, name); + addConnectionToNote(newNote.id(), (WebAppSocket) conn); + conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", newNote))); + broadcastNoteList(); + } + + protected Note importNote(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook, Message fromMessage) + throws IOException { + Note note = null; + if (fromMessage != null) { + String noteName = (String) ((Map) fromMessage.get("notebook")).get("name"); + String noteJson = gson.toJson(fromMessage.get("notebook")); + note = notebook.importNote(noteJson, noteName); + note.persist(); + broadcastNote(note); + broadcastNoteList(); + } + return note; + } + + private void removeParagraph(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook, Message fromMessage) throws IOException { + final String paragraphId = (String) fromMessage.get("id"); + if (paragraphId == null) { + return; + } + String noteId = getOpenNoteId(conn); + final Note note = notebook.getNote(noteId); + NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); + if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { + permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); + return; + } + + /** We dont want to remove the last paragraph */ + if (!note.isLastParagraph(paragraphId)) { + note.removeParagraph(paragraphId); + note.persist(); + broadcastNote(note); + } + } + + private void clearParagraphOutput(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook, Message fromMessage) throws IOException { + final String paragraphId = (String) fromMessage.get("id"); + if (paragraphId == null) { + return; + } + String noteId = getOpenNoteId(conn); + final Note note = notebook.getNote(noteId); + NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); + if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { + permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); + return; + } + + note.clearParagraphOutput(paragraphId); + broadcastNote(note); + } + + private void completion(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, + Message fromMessage) throws IOException { + String paragraphId = (String) fromMessage.get("id"); + String buffer = (String) fromMessage.get("buf"); + int cursor = (int) Double.parseDouble(fromMessage.get("cursor").toString()); + Message resp = new Message(OP.COMPLETION_LIST).put("id", paragraphId); + if (paragraphId == null) { + conn.send(serializeMessage(resp)); + return; + } + + final Note note = notebook.getNote(getOpenNoteId(conn)); + List candidates = note.completion(paragraphId, buffer, cursor); + resp.put("completions", candidates); + conn.send(serializeMessage(resp)); + } + + /** + * When angular object updated from client + * + * @param conn the web socket. + * @param notebook the notebook. + * @param fromMessage the message. + */ + private void angularObjectUpdated(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook, Message fromMessage) { + String noteId = (String) fromMessage.get("noteId"); + String paragraphId = (String) fromMessage.get("paragraphId"); + String interpreterGroupId = (String) fromMessage.get("interpreterGroupId"); + String varName = (String) fromMessage.get("name"); + Object varValue = fromMessage.get("value"); + AngularObject ao = null; + boolean global = false; + // propagate change to (Remote) AngularObjectRegistry + Note note = notebook.getNote(noteId); + if (note != null) { + List settings = note.getNoteReplLoader() + .getInterpreterSettings(); + for (InterpreterSetting setting : settings) { + if (setting.getInterpreterGroup(note.id()) == null) { + continue; + } + if (interpreterGroupId.equals(setting.getInterpreterGroup(note.id()).getId())) { + AngularObjectRegistry angularObjectRegistry = setting + .getInterpreterGroup(note.id()).getAngularObjectRegistry(); + // first trying to get local registry + ao = angularObjectRegistry.get(varName, noteId, paragraphId); + if (ao == null) { + // then try notebook scope registry + ao = angularObjectRegistry.get(varName, noteId, null); + if (ao == null) { + // then try global scope registry + ao = angularObjectRegistry.get(varName, null, null); + if (ao == null) { + LOG.warn("Object {} is not binded", varName); + } else { + // path from client -> server + ao.set(varValue, false); + global = true; + } + } else { + // path from client -> server + ao.set(varValue, false); + global = false; + } + } else { + ao.set(varValue, false); + global = false; + } + break; + } + } + } + + if (global) { // broadcast change to all web session that uses related + // interpreter. + for (Note n : notebook.getAllNotes()) { + List settings = note.getNoteReplLoader() + .getInterpreterSettings(); + for (InterpreterSetting setting : settings) { + if (setting.getInterpreterGroup(n.id()) == null) { + continue; + } + if (interpreterGroupId.equals(setting.getInterpreterGroup(n.id()).getId())) { + AngularObjectRegistry angularObjectRegistry = setting + .getInterpreterGroup(n.id()).getAngularObjectRegistry(); + this.broadcastExcept( + n.id(), + new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao) + .put("interpreterGroupId", interpreterGroupId) + .put("noteId", n.id()) + .put("paragraphId", ao.getParagraphId()), + conn); + } + } + } + } else { // broadcast to all web session for the note + this.broadcastExcept( + note.id(), + new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao) + .put("interpreterGroupId", interpreterGroupId) + .put("noteId", note.id()) + .put("paragraphId", ao.getParagraphId()), + conn); + } + } + + /** + * Push the given Angular variable to the target + * interpreter angular registry given a noteId + * and a paragraph id + * @param conn + * @param notebook + * @param fromMessage + * @throws Exception + */ + protected void angularObjectClientBind(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook, Message fromMessage) + throws Exception { + String noteId = fromMessage.getType("noteId"); + String varName = fromMessage.getType("name"); + Object varValue = fromMessage.get("value"); + String paragraphId = fromMessage.getType("paragraphId"); + Note note = notebook.getNote(noteId); + + if (paragraphId == null) { + throw new IllegalArgumentException("target paragraph not specified for " + + "angular value bind"); + } + + if (note != null) { + final InterpreterGroup interpreterGroup = findInterpreterGroupForParagraph(note, + paragraphId); + + final AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry(); + if (registry instanceof RemoteAngularObjectRegistry) { + + RemoteAngularObjectRegistry remoteRegistry = (RemoteAngularObjectRegistry) registry; + pushAngularObjectToRemoteRegistry(noteId, paragraphId, varName, varValue, remoteRegistry, + interpreterGroup.getId(), conn); + + } else { + pushAngularObjectToLocalRepo(noteId, paragraphId, varName, varValue, registry, + interpreterGroup.getId(), conn); + } + } + } + + /** + * Remove the given Angular variable to the target + * interpreter(s) angular registry given a noteId + * and an optional list of paragraph id(s) + * @param conn + * @param notebook + * @param fromMessage + * @throws Exception + */ + protected void angularObjectClientUnbind(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook, Message fromMessage) + throws Exception{ + String noteId = fromMessage.getType("noteId"); + String varName = fromMessage.getType("name"); + String paragraphId = fromMessage.getType("paragraphId"); + Note note = notebook.getNote(noteId); + + if (paragraphId == null) { + throw new IllegalArgumentException("target paragraph not specified for " + + "angular value unBind"); + } + + if (note != null) { + final InterpreterGroup interpreterGroup = findInterpreterGroupForParagraph(note, + paragraphId); + + final AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry(); + + if (registry instanceof RemoteAngularObjectRegistry) { + RemoteAngularObjectRegistry remoteRegistry = (RemoteAngularObjectRegistry) registry; + removeAngularFromRemoteRegistry(noteId, paragraphId, varName, remoteRegistry, + interpreterGroup.getId(), conn); + } else { + removeAngularObjectFromLocalRepo(noteId, paragraphId, varName, registry, + interpreterGroup.getId(), conn); + } + } + } + + private InterpreterGroup findInterpreterGroupForParagraph(Note note, String paragraphId) + throws Exception { + final Paragraph paragraph = note.getParagraph(paragraphId); + if (paragraph == null) { + throw new IllegalArgumentException("Unknown paragraph with id : " + paragraphId); + } + return paragraph.getCurrentRepl().getInterpreterGroup(); + } + + private void pushAngularObjectToRemoteRegistry(String noteId, String paragraphId, + String varName, Object varValue, RemoteAngularObjectRegistry remoteRegistry, + String interpreterGroupId, WebAppSocket conn) { + + final AngularObject ao = remoteRegistry.addAndNotifyRemoteProcess(varName, varValue, + noteId, paragraphId); + + this.broadcastExcept( + noteId, + new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao) + .put("interpreterGroupId", interpreterGroupId) + .put("noteId", noteId) + .put("paragraphId", paragraphId), + conn); + } + + private void removeAngularFromRemoteRegistry(String noteId, String paragraphId, + String varName, RemoteAngularObjectRegistry remoteRegistry, + String interpreterGroupId, WebAppSocket conn) { + final AngularObject ao = remoteRegistry.removeAndNotifyRemoteProcess(varName, noteId, + paragraphId); + this.broadcastExcept( + noteId, + new Message(OP.ANGULAR_OBJECT_REMOVE).put("angularObject", ao) + .put("interpreterGroupId", interpreterGroupId) + .put("noteId", noteId) + .put("paragraphId", paragraphId), + conn); + } + + private void pushAngularObjectToLocalRepo(String noteId, String paragraphId, String varName, + Object varValue, AngularObjectRegistry registry, + String interpreterGroupId, WebAppSocket conn) { + AngularObject angularObject = registry.get(varName, noteId, paragraphId); + if (angularObject == null) { + angularObject = registry.add(varName, varValue, noteId, paragraphId); + } else { + angularObject.set(varValue, true); + } + + this.broadcastExcept( + noteId, + new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", angularObject) + .put("interpreterGroupId", interpreterGroupId) + .put("noteId", noteId) + .put("paragraphId", paragraphId), + conn); + } + + private void removeAngularObjectFromLocalRepo(String noteId, String paragraphId, String varName, + AngularObjectRegistry registry, String interpreterGroupId, WebAppSocket conn) { + final AngularObject removed = registry.remove(varName, noteId, paragraphId); + if (removed != null) { + this.broadcastExcept( + noteId, + new Message(OP.ANGULAR_OBJECT_REMOVE).put("angularObject", removed) + .put("interpreterGroupId", interpreterGroupId) + .put("noteId", noteId) + .put("paragraphId", paragraphId), + conn); + } + } + + private void moveParagraph(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, + Message fromMessage) throws IOException { + final String paragraphId = (String) fromMessage.get("id"); + if (paragraphId == null) { + return; + } + + final int newIndex = (int) Double.parseDouble(fromMessage.get("index") + .toString()); + String noteId = getOpenNoteId(conn); + final Note note = notebook.getNote(noteId); + NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); + if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { + permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); + return; + } + + note.moveParagraph(paragraphId, newIndex); + note.persist(); + broadcastNote(note); + } + + private void insertParagraph(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook, Message fromMessage) throws IOException { + final int index = (int) Double.parseDouble(fromMessage.get("index") + .toString()); + String noteId = getOpenNoteId(conn); + final Note note = notebook.getNote(noteId); + NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); + if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { + permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); + return; + } + + note.insertParagraph(index); + note.persist(); + broadcastNote(note); + } + + private void cancelParagraph(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, + Message fromMessage) throws IOException { + final String paragraphId = (String) fromMessage.get("id"); + if (paragraphId == null) { + return; + } + + String noteId = getOpenNoteId(conn); + final Note note = notebook.getNote(noteId); + NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); + if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { + permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); + return; + } + + Paragraph p = note.getParagraph(paragraphId); + p.abort(); + } + + private void runParagraph(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, + Message fromMessage) throws IOException { + final String paragraphId = (String) fromMessage.get("id"); + if (paragraphId == null) { + return; + } + + String noteId = getOpenNoteId(conn); + final Note note = notebook.getNote(noteId); + NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); + if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { + permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); + return; + } + + Paragraph p = note.getParagraph(paragraphId); + String text = (String) fromMessage.get("paragraph"); + p.setText(text); + p.setTitle((String) fromMessage.get("title")); + if (!fromMessage.principal.equals("anonymous")) { + AuthenticationInfo authenticationInfo = new AuthenticationInfo(fromMessage.principal, + fromMessage.ticket); + p.setAuthenticationInfo(authenticationInfo); + + } else { + p.setAuthenticationInfo(new AuthenticationInfo()); + } + + Map params = (Map) fromMessage + .get("params"); + p.settings.setParams(params); + Map config = (Map) fromMessage + .get("config"); + p.setConfig(config); + // if it's the last paragraph, let's add a new one + boolean isTheLastParagraph = note.getLastParagraph().getId() + .equals(p.getId()); + if (!Strings.isNullOrEmpty(text) && isTheLastParagraph) { + note.addParagraph(); + } + + note.persist(); + try { + note.run(paragraphId); + } catch (Exception ex) { + LOG.error("Exception from run", ex); + if (p != null) { + p.setReturn( + new InterpreterResult(InterpreterResult.Code.ERROR, ex.getMessage()), + ex); + p.setStatus(Status.ERROR); + broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p)); + } + } + } + + private void sendAllConfigurations(WebAppSocket conn, HashSet userAndRoles, + Notebook notebook) throws IOException { + ZeppelinConfiguration conf = notebook.getConf(); + + Map configurations = conf.dumpConfigurations(conf, + new ZeppelinConfiguration.ConfigurationKeyPredicate() { + @Override + public boolean apply(String key) { + return !key.contains("password") && + !key.equals( + ConfVars + .ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING + .getVarName()); + } + }); + + conn.send(serializeMessage(new Message(OP.CONFIGURATIONS_INFO) + .put("configurations", configurations))); + } + + private void checkpointNotebook(WebAppSocket conn, Notebook notebook, + Message fromMessage) throws IOException { + String noteId = (String) fromMessage.get("noteId"); + String commitMessage = (String) fromMessage.get("commitMessage"); + notebook.checkpointNote(noteId, commitMessage); + } + + /** + * This callback is for the paragraph that runs on ZeppelinServer + * @param noteId + * @param paragraphId + * @param output output to append + */ + @Override + public void onOutputAppend(String noteId, String paragraphId, String output) { + Message msg = new Message(OP.PARAGRAPH_APPEND_OUTPUT) + .put("noteId", noteId) + .put("paragraphId", paragraphId) + .put("data", output); + Paragraph paragraph = notebook().getNote(noteId).getParagraph(paragraphId); + broadcast(noteId, msg); + } + + /** + * This callback is for the paragraph that runs on ZeppelinServer + * @param noteId + * @param paragraphId + * @param output output to update (replace) + */ + @Override + public void onOutputUpdated(String noteId, String paragraphId, String output) { + Message msg = new Message(OP.PARAGRAPH_UPDATE_OUTPUT) + .put("noteId", noteId) + .put("paragraphId", paragraphId) + .put("data", output); + Paragraph paragraph = notebook().getNote(noteId).getParagraph(paragraphId); + broadcast(noteId, msg); + } + + /** + * Need description here. + * + */ + public static class ParagraphListenerImpl implements ParagraphJobListener { + private JobMangerServer appMainServer; + private Note note; + + public ParagraphListenerImpl(JobMangerServer appMainServer, Note note) { + this.appMainServer = appMainServer; + this.note = note; + } + + @Override + public void onProgressUpdate(Job job, int progress) { + appMainServer.broadcast( + note.id(), + new Message(OP.PROGRESS).put("id", job.getId()).put("progress", + job.progress())); + } + + @Override + public void beforeStatusChange(Job job, Status before, Status after) { + } + + @Override + public void afterStatusChange(Job job, Status before, Status after) { + if (after == Status.ERROR) { + if (job.getException() != null) { + LOG.error("Error", job.getException()); + } + } + + if (job.isTerminated()) { + LOG.info("Job {} is finished", job.getId()); + try { + note.persist(); + } catch (IOException e) { + LOG.error(e.toString(), e); + } + } + appMainServer.broadcastNote(note); + } + + /** + * This callback is for praragraph that runs on RemoteInterpreterProcess + * @param paragraph + * @param out + * @param output + */ + @Override + public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String output) { + Message msg = new Message(OP.PARAGRAPH_APPEND_OUTPUT) + .put("noteId", paragraph.getNote().getId()) + .put("paragraphId", paragraph.getId()) + .put("data", output); + + appMainServer.broadcast(paragraph.getNote().getId(), msg); + } + + /** + * This callback is for paragraph that runs on RemoteInterpreterProcess + * @param paragraph + * @param out + * @param output + */ + @Override + public void onOutputUpdate(Paragraph paragraph, InterpreterOutput out, String output) { + Message msg = new Message(OP.PARAGRAPH_UPDATE_OUTPUT) + .put("noteId", paragraph.getNote().getId()) + .put("paragraphId", paragraph.getId()) + .put("data", output); + + appMainServer.broadcast(paragraph.getNote().getId(), msg); + } + } + + @Override + public ParagraphJobListener getParagraphJobListener(Note note) { + return new ParagraphListenerImpl(this, note); + } + + private void sendAllAngularObjects(Note note, WebAppSocket conn) throws IOException { + List settings = note.getNoteReplLoader() + .getInterpreterSettings(); + if (settings == null || settings.size() == 0) { + return; + } + + for (InterpreterSetting intpSetting : settings) { + AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.id()) + .getAngularObjectRegistry(); + List objects = registry.getAllWithGlobal(note.id()); + for (AngularObject object : objects) { + conn.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE) + .put("angularObject", object) + .put("interpreterGroupId", + intpSetting.getInterpreterGroup(note.id()).getId()) + .put("noteId", note.id()) + .put("paragraphId", object.getParagraphId()) + )); + } + } + } + + @Override + public void onAdd(String interpreterGroupId, AngularObject object) { + onUpdate(interpreterGroupId, object); + } + + @Override + public void onUpdate(String interpreterGroupId, AngularObject object) { + Notebook notebook = notebook(); + if (notebook == null) { + return; + } + + List notes = notebook.getAllNotes(); + for (Note note : notes) { + if (object.getNoteId() != null && !note.id().equals(object.getNoteId())) { + continue; + } + + List intpSettings = note.getNoteReplLoader() + .getInterpreterSettings(); + if (intpSettings.isEmpty()) + continue; + for (InterpreterSetting setting : intpSettings) { + if (setting.getInterpreterGroup(note.id()).getId().equals(interpreterGroupId)) { + broadcast( + note.id(), + new Message(OP.ANGULAR_OBJECT_UPDATE) + .put("angularObject", object) + .put("interpreterGroupId", interpreterGroupId) + .put("noteId", note.id()) + .put("paragraphId", object.getParagraphId())); + } + } + } + } + + @Override + public void onRemove(String interpreterGroupId, String name, String noteId, String paragraphId) { + Notebook notebook = notebook(); + List notes = notebook.getAllNotes(); + for (Note note : notes) { + if (noteId != null && !note.id().equals(noteId)) { + continue; + } + + List ids = note.getNoteReplLoader().getInterpreters(); + for (String id : ids) { + if (id.equals(interpreterGroupId)) { + broadcast( + note.id(), + new Message(OP.ANGULAR_OBJECT_REMOVE).put("name", name).put( + "noteId", noteId).put("paragraphId", paragraphId)); + } + } + } + } +} + diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 521402aebf1..9c16285009e 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -16,30 +16,20 @@ */ package org.apache.zeppelin.socket; -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.UnknownHostException; -import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; - -import javax.servlet.http.HttpServletRequest; - import com.google.common.base.Strings; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; - import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; -import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.interpreter.InterpreterOutput; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; import org.apache.zeppelin.notebook.*; import org.apache.zeppelin.notebook.socket.Message; @@ -48,6 +38,7 @@ import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.server.ZeppelinServer; import org.apache.zeppelin.ticket.TicketContainer; +import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.utils.SecurityUtils; import org.eclipse.jetty.websocket.servlet.WebSocketServlet; import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; @@ -55,17 +46,22 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; + /** * Zeppelin websocket service. */ -public class NotebookServer extends WebSocketServlet implements - NotebookSocketListener, JobListenerFactory, AngularObjectRegistryListener, - RemoteInterpreterProcessListener { +public class NotebookServer extends AppMainServer implements JobListenerFactory, + AngularObjectRegistryListener, RemoteInterpreterProcessListener { private static final Logger LOG = LoggerFactory.getLogger(NotebookServer.class); Gson gson = new GsonBuilder() .setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); - final Map> noteSocketMap = new HashMap<>(); - final Queue connectedSockets = new ConcurrentLinkedQueue<>(); + final Queue connectedSockets = new ConcurrentLinkedQueue<>(); private Notebook notebook() { return ZeppelinServer.notebook; @@ -73,7 +69,7 @@ private Notebook notebook() { @Override public void configure(WebSocketServletFactory factory) { - factory.setCreator(new NotebookWebSocketCreator(this)); + factory.setCreator(new WebAppSocketCreator(this)); } public boolean checkOrigin(HttpServletRequest request, String origin) { @@ -87,19 +83,19 @@ public boolean checkOrigin(HttpServletRequest request, String origin) { return false; } - public NotebookSocket doWebSocketConnect(HttpServletRequest req, String protocol) { - return new NotebookSocket(req, protocol, this); + public WebAppSocket doWebSocketConnect(HttpServletRequest req, String protocol) { + return new WebAppSocket(req, protocol, this); } @Override - public void onOpen(NotebookSocket conn) { + public void onOpen(WebAppSocket conn) { LOG.info("New connection from {} : {}", conn.getRequest().getRemoteAddr(), conn.getRequest().getRemotePort()); connectedSockets.add(conn); } @Override - public void onMessage(NotebookSocket conn, String msg) { + public void onMessage(WebAppSocket conn, String msg) { Notebook notebook = notebook(); try { Message messagereceived = deserializeMessage(msg); @@ -111,14 +107,14 @@ public void onMessage(NotebookSocket conn, String msg) { if (LOG.isTraceEnabled()) { LOG.trace("RECEIVE MSG = " + messagereceived); } - + String ticket = TicketContainer.instance.getTicket(messagereceived.principal); if (ticket != null && !ticket.equals(messagereceived.ticket)) throw new Exception("Invalid ticket " + messagereceived.ticket + " != " + ticket); ZeppelinConfiguration conf = ZeppelinConfiguration.create(); boolean allowAnonymous = conf. - getBoolean(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED); + getBoolean(ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED); if (!allowAnonymous && messagereceived.principal.equals("anonymous")) { throw new Exception("Anonymous access not allowed "); } @@ -212,7 +208,7 @@ public void onMessage(NotebookSocket conn, String msg) { } @Override - public void onClose(NotebookSocket conn, int code, String reason) { + public void onClose(WebAppSocket conn, int code, String reason) { LOG.info("Closed connection to {} : {}. ({}) {}", conn.getRequest() .getRemoteAddr(), conn.getRequest().getRemotePort(), code, reason); removeConnectionFromAllNote(conn); @@ -227,60 +223,26 @@ protected String serializeMessage(Message m) { return gson.toJson(m); } - private void addConnectionToNote(String noteId, NotebookSocket socket) { - synchronized (noteSocketMap) { - removeConnectionFromAllNote(socket); // make sure a socket relates only a - // single note. - List socketList = noteSocketMap.get(noteId); - if (socketList == null) { - socketList = new LinkedList<>(); - noteSocketMap.put(noteId, socketList); - } - if (!socketList.contains(socket)) { - socketList.add(socket); - } - } + private void addConnectionToNote(String noteId, WebAppSocket socket) { + addConnectionToKey(noteId, socket); } - private void removeConnectionFromNote(String noteId, NotebookSocket socket) { - synchronized (noteSocketMap) { - List socketList = noteSocketMap.get(noteId); - if (socketList != null) { - socketList.remove(socket); - } - } + private void removeConnectionFromNote(String noteId, WebAppSocket socket) { + removeConnectionFromKey(noteId, socket); } private void removeNote(String noteId) { - synchronized (noteSocketMap) { - List socketList = noteSocketMap.remove(noteId); - } + removeKey(noteId); } - private void removeConnectionFromAllNote(NotebookSocket socket) { - synchronized (noteSocketMap) { - Set keys = noteSocketMap.keySet(); - for (String noteId : keys) { - removeConnectionFromNote(noteId, socket); - } - } + private void removeConnectionFromAllNote(WebAppSocket socket) { + removeConnectionFromAllKey(socket); } - private String getOpenNoteId(NotebookSocket socket) { - String id = null; - synchronized (noteSocketMap) { - Set keys = noteSocketMap.keySet(); - for (String noteId : keys) { - List sockets = noteSocketMap.get(noteId); - if (sockets.contains(socket)) { - id = noteId; - } - } - } - - return id; + private String getOpenNoteId(WebAppSocket socket) { + return getOpenKey(socket); } - +//lcs 여기까지 private void broadcastToNoteBindedInterpreter(String interpreterGroupId, Message m) { Notebook notebook = notebook(); @@ -297,12 +259,12 @@ private void broadcastToNoteBindedInterpreter(String interpreterGroupId, private void broadcast(String noteId, Message m) { synchronized (noteSocketMap) { - List socketLists = noteSocketMap.get(noteId); + List socketLists = noteSocketMap.get(noteId); if (socketLists == null || socketLists.size() == 0) { return; } LOG.debug("SEND >> " + m.op); - for (NotebookSocket conn : socketLists) { + for (WebAppSocket conn : socketLists) { try { conn.send(serializeMessage(m)); } catch (IOException e) { @@ -312,14 +274,14 @@ private void broadcast(String noteId, Message m) { } } - private void broadcastExcept(String noteId, Message m, NotebookSocket exclude) { + private void broadcastExcept(String noteId, Message m, WebAppSocket exclude) { synchronized (noteSocketMap) { - List socketLists = noteSocketMap.get(noteId); + List socketLists = noteSocketMap.get(noteId); if (socketLists == null || socketLists.size() == 0) { return; } LOG.debug("SEND >> " + m.op); - for (NotebookSocket conn : socketLists) { + for (WebAppSocket conn : socketLists) { if (exclude.equals(conn)) { continue; } @@ -333,7 +295,7 @@ private void broadcastExcept(String noteId, Message m, NotebookSocket exclude) { } private void broadcastAll(Message m) { - for (NotebookSocket conn : connectedSockets) { + for (WebAppSocket conn : connectedSockets) { try { conn.send(serializeMessage(m)); } catch (IOException e) { @@ -342,7 +304,7 @@ private void broadcastAll(Message m) { } } - private void unicast(Message m, NotebookSocket conn) { + private void unicast(Message m, WebAppSocket conn) { try { conn.send(serializeMessage(m)); } catch (IOException e) { @@ -393,7 +355,7 @@ public void broadcastNoteList() { broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo)); } - public void unicastNoteList(NotebookSocket conn) { + public void unicastNoteList(WebAppSocket conn) { List> notesInfo = generateNotebooksInfo(false); unicast(new Message(OP.NOTES_INFO).put("notes", notesInfo), conn); } @@ -403,7 +365,7 @@ public void broadcastReloadedNoteList() { broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo)); } - void permissionError(NotebookSocket conn, String op, Set userAndRoles, + void permissionError(WebAppSocket conn, String op, Set userAndRoles, Set allowed) throws IOException { LOG.info("Cannot {}. Connection readers {}. Allowed readers {}", op, userAndRoles, allowed); @@ -416,7 +378,7 @@ void permissionError(NotebookSocket conn, String op, Set userAndRoles, "But the user " + userName + " belongs to: " + userAndRoles.toString()))); } - private void sendNote(NotebookSocket conn, HashSet userAndRoles, Notebook notebook, + private void sendNote(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws IOException { LOG.info("New operation from {} : {} : {} : {} : {}", conn.getRequest().getRemoteAddr(), @@ -442,7 +404,7 @@ private void sendNote(NotebookSocket conn, HashSet userAndRoles, Noteboo } } - private void sendHomeNote(NotebookSocket conn, HashSet userAndRoles, + private void sendHomeNote(WebAppSocket conn, HashSet userAndRoles, Notebook notebook) throws IOException { String noteId = notebook.getConf().getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); @@ -466,7 +428,7 @@ private void sendHomeNote(NotebookSocket conn, HashSet userAndRoles, } } - private void updateNote(NotebookSocket conn, HashSet userAndRoles, + private void updateNote(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws SchedulerException, IOException { String noteId = (String) fromMessage.get("id"); @@ -515,7 +477,7 @@ private boolean isCronUpdated(Map configA, return cronUpdated; } - private void createNote(NotebookSocket conn, HashSet userAndRoles, + private void createNote(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message message) throws IOException { Note note = notebook.createNote(); @@ -529,12 +491,12 @@ private void createNote(NotebookSocket conn, HashSet userAndRoles, } note.persist(); - addConnectionToNote(note.id(), (NotebookSocket) conn); + addConnectionToNote(note.id(), (WebAppSocket) conn); conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note))); broadcastNoteList(); } - private void removeNote(NotebookSocket conn, HashSet userAndRoles, + private void removeNote(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws IOException { String noteId = (String) fromMessage.get("id"); @@ -554,7 +516,7 @@ private void removeNote(NotebookSocket conn, HashSet userAndRoles, broadcastNoteList(); } - private void updateParagraph(NotebookSocket conn, HashSet userAndRoles, + private void updateParagraph(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws IOException { String paragraphId = (String) fromMessage.get("id"); if (paragraphId == null) { @@ -582,18 +544,18 @@ private void updateParagraph(NotebookSocket conn, HashSet userAndRoles, broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p)); } - private void cloneNote(NotebookSocket conn, HashSet userAndRoles, + private void cloneNote(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws IOException, CloneNotSupportedException { String noteId = getOpenNoteId(conn); String name = (String) fromMessage.get("name"); Note newNote = notebook.cloneNote(noteId, name); - addConnectionToNote(newNote.id(), (NotebookSocket) conn); + addConnectionToNote(newNote.id(), (WebAppSocket) conn); conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", newNote))); broadcastNoteList(); } - protected Note importNote(NotebookSocket conn, HashSet userAndRoles, + protected Note importNote(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws IOException { Note note = null; @@ -608,7 +570,7 @@ protected Note importNote(NotebookSocket conn, HashSet userAndRoles, return note; } - private void removeParagraph(NotebookSocket conn, HashSet userAndRoles, + private void removeParagraph(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws IOException { final String paragraphId = (String) fromMessage.get("id"); if (paragraphId == null) { @@ -630,7 +592,7 @@ private void removeParagraph(NotebookSocket conn, HashSet userAndRoles, } } - private void clearParagraphOutput(NotebookSocket conn, HashSet userAndRoles, + private void clearParagraphOutput(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws IOException { final String paragraphId = (String) fromMessage.get("id"); if (paragraphId == null) { @@ -648,7 +610,7 @@ private void clearParagraphOutput(NotebookSocket conn, HashSet userAndRo broadcastNote(note); } - private void completion(NotebookSocket conn, HashSet userAndRoles, Notebook notebook, + private void completion(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws IOException { String paragraphId = (String) fromMessage.get("id"); String buffer = (String) fromMessage.get("buf"); @@ -672,7 +634,7 @@ private void completion(NotebookSocket conn, HashSet userAndRoles, Noteb * @param notebook the notebook. * @param fromMessage the message. */ - private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRoles, + private void angularObjectUpdated(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) { String noteId = (String) fromMessage.get("noteId"); String paragraphId = (String) fromMessage.get("paragraphId"); @@ -764,7 +726,7 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo * @param fromMessage * @throws Exception */ - protected void angularObjectClientBind(NotebookSocket conn, HashSet userAndRoles, + protected void angularObjectClientBind(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws Exception { String noteId = fromMessage.getType("noteId"); @@ -805,7 +767,7 @@ protected void angularObjectClientBind(NotebookSocket conn, HashSet user * @param fromMessage * @throws Exception */ - protected void angularObjectClientUnbind(NotebookSocket conn, HashSet userAndRoles, + protected void angularObjectClientUnbind(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws Exception{ String noteId = fromMessage.getType("noteId"); @@ -846,7 +808,7 @@ private InterpreterGroup findInterpreterGroupForParagraph(Note note, String para private void pushAngularObjectToRemoteRegistry(String noteId, String paragraphId, String varName, Object varValue, RemoteAngularObjectRegistry remoteRegistry, - String interpreterGroupId, NotebookSocket conn) { + String interpreterGroupId, WebAppSocket conn) { final AngularObject ao = remoteRegistry.addAndNotifyRemoteProcess(varName, varValue, noteId, paragraphId); @@ -862,7 +824,7 @@ private void pushAngularObjectToRemoteRegistry(String noteId, String paragraphId private void removeAngularFromRemoteRegistry(String noteId, String paragraphId, String varName, RemoteAngularObjectRegistry remoteRegistry, - String interpreterGroupId, NotebookSocket conn) { + String interpreterGroupId, WebAppSocket conn) { final AngularObject ao = remoteRegistry.removeAndNotifyRemoteProcess(varName, noteId, paragraphId); this.broadcastExcept( @@ -876,7 +838,7 @@ private void removeAngularFromRemoteRegistry(String noteId, String paragraphId, private void pushAngularObjectToLocalRepo(String noteId, String paragraphId, String varName, Object varValue, AngularObjectRegistry registry, - String interpreterGroupId, NotebookSocket conn) { + String interpreterGroupId, WebAppSocket conn) { AngularObject angularObject = registry.get(varName, noteId, paragraphId); if (angularObject == null) { angularObject = registry.add(varName, varValue, noteId, paragraphId); @@ -894,7 +856,7 @@ private void pushAngularObjectToLocalRepo(String noteId, String paragraphId, Str } private void removeAngularObjectFromLocalRepo(String noteId, String paragraphId, String varName, - AngularObjectRegistry registry, String interpreterGroupId, NotebookSocket conn) { + AngularObjectRegistry registry, String interpreterGroupId, WebAppSocket conn) { final AngularObject removed = registry.remove(varName, noteId, paragraphId); if (removed != null) { this.broadcastExcept( @@ -907,7 +869,7 @@ private void removeAngularObjectFromLocalRepo(String noteId, String paragraphId, } } - private void moveParagraph(NotebookSocket conn, HashSet userAndRoles, Notebook notebook, + private void moveParagraph(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws IOException { final String paragraphId = (String) fromMessage.get("id"); if (paragraphId == null) { @@ -929,7 +891,7 @@ private void moveParagraph(NotebookSocket conn, HashSet userAndRoles, No broadcastNote(note); } - private void insertParagraph(NotebookSocket conn, HashSet userAndRoles, + private void insertParagraph(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws IOException { final int index = (int) Double.parseDouble(fromMessage.get("index") .toString()); @@ -946,7 +908,7 @@ private void insertParagraph(NotebookSocket conn, HashSet userAndRoles, broadcastNote(note); } - private void cancelParagraph(NotebookSocket conn, HashSet userAndRoles, Notebook notebook, + private void cancelParagraph(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws IOException { final String paragraphId = (String) fromMessage.get("id"); if (paragraphId == null) { @@ -965,7 +927,7 @@ private void cancelParagraph(NotebookSocket conn, HashSet userAndRoles, p.abort(); } - private void runParagraph(NotebookSocket conn, HashSet userAndRoles, Notebook notebook, + private void runParagraph(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, Message fromMessage) throws IOException { final String paragraphId = (String) fromMessage.get("id"); if (paragraphId == null) { @@ -1021,7 +983,7 @@ private void runParagraph(NotebookSocket conn, HashSet userAndRoles, Not } } - private void sendAllConfigurations(NotebookSocket conn, HashSet userAndRoles, + private void sendAllConfigurations(WebAppSocket conn, HashSet userAndRoles, Notebook notebook) throws IOException { ZeppelinConfiguration conf = notebook.getConf(); @@ -1030,8 +992,8 @@ private void sendAllConfigurations(NotebookSocket conn, HashSet userAndR @Override public boolean apply(String key) { return !key.contains("password") && - !key.equals(ZeppelinConfiguration - .ConfVars + !key.equals( + ConfVars .ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING .getVarName()); } @@ -1041,7 +1003,7 @@ public boolean apply(String key) { .put("configurations", configurations))); } - private void checkpointNotebook(NotebookSocket conn, Notebook notebook, + private void checkpointNotebook(WebAppSocket conn, Notebook notebook, Message fromMessage) throws IOException { String noteId = (String) fromMessage.get("noteId"); String commitMessage = (String) fromMessage.get("commitMessage"); @@ -1085,17 +1047,17 @@ public void onOutputUpdated(String noteId, String paragraphId, String output) { * */ public static class ParagraphListenerImpl implements ParagraphJobListener { - private NotebookServer notebookServer; + private NotebookServer appMainServer; private Note note; - public ParagraphListenerImpl(NotebookServer notebookServer, Note note) { - this.notebookServer = notebookServer; + public ParagraphListenerImpl(NotebookServer appMainServer, Note note) { + this.appMainServer = appMainServer; this.note = note; } @Override public void onProgressUpdate(Job job, int progress) { - notebookServer.broadcast( + appMainServer.broadcast( note.id(), new Message(OP.PROGRESS).put("id", job.getId()).put("progress", job.progress())); @@ -1121,7 +1083,7 @@ public void afterStatusChange(Job job, Status before, Status after) { LOG.error(e.toString(), e); } } - notebookServer.broadcastNote(note); + appMainServer.broadcastNote(note); } /** @@ -1137,7 +1099,7 @@ public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String ou .put("paragraphId", paragraph.getId()) .put("data", output); - notebookServer.broadcast(paragraph.getNote().getId(), msg); + appMainServer.broadcast(paragraph.getNote().getId(), msg); } /** @@ -1153,7 +1115,7 @@ public void onOutputUpdate(Paragraph paragraph, InterpreterOutput out, String ou .put("paragraphId", paragraph.getId()) .put("data", output); - notebookServer.broadcast(paragraph.getNote().getId(), msg); + appMainServer.broadcast(paragraph.getNote().getId(), msg); } } @@ -1162,7 +1124,7 @@ public ParagraphJobListener getParagraphJobListener(Note note) { return new ParagraphListenerImpl(this, note); } - private void sendAllAngularObjects(Note note, NotebookSocket conn) throws IOException { + private void sendAllAngularObjects(Note note, WebAppSocket conn) throws IOException { List settings = note.getNoteReplLoader() .getInterpreterSettings(); if (settings == null || settings.size() == 0) { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocket.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/WebAppSocket.java similarity index 90% rename from zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocket.java rename to zeppelin-server/src/main/java/org/apache/zeppelin/socket/WebAppSocket.java index 5d68bf5ec2d..b875bb529ba 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocket.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/WebAppSocket.java @@ -26,15 +26,15 @@ /** * Notebook websocket */ -public class NotebookSocket extends WebSocketAdapter { +public class WebAppSocket extends WebSocketAdapter { private Session connection; - private NotebookSocketListener listener; + private WebSocketListener listener; private HttpServletRequest request; private String protocol; - public NotebookSocket(HttpServletRequest req, String protocol, - NotebookSocketListener listener) { + public WebAppSocket(HttpServletRequest req, String protocol, + WebSocketListener listener) { this.listener = listener; this.request = req; this.protocol = protocol; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookWebSocketCreator.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/WebAppSocketCreator.java similarity index 75% rename from zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookWebSocketCreator.java rename to zeppelin-server/src/main/java/org/apache/zeppelin/socket/WebAppSocketCreator.java index 1b8e2f44e02..cf7104e173b 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookWebSocketCreator.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/WebAppSocketCreator.java @@ -21,16 +21,16 @@ import org.eclipse.jetty.websocket.servlet.WebSocketCreator; /** - * Responsible to create the WebSockets for the NotebookServer. + * Responsible to create the WebSockets for the AppMainServer. */ -public class NotebookWebSocketCreator implements WebSocketCreator { - private NotebookServer notebookServer; +public class WebAppSocketCreator implements WebSocketCreator { + private AppMainServer appMainServer; - public NotebookWebSocketCreator(NotebookServer notebookServer) { - this.notebookServer = notebookServer; + public WebAppSocketCreator(AppMainServer appMainServer) { + this.appMainServer = appMainServer; } public Object createWebSocket(ServletUpgradeRequest request, ServletUpgradeResponse response) { - return new NotebookSocket(request.getHttpServletRequest(), "", notebookServer); + return new WebAppSocket(request.getHttpServletRequest(), "", appMainServer); } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocketListener.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/WebSocketListener.java similarity index 78% rename from zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocketListener.java rename to zeppelin-server/src/main/java/org/apache/zeppelin/socket/WebSocketListener.java index 6fc073402b4..e517262d4c8 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookSocketListener.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/WebSocketListener.java @@ -17,10 +17,10 @@ package org.apache.zeppelin.socket; /** - * NoteboookSocket listener + * WebSocket listener */ -public interface NotebookSocketListener { - void onClose(NotebookSocket socket, int code, String message); - void onOpen(NotebookSocket socket); - void onMessage(NotebookSocket socket, String message); +public interface WebSocketListener { + void onClose(WebAppSocket socket, int code, String message); + void onOpen(WebAppSocket socket); + void onMessage(WebAppSocket socket, String message); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocketListener.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/WebSocketServer.java similarity index 78% rename from zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocketListener.java rename to zeppelin-server/src/main/java/org/apache/zeppelin/socket/WebSocketServer.java index 5f44eebb755..b67c001e2d5 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobManagerSocketListener.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/WebSocketServer.java @@ -17,10 +17,8 @@ package org.apache.zeppelin.socket; /** - * NoteboookSocket listener + * WebSocket Server Interface */ -public interface JobManagerSocketListener { - void onClose(JobManagerSocket socket, int code, String message); - void onOpen(JobManagerSocket socket); - void onMessage(JobManagerSocket socket, String message); +public interface WebSocketServer { + void onMessage(WebAppSocket conn, String msg); } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/AppMainServerTest.java similarity index 92% rename from zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java rename to zeppelin-server/src/test/java/org/apache/zeppelin/socket/AppMainServerTest.java index b904b6849e0..d4bfea2f22f 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/AppMainServerTest.java @@ -53,9 +53,9 @@ /** * BASIC Zeppelin rest api tests */ -public class NotebookServerTest extends AbstractTestRestApi { +public class AppMainServerTest extends AbstractTestRestApi { private static Notebook notebook; - private static NotebookServer notebookServer; + private static AppMainServer appMainServer; private static Gson gson; @BeforeClass @@ -63,7 +63,7 @@ public static void init() throws Exception { AbstractTestRestApi.startUp(); gson = new Gson(); notebook = ZeppelinServer.notebook; - notebookServer = ZeppelinServer.notebookWsServer; + appMainServer = ZeppelinServer.notebookWsServer; } @AfterClass @@ -73,7 +73,7 @@ public static void destroy() throws Exception { @Test public void checkOrigin() throws UnknownHostException { - NotebookServer server = new NotebookServer(); + AppMainServer server = new AppMainServer(); String origin = "http://" + InetAddress.getLocalHost().getHostName() + ":8080"; assertTrue("Origin " + origin + " is not allowed. Please check your hostname.", @@ -82,7 +82,7 @@ public void checkOrigin() throws UnknownHostException { @Test public void checkInvalidOrigin(){ - NotebookServer server = new NotebookServer(); + AppMainServer server = new AppMainServer(); assertFalse(server.checkOrigin(new TestHttpServletRequest(), "http://evillocalhost:8080")); } @@ -116,18 +116,18 @@ public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() thr assertEquals(sock1, sock1); assertNotEquals(sock1, sock2); - notebookServer.onOpen(sock1); - notebookServer.onOpen(sock2); + appMainServer.onOpen(sock1); + appMainServer.onOpen(sock2); verify(sock1, times(0)).send(anyString()); // getNote, getAngularObject // open the same notebook from sockets - notebookServer.onMessage(sock1, gson.toJson(new Message(OP.GET_NOTE).put("id", note1.getId()))); - notebookServer.onMessage(sock2, gson.toJson(new Message(OP.GET_NOTE).put("id", note1.getId()))); + appMainServer.onMessage(sock1, gson.toJson(new Message(OP.GET_NOTE).put("id", note1.getId()))); + appMainServer.onMessage(sock2, gson.toJson(new Message(OP.GET_NOTE).put("id", note1.getId()))); reset(sock1); reset(sock2); // update object from sock1 - notebookServer.onMessage(sock1, gson.toJson( + appMainServer.onMessage(sock1, gson.toJson( new Message(OP.ANGULAR_OBJECT_UPDATED) .put("noteId", note1.getId()) .put("name", "object1") @@ -149,13 +149,13 @@ public void testImportNotebook() throws IOException { "paragraphs import\",\"config\":{},\"settings\":{}}]," + "\"name\": \"Test Zeppelin notebook import\",\"config\": " + "{}}}}"; - Message messageReceived = notebookServer.deserializeMessage(msg); + Message messageReceived = appMainServer.deserializeMessage(msg); Note note = null; try { - note = notebookServer.importNote(null, null, notebook, messageReceived); + note = appMainServer.importNote(null, null, notebook, messageReceived); } catch (NullPointerException e) { //broadcastNoteList(); failed nothing to worry. - LOG.error("Exception in NotebookServerTest while testImportNotebook, failed nothing to " + + LOG.error("Exception in AppMainServerTest while testImportNotebook, failed nothing to " + "worry ", e); } @@ -176,7 +176,7 @@ public void should_bind_angular_object_to_remote_for_paragraphs() throws Excepti .put("value", value) .put("paragraphId", "paragraphId"); - final NotebookServer server = new NotebookServer(); + final AppMainServer server = new AppMainServer(); final Notebook notebook = mock(Notebook.class); final Note note = mock(Note.class, RETURNS_DEEP_STUBS); @@ -227,7 +227,7 @@ public void should_bind_angular_object_to_local_for_paragraphs() throws Exceptio .put("value", value) .put("paragraphId", "paragraphId"); - final NotebookServer server = new NotebookServer(); + final AppMainServer server = new AppMainServer(); final Notebook notebook = mock(Notebook.class); final Note note = mock(Note.class, RETURNS_DEEP_STUBS); when(notebook.getNote("noteId")).thenReturn(note); @@ -273,7 +273,7 @@ public void should_unbind_angular_object_from_remote_for_paragraphs() throws Exc .put("name", varName) .put("paragraphId", "paragraphId"); - final NotebookServer server = new NotebookServer(); + final AppMainServer server = new AppMainServer(); final Notebook notebook = mock(Notebook.class); final Note note = mock(Note.class, RETURNS_DEEP_STUBS); when(notebook.getNote("noteId")).thenReturn(note); @@ -318,7 +318,7 @@ public void should_unbind_angular_object_from_local_for_paragraphs() throws Exce .put("name", varName) .put("paragraphId", "paragraphId"); - final NotebookServer server = new NotebookServer(); + final AppMainServer server = new AppMainServer(); final Notebook notebook = mock(Notebook.class); final Note note = mock(Note.class, RETURNS_DEEP_STUBS); when(notebook.getNote("noteId")).thenReturn(note); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 13f17d6417f..4d697dd44d4 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -85,15 +85,15 @@ angular.module('zeppelinWebApp') var newItem = angular.copy(changedItem); jobInfomations.push(newItem); indexStore[changedItem.notebookId] = newItem; - console.log('111111111111111111111111'); + console.log('new Item Notebook Job ', newItem); } else { - console.log('222222222222222222222222'); var changeOriginTarget = indexStore[changedItem.notebookId]; changeOriginTarget.isRunningJob = changedItem.isRunningJob; changeOriginTarget.notebookName = changedItem.notebookName; changeOriginTarget.notebookType = changedItem.notebookType; changeOriginTarget.unixTimeLastRun = changedItem.unixTimeLastRun; changeOriginTarget.paragraphs = changedItem.paragraphs; + console.log('new Event for Notebook job ', changedItem.notebookName); } $scope.doFiltering(jobInfomations, $scope.filterConfig); }); From 3878de3628e536671ef32d834bb68f6a6b727e44 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Thu, 2 Jun 2016 02:00:42 +0900 Subject: [PATCH 16/25] modifed test case and WebSocket server class extends structure changes --- .../apache/zeppelin/rest/NotebookRestApi.java | 28 +- .../zeppelin/server/ZeppelinServer.java | 54 +- .../apache/zeppelin/socket/AppMainServer.java | 46 +- .../zeppelin/socket/JobMangerServer.java | 1054 +---------------- .../zeppelin/socket/NotebookServer.java | 107 +- ...erverTest.java => NotebookServerTest.java} | 66 +- .../app/jobmanager/jobmanager.controller.js | 1 + .../src/app/notebook/notebook.controller.js | 2 +- .../components/navbar/navbar.controller.js | 2 +- .../websocketEvents.factory.js | 99 +- .../websocketEvents/websocketMsg.service.js | 67 +- .../zeppelin/notebook/socket/Message.java | 1 + 12 files changed, 183 insertions(+), 1344 deletions(-) rename zeppelin-server/src/test/java/org/apache/zeppelin/socket/{AppMainServerTest.java => NotebookServerTest.java} (86%) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 66be56aacf9..e887533a44c 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -45,7 +45,7 @@ import org.apache.zeppelin.rest.message.RunParagraphWithParametersRequest; import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.server.JsonResponse; -import org.apache.zeppelin.socket.AppMainServer; +import org.apache.zeppelin.socket.NotebookServer; import org.apache.zeppelin.utils.SecurityUtils; import org.quartz.CronExpression; import org.slf4j.Logger; @@ -64,15 +64,15 @@ public class NotebookRestApi { private static final Logger LOG = LoggerFactory.getLogger(NotebookRestApi.class); Gson gson = new Gson(); private Notebook notebook; - private AppMainServer appMainServer; + private NotebookServer notebookServer; private SearchService notebookIndex; private NotebookAuthorization notebookAuthorization; public NotebookRestApi() {} - public NotebookRestApi(Notebook notebook, AppMainServer appMainServer, SearchService search) { + public NotebookRestApi(Notebook notebook, NotebookServer notebookServer, SearchService search) { this.notebook = notebook; - this.appMainServer = appMainServer; + this.notebookServer = notebookServer; this.notebookIndex = search; this.notebookAuthorization = notebook.getNotebookAuthorization(); } @@ -155,7 +155,7 @@ public Response putNotePermissions(@PathParam("noteId") String noteId, String re notebookAuthorization.getReaders(noteId), notebookAuthorization.getWriters(noteId)); note.persist(); - appMainServer.broadcastNote(note); + notebookServer.broadcastNote(note); return new JsonResponse<>(Status.OK).build(); } @@ -217,7 +217,7 @@ public Response bind(@PathParam("noteId") String noteId) { @GET @Path("/") public Response getNotebookList() throws IOException { - List> notesInfo = appMainServer.generateNotebooksInfo(false); + List> notesInfo = notebookServer.generateNotebooksInfo(false); return new JsonResponse<>(Status.OK, "", notesInfo ).build(); } @@ -288,8 +288,8 @@ public Response createNote(String message) throws IOException { } note.setName(noteName); note.persist(); - appMainServer.broadcastNote(note); - appMainServer.broadcastNoteList(); + notebookServer.broadcastNote(note); + notebookServer.broadcastNoteList(); return new JsonResponse<>(Status.CREATED, "", note.getId() ).build(); } @@ -309,7 +309,7 @@ public Response deleteNote(@PathParam("notebookId") String notebookId) throws IO notebook.removeNote(notebookId); } } - appMainServer.broadcastNoteList(); + notebookServer.broadcastNoteList(); return new JsonResponse<>(Status.OK, "").build(); } @@ -328,8 +328,8 @@ public Response cloneNote(@PathParam("notebookId") String notebookId, String mes NewNotebookRequest.class); String newNoteName = request.getName(); Note newNote = notebook.cloneNote(notebookId, newNoteName); - appMainServer.broadcastNote(newNote); - appMainServer.broadcastNoteList(); + notebookServer.broadcastNote(newNote); + notebookServer.broadcastNoteList(); return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build(); } @@ -363,7 +363,7 @@ public Response insertParagraph(@PathParam("notebookId") String notebookId, Stri p.setText(request.getText()); note.persist(); - appMainServer.broadcastNote(note); + notebookServer.broadcastNote(note); return new JsonResponse(Status.CREATED, "", p.getId()).build(); } @@ -419,7 +419,7 @@ public Response moveParagraph(@PathParam("notebookId") String notebookId, note.moveParagraph(paragraphId, Integer.parseInt(newIndex), true); note.persist(); - appMainServer.broadcastNote(note); + notebookServer.broadcastNote(note); return new JsonResponse(Status.OK, "").build(); } catch (IndexOutOfBoundsException e) { LOG.error("Exception in NotebookRestApi while moveParagraph ", e); @@ -451,7 +451,7 @@ public Response deleteParagraph(@PathParam("notebookId") String notebookId, note.removeParagraph(paragraphId); note.persist(); - appMainServer.broadcastNote(note); + notebookServer.broadcastNote(note); return new JsonResponse(Status.OK, "").build(); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 9c24bbc1ac7..f554acaeeba 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -31,6 +31,9 @@ import org.apache.zeppelin.search.LuceneSearch; import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.socket.AppMainServer; +import org.apache.zeppelin.socket.JobMangerServer; +import org.apache.zeppelin.socket.NotebookServer; +import org.apache.zeppelin.socket.WebSocketServer; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.server.*; import org.eclipse.jetty.server.handler.ContextHandlerCollection; @@ -45,6 +48,7 @@ import org.slf4j.LoggerFactory; import javax.servlet.DispatcherType; +import javax.servlet.Servlet; import javax.ws.rs.core.Application; import java.io.File; import java.io.IOException; @@ -60,7 +64,7 @@ public class ZeppelinServer extends Application { public static Notebook notebook; public static Server jettyWebServer; - public static AppMainServer notebookWsServer; + public static NotebookServer notebookWsServer; private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; @@ -101,8 +105,16 @@ public static void main(String[] args) throws InterruptedException { // REST api setupRestApiContextHandler(webApp, conf); - // Notebook server - setupNotebookServer(webApp, conf); + // Main WS server + AppMainServer mainWsServer = setupWSAppMainServer(webApp, conf); + + // Notebook WS server + notebookWsServer = setupNotebookServer(webApp, conf); + mainWsServer.setSubWebSocketServer("notebookServer", notebookWsServer); + + // Job Manager WS server + WebSocketServer jobManagerServer = setupJobManagerServer(webApp, conf); + mainWsServer.setSubWebSocketServer("jobManagerServer", jobManagerServer); //Below is commented since zeppelin-docs module is removed. //final WebAppContext webAppSwagg = setupWebAppSwagger(conf); @@ -189,17 +201,41 @@ private static Server setupJettyServer(ZeppelinConfiguration conf) { return server; } - private static void setupNotebookServer(WebAppContext webapp, - ZeppelinConfiguration conf) { - notebookWsServer = new AppMainServer(); + private static AppMainServer setupWSAppMainServer(WebAppContext webapp, + ZeppelinConfiguration conf) { + AppMainServer appMainWsServer = new AppMainServer(); String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); - final ServletHolder servletHolder = new ServletHolder(notebookWsServer); + final ServletHolder servletHolder = new ServletHolder(appMainWsServer); servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); - final ServletContextHandler cxfContext = new ServletContextHandler( ServletContextHandler.SESSIONS); - webapp.addServlet(servletHolder, "/ws/*"); + + return appMainWsServer; + } + + private static NotebookServer setupNotebookServer(WebAppContext webapp, + ZeppelinConfiguration conf) { + NotebookServer notebookWsServer = new NotebookServer(); + String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); + final ServletHolder servletHolder = new ServletHolder(notebookWsServer); + servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); + final ServletContextHandler cxfContext = new ServletContextHandler( + ServletContextHandler.SESSIONS); + webapp.addServlet(servletHolder, "/notebook/*"); + return notebookWsServer; + } + + private static JobMangerServer setupJobManagerServer(WebAppContext webapp, + ZeppelinConfiguration conf) { + JobMangerServer jobMangerWsServer = new JobMangerServer(); + String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); + final ServletHolder servletHolder = new ServletHolder(jobMangerWsServer); + servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); + final ServletContextHandler cxfContext = new ServletContextHandler( + ServletContextHandler.SESSIONS); + webapp.addServlet(servletHolder, "/jobmanager/*"); + return jobMangerWsServer; } private static SslContextFactory getSslContextFactory(ZeppelinConfiguration conf) { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/AppMainServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/AppMainServer.java index eda45ab416d..60559d908b8 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/AppMainServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/AppMainServer.java @@ -46,9 +46,13 @@ public class AppMainServer extends WebSocketServlet implements private static final Logger LOG = LoggerFactory.getLogger(AppMainServer.class); Gson gson = new GsonBuilder() .setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); + + final Map subWebSocketServer = new HashMap<>(); + final Map> userWebSocketMap = new HashMap<>(); final Queue connectedSockets = new ConcurrentLinkedQueue<>(); + @Override public void configure(WebSocketServletFactory factory) { factory.setCreator(new WebAppSocketCreator(this)); @@ -69,6 +73,13 @@ public WebAppSocket doWebSocketConnect(HttpServletRequest req, String protocol) return new WebAppSocket(req, protocol, this); } + public void setSubWebSocketServer(String key, WebSocketServer server) { + synchronized (subWebSocketServer) { + subWebSocketServer.remove(key); + subWebSocketServer.put(key, server); + } + } + @Override public void onOpen(WebAppSocket conn) { LOG.info("New connection from {} : {}", conn.getRequest().getRemoteAddr(), @@ -88,28 +99,15 @@ public void onMessage(WebAppSocket conn, String msg) { if (LOG.isTraceEnabled()) { LOG.trace("RECEIVE MSG = " + messagereceived); } - - String ticket = TicketContainer.instance.getTicket(messagereceived.principal); - if (ticket != null && !ticket.equals(messagereceived.ticket)) - throw new Exception("Invalid ticket " + messagereceived.ticket + " != " + ticket); - - ZeppelinConfiguration conf = ZeppelinConfiguration.create(); - boolean allowAnonymous = conf. - getBoolean(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED); - if (!allowAnonymous && messagereceived.principal.equals("anonymous")) { - throw new Exception("Anonymous access not allowed "); - } - HashSet userAndRoles = new HashSet(); - userAndRoles.add(messagereceived.principal); - if (!messagereceived.roles.equals("")) { - HashSet roles = gson.fromJson(messagereceived.roles, - new TypeToken>(){}.getType()); - if (roles != null) { - userAndRoles.addAll(roles); - } + WebSocketServer processServer = subWebSocketServer.get(messagereceived.target); + if (processServer != null) { + LOG.info("target {} received.", messagereceived.target); + processServer.onMessage(conn, msg); + } else { + LOG.info("target {} received.", AppMainServer.class.toString()); } - LOG.info("lcs main server received"); + /** Lets be elegant here */ switch (messagereceived.op) { case PING: @@ -192,7 +190,7 @@ protected String getOpenKey(WebAppSocket socket) { return key; } - private void broadcast(String key, Message m) { + protected void broadcast(String key, Message m) { synchronized (userWebSocketMap) { List socketLists = userWebSocketMap.get(key); if (socketLists == null || socketLists.size() == 0) { @@ -209,7 +207,7 @@ private void broadcast(String key, Message m) { } } - private void broadcastExcept(String key, Message m, WebAppSocket exclude) { + protected void broadcastExcept(String key, Message m, WebAppSocket exclude) { synchronized (userWebSocketMap) { List socketLists = userWebSocketMap.get(key); if (socketLists == null || socketLists.size() == 0) { @@ -229,7 +227,7 @@ private void broadcastExcept(String key, Message m, WebAppSocket exclude) { } } - private void broadcastAll(Message m) { + protected void broadcastAll(Message m) { for (WebAppSocket conn : connectedSockets) { try { conn.send(serializeMessage(m)); @@ -239,7 +237,7 @@ private void broadcastAll(Message m) { } } - private void unicast(Message m, WebAppSocket conn) { + protected void unicast(Message m, WebAppSocket conn) { try { conn.send(serializeMessage(m)); } catch (IOException e) { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java index 015946dde4d..7391b6501f1 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java @@ -58,42 +58,11 @@ */ public class JobMangerServer extends AppMainServer implements WebSocketServer { private static final Logger LOG = LoggerFactory.getLogger(JobMangerServer.class); - Gson gson = new GsonBuilder() - .setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); - final Map> noteSocketMap = new HashMap<>(); - final Queue connectedSockets = new ConcurrentLinkedQueue<>(); private Notebook notebook() { return ZeppelinServer.notebook; } - @Override - public void configure(WebSocketServletFactory factory) { - factory.setCreator(new WebAppSocketCreator(this)); - } - - public boolean checkOrigin(HttpServletRequest request, String origin) { - try { - return SecurityUtils.isValidOrigin(origin, ZeppelinConfiguration.create()); - } catch (UnknownHostException e) { - LOG.error(e.toString(), e); - } catch (URISyntaxException e) { - LOG.error(e.toString(), e); - } - return false; - } - - public WebAppSocket doWebSocketConnect(HttpServletRequest req, String protocol) { - return new WebAppSocket(req, protocol, this); - } - - @Override - public void onOpen(WebAppSocket conn) { - LOG.info("New connection from {} : {}", conn.getRequest().getRemoteAddr(), - conn.getRequest().getRemotePort()); - connectedSockets.add(conn); - } - @Override public void onMessage(WebAppSocket conn, String msg) { Notebook notebook = notebook(); @@ -145,14 +114,6 @@ public void onMessage(WebAppSocket conn, String msg) { } } - @Override - public void onClose(WebAppSocket conn, int code, String reason) { - LOG.info("Closed connection to {} : {}. ({}) {}", conn.getRequest() - .getRemoteAddr(), conn.getRequest().getRemotePort(), code, reason); - removeConnectionFromAllNote(conn); - connectedSockets.remove(conn); - } - protected Message deserializeMessage(String msg) { return gson.fromJson(msg, Message.class); } @@ -173,6 +134,7 @@ public void unicastNotebookJobInfo(WebAppSocket conn) { } public void unicastUpdateNotebookJobInfo(WebAppSocket conn, Message fromMessage) { + LOG.info("update time {}", fromMessage); double lastUpdateUnixTimeRaw = (double) fromMessage.get("lastUpdateUnixTime"); long lastUpdateUnixTime = new Double(lastUpdateUnixTimeRaw).longValue(); List> notebookJobs; @@ -421,1019 +383,5 @@ public List> generateUpdateNotebooksJobInfo( return notesInfo; } - private void addConnectionToNote(String noteId, WebAppSocket socket) { - synchronized (noteSocketMap) { - removeConnectionFromAllNote(socket); // make sure a socket relates only a - // single note. - List socketList = noteSocketMap.get(noteId); - if (socketList == null) { - socketList = new LinkedList<>(); - noteSocketMap.put(noteId, socketList); - } - if (!socketList.contains(socket)) { - socketList.add(socket); - } - } - } - - private void removeConnectionFromNote(String noteId, WebAppSocket socket) { - synchronized (noteSocketMap) { - List socketList = noteSocketMap.get(noteId); - if (socketList != null) { - socketList.remove(socket); - } - } - } - - private void removeNote(String noteId) { - synchronized (noteSocketMap) { - List socketList = noteSocketMap.remove(noteId); - } - } - - private void removeConnectionFromAllNote(WebAppSocket socket) { - synchronized (noteSocketMap) { - Set keys = noteSocketMap.keySet(); - for (String noteId : keys) { - removeConnectionFromNote(noteId, socket); - } - } - } - - private String getOpenNoteId(WebAppSocket socket) { - String id = null; - synchronized (noteSocketMap) { - Set keys = noteSocketMap.keySet(); - for (String noteId : keys) { - List sockets = noteSocketMap.get(noteId); - if (sockets.contains(socket)) { - id = noteId; - } - } - } - - return id; - } - - private void broadcastToNoteBindedInterpreter(String interpreterGroupId, - Message m) { - Notebook notebook = notebook(); - List notes = notebook.getAllNotes(); - for (Note note : notes) { - List ids = note.getNoteReplLoader().getInterpreters(); - for (String id : ids) { - if (id.equals(interpreterGroupId)) { - broadcast(note.id(), m); - } - } - } - } - - private void broadcast(String noteId, Message m) { - synchronized (noteSocketMap) { - List socketLists = noteSocketMap.get(noteId); - if (socketLists == null || socketLists.size() == 0) { - return; - } - LOG.debug("SEND >> " + m.op); - for (WebAppSocket conn : socketLists) { - try { - conn.send(serializeMessage(m)); - } catch (IOException e) { - LOG.error("socket error", e); - } - } - } - } - - private void broadcastExcept(String noteId, Message m, WebAppSocket exclude) { - synchronized (noteSocketMap) { - List socketLists = noteSocketMap.get(noteId); - if (socketLists == null || socketLists.size() == 0) { - return; - } - LOG.debug("SEND >> " + m.op); - for (WebAppSocket conn : socketLists) { - if (exclude.equals(conn)) { - continue; - } - try { - conn.send(serializeMessage(m)); - } catch (IOException e) { - LOG.error("socket error", e); - } - } - } - } - - private void broadcastAll(Message m) { - for (WebAppSocket conn : connectedSockets) { - try { - conn.send(serializeMessage(m)); - } catch (IOException e) { - LOG.error("socket error", e); - } - } - } - - private void unicast(Message m, WebAppSocket conn) { - try { - conn.send(serializeMessage(m)); - } catch (IOException e) { - LOG.error("socket error", e); - } - } - - - public List> generateNotebooksInfo(boolean needsReload) { - Notebook notebook = notebook(); - - ZeppelinConfiguration conf = notebook.getConf(); - String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); - boolean hideHomeScreenNotebookFromList = conf - .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); - - if (needsReload) { - try { - notebook.reloadAllNotes(); - } catch (IOException e) { - LOG.error("Fail to reload notes from repository", e); - } - } - - List notes = notebook.getAllNotes(); - List> notesInfo = new LinkedList<>(); - for (Note note : notes) { - Map info = new HashMap<>(); - - if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { - continue; - } - - info.put("id", note.id()); - info.put("name", note.getName()); - notesInfo.add(info); - } - - return notesInfo; - } - - public void broadcastNote(Note note) { - broadcast(note.id(), new Message(OP.NOTE).put("note", note)); - } - - public void broadcastNoteList() { - List> notesInfo = generateNotebooksInfo(false); - broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo)); - } - - public void unicastNoteList(WebAppSocket conn) { - List> notesInfo = generateNotebooksInfo(false); - unicast(new Message(OP.NOTES_INFO).put("notes", notesInfo), conn); - } - - public void broadcastReloadedNoteList() { - List> notesInfo = generateNotebooksInfo(true); - broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo)); - } - - void permissionError(WebAppSocket conn, String op, Set userAndRoles, - Set allowed) throws IOException { - LOG.info("Cannot {}. Connection readers {}. Allowed readers {}", - op, userAndRoles, allowed); - - String userName = userAndRoles.iterator().next(); - - conn.send(serializeMessage(new Message(OP.AUTH_INFO).put("info", - "Insufficient privileges to " + op + " notebook.\n\n" + - "Allowed users or roles: " + allowed.toString() + "\n\n" + - "But the user " + userName + " belongs to: " + userAndRoles.toString()))); - } - - private void sendNote(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, - Message fromMessage) throws IOException { - - LOG.info("New operation from {} : {} : {} : {} : {}", conn.getRequest().getRemoteAddr(), - conn.getRequest().getRemotePort(), - fromMessage.principal, fromMessage.op, fromMessage.get("id") - ); - - String noteId = (String) fromMessage.get("id"); - if (noteId == null) { - return; - } - - Note note = notebook.getNote(noteId); - NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); - if (note != null) { - if (!notebookAuthorization.isReader(noteId, userAndRoles)) { - permissionError(conn, "read", userAndRoles, notebookAuthorization.getReaders(noteId)); - return; - } - addConnectionToNote(note.id(), conn); - conn.send(serializeMessage(new Message(OP.NOTE).put("note", note))); - sendAllAngularObjects(note, conn); - } - } - - private void sendHomeNote(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook) throws IOException { - String noteId = notebook.getConf().getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); - - Note note = null; - if (noteId != null) { - note = notebook.getNote(noteId); - } - - if (note != null) { - NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); - if (!notebookAuthorization.isReader(noteId, userAndRoles)) { - permissionError(conn, "read", userAndRoles, notebookAuthorization.getReaders(noteId)); - return; - } - addConnectionToNote(note.id(), conn); - conn.send(serializeMessage(new Message(OP.NOTE).put("note", note))); - sendAllAngularObjects(note, conn); - } else { - removeConnectionFromAllNote(conn); - conn.send(serializeMessage(new Message(OP.NOTE).put("note", null))); - } - } - - private void updateNote(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook, Message fromMessage) - throws SchedulerException, IOException { - String noteId = (String) fromMessage.get("id"); - String name = (String) fromMessage.get("name"); - Map config = (Map) fromMessage - .get("config"); - if (noteId == null) { - return; - } - if (config == null) { - return; - } - - NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); - if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { - permissionError(conn, "update", userAndRoles, notebookAuthorization.getWriters(noteId)); - return; - } - - Note note = notebook.getNote(noteId); - if (note != null) { - boolean cronUpdated = isCronUpdated(config, note.getConfig()); - note.setName(name); - note.setConfig(config); - if (cronUpdated) { - notebook.refreshCron(note.id()); - } - - note.persist(); - broadcastNote(note); - broadcastNoteList(); - } - } - - private boolean isCronUpdated(Map configA, - Map configB) { - boolean cronUpdated = false; - if (configA.get("cron") != null && configB.get("cron") != null - && configA.get("cron").equals(configB.get("cron"))) { - cronUpdated = true; - } else if (configA.get("cron") == null && configB.get("cron") == null) { - cronUpdated = false; - } else if (configA.get("cron") != null || configB.get("cron") != null) { - cronUpdated = true; - } - - return cronUpdated; - } - private void createNote(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook, Message message) - throws IOException { - Note note = notebook.createNote(); - note.addParagraph(); // it's an empty note. so add one paragraph - if (message != null) { - String noteName = (String) message.get("name"); - if (noteName == null || noteName.isEmpty()){ - noteName = "Note " + note.getId(); - } - note.setName(noteName); - } - - note.persist(); - addConnectionToNote(note.id(), (WebAppSocket) conn); - conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note))); - broadcastNoteList(); - } - - private void removeNote(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook, Message fromMessage) - throws IOException { - String noteId = (String) fromMessage.get("id"); - if (noteId == null) { - return; - } - - Note note = notebook.getNote(noteId); - NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); - if (!notebookAuthorization.isOwner(noteId, userAndRoles)) { - permissionError(conn, "remove", userAndRoles, notebookAuthorization.getOwners(noteId)); - return; - } - - notebook.removeNote(noteId); - removeNote(noteId); - broadcastNoteList(); - } - - private void updateParagraph(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook, Message fromMessage) throws IOException { - String paragraphId = (String) fromMessage.get("id"); - if (paragraphId == null) { - return; - } - - Map params = (Map) fromMessage - .get("params"); - Map config = (Map) fromMessage - .get("config"); - String noteId = getOpenNoteId(conn); - final Note note = notebook.getNote(noteId); - NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); - if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { - permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); - return; - } - - Paragraph p = note.getParagraph(paragraphId); - p.settings.setParams(params); - p.setConfig(config); - p.setTitle((String) fromMessage.get("title")); - p.setText((String) fromMessage.get("paragraph")); - note.persist(); - broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p)); - } - - private void cloneNote(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook, Message fromMessage) - throws IOException, CloneNotSupportedException { - String noteId = getOpenNoteId(conn); - String name = (String) fromMessage.get("name"); - Note newNote = notebook.cloneNote(noteId, name); - addConnectionToNote(newNote.id(), (WebAppSocket) conn); - conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", newNote))); - broadcastNoteList(); - } - - protected Note importNote(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook, Message fromMessage) - throws IOException { - Note note = null; - if (fromMessage != null) { - String noteName = (String) ((Map) fromMessage.get("notebook")).get("name"); - String noteJson = gson.toJson(fromMessage.get("notebook")); - note = notebook.importNote(noteJson, noteName); - note.persist(); - broadcastNote(note); - broadcastNoteList(); - } - return note; - } - - private void removeParagraph(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook, Message fromMessage) throws IOException { - final String paragraphId = (String) fromMessage.get("id"); - if (paragraphId == null) { - return; - } - String noteId = getOpenNoteId(conn); - final Note note = notebook.getNote(noteId); - NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); - if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { - permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); - return; - } - - /** We dont want to remove the last paragraph */ - if (!note.isLastParagraph(paragraphId)) { - note.removeParagraph(paragraphId); - note.persist(); - broadcastNote(note); - } - } - - private void clearParagraphOutput(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook, Message fromMessage) throws IOException { - final String paragraphId = (String) fromMessage.get("id"); - if (paragraphId == null) { - return; - } - String noteId = getOpenNoteId(conn); - final Note note = notebook.getNote(noteId); - NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); - if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { - permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); - return; - } - - note.clearParagraphOutput(paragraphId); - broadcastNote(note); - } - - private void completion(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, - Message fromMessage) throws IOException { - String paragraphId = (String) fromMessage.get("id"); - String buffer = (String) fromMessage.get("buf"); - int cursor = (int) Double.parseDouble(fromMessage.get("cursor").toString()); - Message resp = new Message(OP.COMPLETION_LIST).put("id", paragraphId); - if (paragraphId == null) { - conn.send(serializeMessage(resp)); - return; - } - - final Note note = notebook.getNote(getOpenNoteId(conn)); - List candidates = note.completion(paragraphId, buffer, cursor); - resp.put("completions", candidates); - conn.send(serializeMessage(resp)); - } - - /** - * When angular object updated from client - * - * @param conn the web socket. - * @param notebook the notebook. - * @param fromMessage the message. - */ - private void angularObjectUpdated(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook, Message fromMessage) { - String noteId = (String) fromMessage.get("noteId"); - String paragraphId = (String) fromMessage.get("paragraphId"); - String interpreterGroupId = (String) fromMessage.get("interpreterGroupId"); - String varName = (String) fromMessage.get("name"); - Object varValue = fromMessage.get("value"); - AngularObject ao = null; - boolean global = false; - // propagate change to (Remote) AngularObjectRegistry - Note note = notebook.getNote(noteId); - if (note != null) { - List settings = note.getNoteReplLoader() - .getInterpreterSettings(); - for (InterpreterSetting setting : settings) { - if (setting.getInterpreterGroup(note.id()) == null) { - continue; - } - if (interpreterGroupId.equals(setting.getInterpreterGroup(note.id()).getId())) { - AngularObjectRegistry angularObjectRegistry = setting - .getInterpreterGroup(note.id()).getAngularObjectRegistry(); - // first trying to get local registry - ao = angularObjectRegistry.get(varName, noteId, paragraphId); - if (ao == null) { - // then try notebook scope registry - ao = angularObjectRegistry.get(varName, noteId, null); - if (ao == null) { - // then try global scope registry - ao = angularObjectRegistry.get(varName, null, null); - if (ao == null) { - LOG.warn("Object {} is not binded", varName); - } else { - // path from client -> server - ao.set(varValue, false); - global = true; - } - } else { - // path from client -> server - ao.set(varValue, false); - global = false; - } - } else { - ao.set(varValue, false); - global = false; - } - break; - } - } - } - - if (global) { // broadcast change to all web session that uses related - // interpreter. - for (Note n : notebook.getAllNotes()) { - List settings = note.getNoteReplLoader() - .getInterpreterSettings(); - for (InterpreterSetting setting : settings) { - if (setting.getInterpreterGroup(n.id()) == null) { - continue; - } - if (interpreterGroupId.equals(setting.getInterpreterGroup(n.id()).getId())) { - AngularObjectRegistry angularObjectRegistry = setting - .getInterpreterGroup(n.id()).getAngularObjectRegistry(); - this.broadcastExcept( - n.id(), - new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao) - .put("interpreterGroupId", interpreterGroupId) - .put("noteId", n.id()) - .put("paragraphId", ao.getParagraphId()), - conn); - } - } - } - } else { // broadcast to all web session for the note - this.broadcastExcept( - note.id(), - new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao) - .put("interpreterGroupId", interpreterGroupId) - .put("noteId", note.id()) - .put("paragraphId", ao.getParagraphId()), - conn); - } - } - - /** - * Push the given Angular variable to the target - * interpreter angular registry given a noteId - * and a paragraph id - * @param conn - * @param notebook - * @param fromMessage - * @throws Exception - */ - protected void angularObjectClientBind(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook, Message fromMessage) - throws Exception { - String noteId = fromMessage.getType("noteId"); - String varName = fromMessage.getType("name"); - Object varValue = fromMessage.get("value"); - String paragraphId = fromMessage.getType("paragraphId"); - Note note = notebook.getNote(noteId); - - if (paragraphId == null) { - throw new IllegalArgumentException("target paragraph not specified for " + - "angular value bind"); - } - - if (note != null) { - final InterpreterGroup interpreterGroup = findInterpreterGroupForParagraph(note, - paragraphId); - - final AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry(); - if (registry instanceof RemoteAngularObjectRegistry) { - - RemoteAngularObjectRegistry remoteRegistry = (RemoteAngularObjectRegistry) registry; - pushAngularObjectToRemoteRegistry(noteId, paragraphId, varName, varValue, remoteRegistry, - interpreterGroup.getId(), conn); - - } else { - pushAngularObjectToLocalRepo(noteId, paragraphId, varName, varValue, registry, - interpreterGroup.getId(), conn); - } - } - } - - /** - * Remove the given Angular variable to the target - * interpreter(s) angular registry given a noteId - * and an optional list of paragraph id(s) - * @param conn - * @param notebook - * @param fromMessage - * @throws Exception - */ - protected void angularObjectClientUnbind(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook, Message fromMessage) - throws Exception{ - String noteId = fromMessage.getType("noteId"); - String varName = fromMessage.getType("name"); - String paragraphId = fromMessage.getType("paragraphId"); - Note note = notebook.getNote(noteId); - - if (paragraphId == null) { - throw new IllegalArgumentException("target paragraph not specified for " + - "angular value unBind"); - } - - if (note != null) { - final InterpreterGroup interpreterGroup = findInterpreterGroupForParagraph(note, - paragraphId); - - final AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry(); - - if (registry instanceof RemoteAngularObjectRegistry) { - RemoteAngularObjectRegistry remoteRegistry = (RemoteAngularObjectRegistry) registry; - removeAngularFromRemoteRegistry(noteId, paragraphId, varName, remoteRegistry, - interpreterGroup.getId(), conn); - } else { - removeAngularObjectFromLocalRepo(noteId, paragraphId, varName, registry, - interpreterGroup.getId(), conn); - } - } - } - - private InterpreterGroup findInterpreterGroupForParagraph(Note note, String paragraphId) - throws Exception { - final Paragraph paragraph = note.getParagraph(paragraphId); - if (paragraph == null) { - throw new IllegalArgumentException("Unknown paragraph with id : " + paragraphId); - } - return paragraph.getCurrentRepl().getInterpreterGroup(); - } - - private void pushAngularObjectToRemoteRegistry(String noteId, String paragraphId, - String varName, Object varValue, RemoteAngularObjectRegistry remoteRegistry, - String interpreterGroupId, WebAppSocket conn) { - - final AngularObject ao = remoteRegistry.addAndNotifyRemoteProcess(varName, varValue, - noteId, paragraphId); - - this.broadcastExcept( - noteId, - new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao) - .put("interpreterGroupId", interpreterGroupId) - .put("noteId", noteId) - .put("paragraphId", paragraphId), - conn); - } - - private void removeAngularFromRemoteRegistry(String noteId, String paragraphId, - String varName, RemoteAngularObjectRegistry remoteRegistry, - String interpreterGroupId, WebAppSocket conn) { - final AngularObject ao = remoteRegistry.removeAndNotifyRemoteProcess(varName, noteId, - paragraphId); - this.broadcastExcept( - noteId, - new Message(OP.ANGULAR_OBJECT_REMOVE).put("angularObject", ao) - .put("interpreterGroupId", interpreterGroupId) - .put("noteId", noteId) - .put("paragraphId", paragraphId), - conn); - } - - private void pushAngularObjectToLocalRepo(String noteId, String paragraphId, String varName, - Object varValue, AngularObjectRegistry registry, - String interpreterGroupId, WebAppSocket conn) { - AngularObject angularObject = registry.get(varName, noteId, paragraphId); - if (angularObject == null) { - angularObject = registry.add(varName, varValue, noteId, paragraphId); - } else { - angularObject.set(varValue, true); - } - - this.broadcastExcept( - noteId, - new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", angularObject) - .put("interpreterGroupId", interpreterGroupId) - .put("noteId", noteId) - .put("paragraphId", paragraphId), - conn); - } - - private void removeAngularObjectFromLocalRepo(String noteId, String paragraphId, String varName, - AngularObjectRegistry registry, String interpreterGroupId, WebAppSocket conn) { - final AngularObject removed = registry.remove(varName, noteId, paragraphId); - if (removed != null) { - this.broadcastExcept( - noteId, - new Message(OP.ANGULAR_OBJECT_REMOVE).put("angularObject", removed) - .put("interpreterGroupId", interpreterGroupId) - .put("noteId", noteId) - .put("paragraphId", paragraphId), - conn); - } - } - - private void moveParagraph(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, - Message fromMessage) throws IOException { - final String paragraphId = (String) fromMessage.get("id"); - if (paragraphId == null) { - return; - } - - final int newIndex = (int) Double.parseDouble(fromMessage.get("index") - .toString()); - String noteId = getOpenNoteId(conn); - final Note note = notebook.getNote(noteId); - NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); - if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { - permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); - return; - } - - note.moveParagraph(paragraphId, newIndex); - note.persist(); - broadcastNote(note); - } - - private void insertParagraph(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook, Message fromMessage) throws IOException { - final int index = (int) Double.parseDouble(fromMessage.get("index") - .toString()); - String noteId = getOpenNoteId(conn); - final Note note = notebook.getNote(noteId); - NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); - if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { - permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); - return; - } - - note.insertParagraph(index); - note.persist(); - broadcastNote(note); - } - - private void cancelParagraph(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, - Message fromMessage) throws IOException { - final String paragraphId = (String) fromMessage.get("id"); - if (paragraphId == null) { - return; - } - - String noteId = getOpenNoteId(conn); - final Note note = notebook.getNote(noteId); - NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); - if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { - permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); - return; - } - - Paragraph p = note.getParagraph(paragraphId); - p.abort(); - } - - private void runParagraph(WebAppSocket conn, HashSet userAndRoles, Notebook notebook, - Message fromMessage) throws IOException { - final String paragraphId = (String) fromMessage.get("id"); - if (paragraphId == null) { - return; - } - - String noteId = getOpenNoteId(conn); - final Note note = notebook.getNote(noteId); - NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization(); - if (!notebookAuthorization.isWriter(noteId, userAndRoles)) { - permissionError(conn, "write", userAndRoles, notebookAuthorization.getWriters(noteId)); - return; - } - - Paragraph p = note.getParagraph(paragraphId); - String text = (String) fromMessage.get("paragraph"); - p.setText(text); - p.setTitle((String) fromMessage.get("title")); - if (!fromMessage.principal.equals("anonymous")) { - AuthenticationInfo authenticationInfo = new AuthenticationInfo(fromMessage.principal, - fromMessage.ticket); - p.setAuthenticationInfo(authenticationInfo); - - } else { - p.setAuthenticationInfo(new AuthenticationInfo()); - } - - Map params = (Map) fromMessage - .get("params"); - p.settings.setParams(params); - Map config = (Map) fromMessage - .get("config"); - p.setConfig(config); - // if it's the last paragraph, let's add a new one - boolean isTheLastParagraph = note.getLastParagraph().getId() - .equals(p.getId()); - if (!Strings.isNullOrEmpty(text) && isTheLastParagraph) { - note.addParagraph(); - } - - note.persist(); - try { - note.run(paragraphId); - } catch (Exception ex) { - LOG.error("Exception from run", ex); - if (p != null) { - p.setReturn( - new InterpreterResult(InterpreterResult.Code.ERROR, ex.getMessage()), - ex); - p.setStatus(Status.ERROR); - broadcast(note.id(), new Message(OP.PARAGRAPH).put("paragraph", p)); - } - } - } - - private void sendAllConfigurations(WebAppSocket conn, HashSet userAndRoles, - Notebook notebook) throws IOException { - ZeppelinConfiguration conf = notebook.getConf(); - - Map configurations = conf.dumpConfigurations(conf, - new ZeppelinConfiguration.ConfigurationKeyPredicate() { - @Override - public boolean apply(String key) { - return !key.contains("password") && - !key.equals( - ConfVars - .ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING - .getVarName()); - } - }); - - conn.send(serializeMessage(new Message(OP.CONFIGURATIONS_INFO) - .put("configurations", configurations))); - } - - private void checkpointNotebook(WebAppSocket conn, Notebook notebook, - Message fromMessage) throws IOException { - String noteId = (String) fromMessage.get("noteId"); - String commitMessage = (String) fromMessage.get("commitMessage"); - notebook.checkpointNote(noteId, commitMessage); - } - - /** - * This callback is for the paragraph that runs on ZeppelinServer - * @param noteId - * @param paragraphId - * @param output output to append - */ - @Override - public void onOutputAppend(String noteId, String paragraphId, String output) { - Message msg = new Message(OP.PARAGRAPH_APPEND_OUTPUT) - .put("noteId", noteId) - .put("paragraphId", paragraphId) - .put("data", output); - Paragraph paragraph = notebook().getNote(noteId).getParagraph(paragraphId); - broadcast(noteId, msg); - } - - /** - * This callback is for the paragraph that runs on ZeppelinServer - * @param noteId - * @param paragraphId - * @param output output to update (replace) - */ - @Override - public void onOutputUpdated(String noteId, String paragraphId, String output) { - Message msg = new Message(OP.PARAGRAPH_UPDATE_OUTPUT) - .put("noteId", noteId) - .put("paragraphId", paragraphId) - .put("data", output); - Paragraph paragraph = notebook().getNote(noteId).getParagraph(paragraphId); - broadcast(noteId, msg); - } - - /** - * Need description here. - * - */ - public static class ParagraphListenerImpl implements ParagraphJobListener { - private JobMangerServer appMainServer; - private Note note; - - public ParagraphListenerImpl(JobMangerServer appMainServer, Note note) { - this.appMainServer = appMainServer; - this.note = note; - } - - @Override - public void onProgressUpdate(Job job, int progress) { - appMainServer.broadcast( - note.id(), - new Message(OP.PROGRESS).put("id", job.getId()).put("progress", - job.progress())); - } - - @Override - public void beforeStatusChange(Job job, Status before, Status after) { - } - - @Override - public void afterStatusChange(Job job, Status before, Status after) { - if (after == Status.ERROR) { - if (job.getException() != null) { - LOG.error("Error", job.getException()); - } - } - - if (job.isTerminated()) { - LOG.info("Job {} is finished", job.getId()); - try { - note.persist(); - } catch (IOException e) { - LOG.error(e.toString(), e); - } - } - appMainServer.broadcastNote(note); - } - - /** - * This callback is for praragraph that runs on RemoteInterpreterProcess - * @param paragraph - * @param out - * @param output - */ - @Override - public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String output) { - Message msg = new Message(OP.PARAGRAPH_APPEND_OUTPUT) - .put("noteId", paragraph.getNote().getId()) - .put("paragraphId", paragraph.getId()) - .put("data", output); - - appMainServer.broadcast(paragraph.getNote().getId(), msg); - } - - /** - * This callback is for paragraph that runs on RemoteInterpreterProcess - * @param paragraph - * @param out - * @param output - */ - @Override - public void onOutputUpdate(Paragraph paragraph, InterpreterOutput out, String output) { - Message msg = new Message(OP.PARAGRAPH_UPDATE_OUTPUT) - .put("noteId", paragraph.getNote().getId()) - .put("paragraphId", paragraph.getId()) - .put("data", output); - - appMainServer.broadcast(paragraph.getNote().getId(), msg); - } - } - - @Override - public ParagraphJobListener getParagraphJobListener(Note note) { - return new ParagraphListenerImpl(this, note); - } - - private void sendAllAngularObjects(Note note, WebAppSocket conn) throws IOException { - List settings = note.getNoteReplLoader() - .getInterpreterSettings(); - if (settings == null || settings.size() == 0) { - return; - } - - for (InterpreterSetting intpSetting : settings) { - AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.id()) - .getAngularObjectRegistry(); - List objects = registry.getAllWithGlobal(note.id()); - for (AngularObject object : objects) { - conn.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE) - .put("angularObject", object) - .put("interpreterGroupId", - intpSetting.getInterpreterGroup(note.id()).getId()) - .put("noteId", note.id()) - .put("paragraphId", object.getParagraphId()) - )); - } - } - } - - @Override - public void onAdd(String interpreterGroupId, AngularObject object) { - onUpdate(interpreterGroupId, object); - } - - @Override - public void onUpdate(String interpreterGroupId, AngularObject object) { - Notebook notebook = notebook(); - if (notebook == null) { - return; - } - - List notes = notebook.getAllNotes(); - for (Note note : notes) { - if (object.getNoteId() != null && !note.id().equals(object.getNoteId())) { - continue; - } - - List intpSettings = note.getNoteReplLoader() - .getInterpreterSettings(); - if (intpSettings.isEmpty()) - continue; - for (InterpreterSetting setting : intpSettings) { - if (setting.getInterpreterGroup(note.id()).getId().equals(interpreterGroupId)) { - broadcast( - note.id(), - new Message(OP.ANGULAR_OBJECT_UPDATE) - .put("angularObject", object) - .put("interpreterGroupId", interpreterGroupId) - .put("noteId", note.id()) - .put("paragraphId", object.getParagraphId())); - } - } - } - } - - @Override - public void onRemove(String interpreterGroupId, String name, String noteId, String paragraphId) { - Notebook notebook = notebook(); - List notes = notebook.getAllNotes(); - for (Note note : notes) { - if (noteId != null && !note.id().equals(noteId)) { - continue; - } - - List ids = note.getNoteReplLoader().getInterpreters(); - for (String id : ids) { - if (id.equals(interpreterGroupId)) { - broadcast( - note.id(), - new Message(OP.ANGULAR_OBJECT_REMOVE).put("name", name).put( - "noteId", noteId).put("paragraphId", paragraphId)); - } - } - } - } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 9c16285009e..151249083d7 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -59,41 +59,13 @@ public class NotebookServer extends AppMainServer implements JobListenerFactory, AngularObjectRegistryListener, RemoteInterpreterProcessListener { private static final Logger LOG = LoggerFactory.getLogger(NotebookServer.class); - Gson gson = new GsonBuilder() - .setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); + final Queue connectedSockets = new ConcurrentLinkedQueue<>(); private Notebook notebook() { return ZeppelinServer.notebook; } - @Override - public void configure(WebSocketServletFactory factory) { - factory.setCreator(new WebAppSocketCreator(this)); - } - - public boolean checkOrigin(HttpServletRequest request, String origin) { - try { - return SecurityUtils.isValidOrigin(origin, ZeppelinConfiguration.create()); - } catch (UnknownHostException e) { - LOG.error(e.toString(), e); - } catch (URISyntaxException e) { - LOG.error(e.toString(), e); - } - return false; - } - - public WebAppSocket doWebSocketConnect(HttpServletRequest req, String protocol) { - return new WebAppSocket(req, protocol, this); - } - - @Override - public void onOpen(WebAppSocket conn) { - LOG.info("New connection from {} : {}", conn.getRequest().getRemoteAddr(), - conn.getRequest().getRemotePort()); - connectedSockets.add(conn); - } - @Override public void onMessage(WebAppSocket conn, String msg) { Notebook notebook = notebook(); @@ -207,22 +179,6 @@ public void onMessage(WebAppSocket conn, String msg) { } } - @Override - public void onClose(WebAppSocket conn, int code, String reason) { - LOG.info("Closed connection to {} : {}. ({}) {}", conn.getRequest() - .getRemoteAddr(), conn.getRequest().getRemotePort(), code, reason); - removeConnectionFromAllNote(conn); - connectedSockets.remove(conn); - } - - protected Message deserializeMessage(String msg) { - return gson.fromJson(msg, Message.class); - } - - protected String serializeMessage(Message m) { - return gson.toJson(m); - } - private void addConnectionToNote(String noteId, WebAppSocket socket) { addConnectionToKey(noteId, socket); } @@ -242,9 +198,8 @@ private void removeConnectionFromAllNote(WebAppSocket socket) { private String getOpenNoteId(WebAppSocket socket) { return getOpenKey(socket); } -//lcs 여기까지 - private void broadcastToNoteBindedInterpreter(String interpreterGroupId, - Message m) { + + private void broadcastToNoteBindedInterpreter(String interpreterGroupId, Message m) { Notebook notebook = notebook(); List notes = notebook.getAllNotes(); for (Note note : notes) { @@ -257,62 +212,6 @@ private void broadcastToNoteBindedInterpreter(String interpreterGroupId, } } - private void broadcast(String noteId, Message m) { - synchronized (noteSocketMap) { - List socketLists = noteSocketMap.get(noteId); - if (socketLists == null || socketLists.size() == 0) { - return; - } - LOG.debug("SEND >> " + m.op); - for (WebAppSocket conn : socketLists) { - try { - conn.send(serializeMessage(m)); - } catch (IOException e) { - LOG.error("socket error", e); - } - } - } - } - - private void broadcastExcept(String noteId, Message m, WebAppSocket exclude) { - synchronized (noteSocketMap) { - List socketLists = noteSocketMap.get(noteId); - if (socketLists == null || socketLists.size() == 0) { - return; - } - LOG.debug("SEND >> " + m.op); - for (WebAppSocket conn : socketLists) { - if (exclude.equals(conn)) { - continue; - } - try { - conn.send(serializeMessage(m)); - } catch (IOException e) { - LOG.error("socket error", e); - } - } - } - } - - private void broadcastAll(Message m) { - for (WebAppSocket conn : connectedSockets) { - try { - conn.send(serializeMessage(m)); - } catch (IOException e) { - LOG.error("socket error", e); - } - } - } - - private void unicast(Message m, WebAppSocket conn) { - try { - conn.send(serializeMessage(m)); - } catch (IOException e) { - LOG.error("socket error", e); - } - } - - public List> generateNotebooksInfo(boolean needsReload) { Notebook notebook = notebook(); diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/AppMainServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java similarity index 86% rename from zeppelin-server/src/test/java/org/apache/zeppelin/socket/AppMainServerTest.java rename to zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java index d4bfea2f22f..43d947d89c7 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/AppMainServerTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java @@ -53,9 +53,9 @@ /** * BASIC Zeppelin rest api tests */ -public class AppMainServerTest extends AbstractTestRestApi { +public class NotebookServerTest extends AbstractTestRestApi { private static Notebook notebook; - private static AppMainServer appMainServer; + private static NotebookServer notebookServer; private static Gson gson; @BeforeClass @@ -63,7 +63,7 @@ public static void init() throws Exception { AbstractTestRestApi.startUp(); gson = new Gson(); notebook = ZeppelinServer.notebook; - appMainServer = ZeppelinServer.notebookWsServer; + notebookServer = ZeppelinServer.notebookWsServer; } @AfterClass @@ -73,7 +73,7 @@ public static void destroy() throws Exception { @Test public void checkOrigin() throws UnknownHostException { - AppMainServer server = new AppMainServer(); + NotebookServer server = new NotebookServer(); String origin = "http://" + InetAddress.getLocalHost().getHostName() + ":8080"; assertTrue("Origin " + origin + " is not allowed. Please check your hostname.", @@ -82,7 +82,7 @@ public void checkOrigin() throws UnknownHostException { @Test public void checkInvalidOrigin(){ - AppMainServer server = new AppMainServer(); + NotebookServer server = new NotebookServer(); assertFalse(server.checkOrigin(new TestHttpServletRequest(), "http://evillocalhost:8080")); } @@ -110,24 +110,24 @@ public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() thr interpreterGroup.getAngularObjectRegistry().add("object1", "value1", note1.getId(), null); // create two sockets and open it - NotebookSocket sock1 = createWebSocket(); - NotebookSocket sock2 = createWebSocket(); + WebAppSocket sock1 = createWebSocket(); + WebAppSocket sock2 = createWebSocket(); assertEquals(sock1, sock1); assertNotEquals(sock1, sock2); - appMainServer.onOpen(sock1); - appMainServer.onOpen(sock2); + notebookServer.onOpen(sock1); + notebookServer.onOpen(sock2); verify(sock1, times(0)).send(anyString()); // getNote, getAngularObject // open the same notebook from sockets - appMainServer.onMessage(sock1, gson.toJson(new Message(OP.GET_NOTE).put("id", note1.getId()))); - appMainServer.onMessage(sock2, gson.toJson(new Message(OP.GET_NOTE).put("id", note1.getId()))); + notebookServer.onMessage(sock1, gson.toJson(new Message(OP.GET_NOTE).put("id", note1.getId()))); + notebookServer.onMessage(sock2, gson.toJson(new Message(OP.GET_NOTE).put("id", note1.getId()))); reset(sock1); reset(sock2); // update object from sock1 - appMainServer.onMessage(sock1, gson.toJson( + notebookServer.onMessage(sock1, gson.toJson( new Message(OP.ANGULAR_OBJECT_UPDATED) .put("noteId", note1.getId()) .put("name", "object1") @@ -149,13 +149,13 @@ public void testImportNotebook() throws IOException { "paragraphs import\",\"config\":{},\"settings\":{}}]," + "\"name\": \"Test Zeppelin notebook import\",\"config\": " + "{}}}}"; - Message messageReceived = appMainServer.deserializeMessage(msg); + Message messageReceived = notebookServer.deserializeMessage(msg); Note note = null; try { - note = appMainServer.importNote(null, null, notebook, messageReceived); + note = notebookServer.importNote(null, null, notebook, messageReceived); } catch (NullPointerException e) { //broadcastNoteList(); failed nothing to worry. - LOG.error("Exception in AppMainServerTest while testImportNotebook, failed nothing to " + + LOG.error("Exception in NotebookServerTest while testImportNotebook, failed nothing to " + "worry ", e); } @@ -176,7 +176,7 @@ public void should_bind_angular_object_to_remote_for_paragraphs() throws Excepti .put("value", value) .put("paragraphId", "paragraphId"); - final AppMainServer server = new AppMainServer(); + final NotebookServer server = new NotebookServer(); final Notebook notebook = mock(Notebook.class); final Note note = mock(Note.class, RETURNS_DEEP_STUBS); @@ -196,8 +196,8 @@ public void should_bind_angular_object_to_remote_for_paragraphs() throws Excepti when(mdRegistry.addAndNotifyRemoteProcess(varName, value, "noteId", "paragraphId")).thenReturn(ao1); - NotebookSocket conn = mock(NotebookSocket.class); - NotebookSocket otherConn = mock(NotebookSocket.class); + WebAppSocket conn = mock(WebAppSocket.class); + WebAppSocket otherConn = mock(WebAppSocket.class); final String mdMsg1 = server.serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE) .put("angularObject", ao1) @@ -205,7 +205,7 @@ public void should_bind_angular_object_to_remote_for_paragraphs() throws Excepti .put("noteId", "noteId") .put("paragraphId", "paragraphId")); - server.noteSocketMap.put("noteId", asList(conn, otherConn)); + server.userWebSocketMap.put("noteId", asList(conn, otherConn)); // When server.angularObjectClientBind(conn, new HashSet(), notebook, messageReceived); @@ -227,7 +227,7 @@ public void should_bind_angular_object_to_local_for_paragraphs() throws Exceptio .put("value", value) .put("paragraphId", "paragraphId"); - final AppMainServer server = new AppMainServer(); + final NotebookServer server = new NotebookServer(); final Notebook notebook = mock(Notebook.class); final Note note = mock(Note.class, RETURNS_DEEP_STUBS); when(notebook.getNote("noteId")).thenReturn(note); @@ -245,8 +245,8 @@ public void should_bind_angular_object_to_local_for_paragraphs() throws Exceptio when(mdRegistry.add(varName, value, "noteId", "paragraphId")).thenReturn(ao1); - NotebookSocket conn = mock(NotebookSocket.class); - NotebookSocket otherConn = mock(NotebookSocket.class); + WebAppSocket conn = mock(WebAppSocket.class); + WebAppSocket otherConn = mock(WebAppSocket.class); final String mdMsg1 = server.serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE) .put("angularObject", ao1) @@ -254,7 +254,7 @@ public void should_bind_angular_object_to_local_for_paragraphs() throws Exceptio .put("noteId", "noteId") .put("paragraphId", "paragraphId")); - server.noteSocketMap.put("noteId", asList(conn, otherConn)); + server.userWebSocketMap.put("noteId", asList(conn, otherConn)); // When server.angularObjectClientBind(conn, new HashSet(), notebook, messageReceived); @@ -273,7 +273,7 @@ public void should_unbind_angular_object_from_remote_for_paragraphs() throws Exc .put("name", varName) .put("paragraphId", "paragraphId"); - final AppMainServer server = new AppMainServer(); + final NotebookServer server = new NotebookServer(); final Notebook notebook = mock(Notebook.class); final Note note = mock(Note.class, RETURNS_DEEP_STUBS); when(notebook.getNote("noteId")).thenReturn(note); @@ -288,8 +288,8 @@ public void should_unbind_angular_object_from_remote_for_paragraphs() throws Exc final AngularObject ao1 = AngularObjectBuilder.build(varName, value, "noteId", "paragraphId"); when(mdRegistry.removeAndNotifyRemoteProcess(varName, "noteId", "paragraphId")).thenReturn(ao1); - NotebookSocket conn = mock(NotebookSocket.class); - NotebookSocket otherConn = mock(NotebookSocket.class); + WebAppSocket conn = mock(WebAppSocket.class); + WebAppSocket otherConn = mock(WebAppSocket.class); final String mdMsg1 = server.serializeMessage(new Message(OP.ANGULAR_OBJECT_REMOVE) .put("angularObject", ao1) @@ -297,7 +297,7 @@ public void should_unbind_angular_object_from_remote_for_paragraphs() throws Exc .put("noteId", "noteId") .put("paragraphId", "paragraphId")); - server.noteSocketMap.put("noteId", asList(conn, otherConn)); + server.userWebSocketMap.put("noteId", asList(conn, otherConn)); // When server.angularObjectClientUnbind(conn, new HashSet(), notebook, messageReceived); @@ -318,7 +318,7 @@ public void should_unbind_angular_object_from_local_for_paragraphs() throws Exce .put("name", varName) .put("paragraphId", "paragraphId"); - final AppMainServer server = new AppMainServer(); + final NotebookServer server = new NotebookServer(); final Notebook notebook = mock(Notebook.class); final Note note = mock(Note.class, RETURNS_DEEP_STUBS); when(notebook.getNote("noteId")).thenReturn(note); @@ -336,15 +336,15 @@ public void should_unbind_angular_object_from_local_for_paragraphs() throws Exce when(mdRegistry.remove(varName, "noteId", "paragraphId")).thenReturn(ao1); - NotebookSocket conn = mock(NotebookSocket.class); - NotebookSocket otherConn = mock(NotebookSocket.class); + WebAppSocket conn = mock(WebAppSocket.class); + WebAppSocket otherConn = mock(WebAppSocket.class); final String mdMsg1 = server.serializeMessage(new Message(OP.ANGULAR_OBJECT_REMOVE) .put("angularObject", ao1) .put("interpreterGroupId", "mdGroup") .put("noteId", "noteId") .put("paragraphId", "paragraphId")); - server.noteSocketMap.put("noteId", asList(conn, otherConn)); + server.userWebSocketMap.put("noteId", asList(conn, otherConn)); // When server.angularObjectClientUnbind(conn, new HashSet(), notebook, messageReceived); @@ -353,8 +353,8 @@ public void should_unbind_angular_object_from_local_for_paragraphs() throws Exce verify(otherConn).send(mdMsg1); } - private NotebookSocket createWebSocket() { - NotebookSocket sock = mock(NotebookSocket.class); + private WebAppSocket createWebSocket() { + WebAppSocket sock = mock(WebAppSocket.class); when(sock.getRequest()).thenReturn(createHttpServletRequest()); return sock; } diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 4d697dd44d4..87fbd127101 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -57,6 +57,7 @@ angular.module('zeppelinWebApp') $scope.$on('setNotebookJobs', function(event, responseData) { $scope.lastJobServerUnixTime = responseData.lastResponseUnixTime; + console.log('recev time ', $scope.lastJobServerUnixTime); $scope.jobInfomations = responseData.jobs; $scope.jobInfomationsIndexs = $scope.jobInfomations? _.indexBy($scope.jobInfomations, 'notebookId') : {}; $scope.JobInfomationsByFilter = $scope.jobTypeFilter($scope.jobInfomations, $scope.filterConfig); diff --git a/zeppelin-web/src/app/notebook/notebook.controller.js b/zeppelin-web/src/app/notebook/notebook.controller.js index 3011e7f29cf..5eb59fde56e 100644 --- a/zeppelin-web/src/app/notebook/notebook.controller.js +++ b/zeppelin-web/src/app/notebook/notebook.controller.js @@ -43,7 +43,7 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', var connectedOnce = false; - $scope.$on('setNotebookWSConnectedStatus', function(event, param) { + $scope.$on('setWSConnectedStatus', function(event, param) { if(connectedOnce && param){ initNotebook(); } diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index 22ce13a1d77..07f60dd2d23 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -42,7 +42,7 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco notebookListDataFactory.setNotes(notes); }); - $scope.$on('setNotebookWSConnectedStatus', function(event, param) { + $scope.$on('setWSConnectedStatus', function(event, param) { vm.connected = param; }); diff --git a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js index 2ed254798a8..4ee94a39903 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js +++ b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js @@ -14,21 +14,20 @@ 'use strict'; angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, $websocket, $location, baseUrlSrv) { - var websocketCalls = { notebookServer : {}, jobManagerServer : {} }; + var websocketCalls = {}; - // create socket for notebook server - websocketCalls.notebookServer.ws = $websocket(baseUrlSrv.getWebsocketUrl()); - websocketCalls.notebookServer.ws.reconnectIfNotNormalClose = true; + websocketCalls.ws = $websocket(baseUrlSrv.getWebsocketUrl()); + websocketCalls.ws.reconnectIfNotNormalClose = true; - websocketCalls.notebookServer.ws.onOpen(function() { - console.log('Websocket Notebook Server created'); - $rootScope.$broadcast('setNotebookWSConnectedStatus', true); + websocketCalls.ws.onOpen(function() { + console.log('Websocket created'); + $rootScope.$broadcast('setWSConnectedStatus', true); setInterval(function(){ - websocketCalls.notebookServer.sendNewEvent({op: 'PING'}); + websocketCalls.sendNewEvent({op: 'PING'}); }, 10000); }); - websocketCalls.notebookServer.sendNewEvent = function(data) { + websocketCalls.sendNewEvent = function(data, targetServer) { if ($rootScope.ticket !== undefined) { data.principal = $rootScope.ticket.principal; data.ticket = $rootScope.ticket.ticket; @@ -38,15 +37,22 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, data.ticket = ''; data.roles = ''; } + + if (targetServer !== undefined) { + data.target = targetServer; + } else { + data.target = ''; + } + console.log('Send Notebook Server >> %o, %o, %o, %o, %o', data.op, data.principal, data.ticket, data.roles, data); - websocketCalls.notebookServer.ws.send(JSON.stringify(data)); + websocketCalls.ws.send(JSON.stringify(data)); }; - websocketCalls.notebookServer.isConnected = function() { - return (websocketCalls.notebookServer.ws.socket.readyState === 1); + websocketCalls.isConnected = function() { + return (websocketCalls.ws.socket.readyState === 1); }; - websocketCalls.notebookServer.ws.onMessage(function(event) { + websocketCalls.ws.onMessage(function(event) { var payload; if (event.data) { payload = angular.fromJson(event.data); @@ -60,6 +66,10 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, $location.path('notebook/' + data.note.id); } else if (op === 'NOTES_INFO') { $rootScope.$broadcast('setNoteMenu', data.notes); + } else if (op === 'LIST_NOTEBOOK_JOBS') { + $rootScope.$broadcast('setNotebookJobs', data.notebookJobs); + } else if (op === 'LIST_UPDATE_NOTEBOOK_JOBS') { + $rootScope.$broadcast('setUpdateNotebookJobs', data.notebookRunningJobs); } else if (op === 'AUTH_INFO') { BootstrapDialog.show({ closable: true, @@ -97,69 +107,14 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, } }); - websocketCalls.notebookServer.ws.onError(function(event) { + websocketCalls.ws.onError(function(event) { console.log('[notebookWS] Notebook Server message: ', event); - $rootScope.$broadcast('setNotebookWSConnectedStatus', false); + $rootScope.$broadcast('setWSConnectedStatus', false); }); - websocketCalls.notebookServer.ws.onClose(function(event) { + websocketCalls.ws.onClose(function(event) { console.log('[notebookWS] close message: ', event); - $rootScope.$broadcast('setNotebookWSConnectedStatus', false); - }); - - // create socket for job manager server - websocketCalls.jobManagerServer.ws = $websocket(baseUrlSrv.getJobManagerWebsocketUrl()); - websocketCalls.jobManagerServer.ws.reconnectIfNotNormalClose = true; - - websocketCalls.jobManagerServer.ws.onOpen(function() { - console.log('Websocket Job Manager Server created'); - $rootScope.$broadcast('setJobManagerWSConnectedStatus', true); - setInterval(function(){ - websocketCalls.jobManagerServer.sendNewEvent({op: 'PING'}); - }, 10000); - }); - - websocketCalls.jobManagerServer.sendNewEvent = function(data) { - if ($rootScope.ticket !== undefined) { - data.principal = $rootScope.ticket.principal; - data.ticket = $rootScope.ticket.ticket; - data.roles = $rootScope.ticket.roles; - } else { - data.principal = ''; - data.ticket = ''; - data.roles = ''; - } - console.log('Send Job Manager Server >> %o, %o, %o, %o, %o', data.op, data.principal, data.ticket, data.roles, data); - websocketCalls.jobManagerServer.ws.send(JSON.stringify(data)); - }; - - websocketCalls.jobManagerServer.isConnected = function() { - return (websocketCalls.notebookServer.ws.socket.readyState === 1); - }; - - websocketCalls.jobManagerServer.ws.onMessage(function(event) { - var payload; - if (event.data) { - payload = angular.fromJson(event.data); - } - console.log('Receive Job Manager Server<< %o, %o', payload.op, payload); - var op = payload.op; - var data = payload.data; - if (op === 'LIST_NOTEBOOK_JOBS') { - $rootScope.$broadcast('setNotebookJobs', data.notebookJobs); - } else if (op === 'LIST_UPDATE_NOTEBOOK_JOBS') { - $rootScope.$broadcast('setUpdateNotebookJobs', data.notebookRunningJobs); - } - }); - - websocketCalls.jobManagerServer.ws.onError(function(event) { - console.log('[jobManagerWS] error message: ', event); - $rootScope.$broadcast('setJobManagerWSConnectedStatus', false); - }); - - websocketCalls.jobManagerServer.ws.onClose(function(event) { - console.log('[jobManagerWS] close message: ', event); - $rootScope.$broadcast('setJobManagerWSConnectedStatus', false); + $rootScope.$broadcast('setWSConnectedStatus', false); }); return websocketCalls; diff --git a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js index aa857db3036..8042910f598 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js +++ b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js @@ -18,47 +18,47 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, return { getHomeNotebook: function() { - websocketEvents.notebookServer.sendNewEvent({op: 'GET_HOME_NOTE'}); + websocketEvents.sendNewEvent({op: 'GET_HOME_NOTE'}, 'notebookServer'); }, createNotebook: function(noteName) { - websocketEvents.notebookServer.sendNewEvent({op: 'NEW_NOTE',data: {name: noteName}}); + websocketEvents.sendNewEvent({op: 'NEW_NOTE',data: {name: noteName}}, 'notebookServer'); }, deleteNotebook: function(noteId) { - websocketEvents.notebookServer.sendNewEvent({op: 'DEL_NOTE', data: {id: noteId}}); + websocketEvents.sendNewEvent({op: 'DEL_NOTE', data: {id: noteId}}, 'notebookServer'); }, cloneNotebook: function(noteIdToClone, newNoteName ) { - websocketEvents.notebookServer.sendNewEvent({op: 'CLONE_NOTE', data: {id: noteIdToClone, name: newNoteName}}); + websocketEvents.sendNewEvent({op: 'CLONE_NOTE', data: {id: noteIdToClone, name: newNoteName}}, 'notebookServer'); }, getNotebookList: function() { - websocketEvents.notebookServer.sendNewEvent({op: 'LIST_NOTES'}); + websocketEvents.sendNewEvent({op: 'LIST_NOTES'}, 'notebookServer'); }, reloadAllNotesFromRepo: function() { - websocketEvents.notebookServer.sendNewEvent({op: 'RELOAD_NOTES_FROM_REPO'}); + websocketEvents.sendNewEvent({op: 'RELOAD_NOTES_FROM_REPO'}, 'notebookServer'); }, getNotebook: function(noteId) { - websocketEvents.notebookServer.sendNewEvent({op: 'GET_NOTE', data: {id: noteId}}); + websocketEvents.sendNewEvent({op: 'GET_NOTE', data: {id: noteId}}, 'notebookServer'); }, updateNotebook: function(noteId, noteName, noteConfig) { - websocketEvents.notebookServer.sendNewEvent({op: 'NOTE_UPDATE', data: {id: noteId, name: noteName, config : noteConfig}}); + websocketEvents.sendNewEvent({op: 'NOTE_UPDATE', data: {id: noteId, name: noteName, config : noteConfig}}, 'notebookServer'); }, moveParagraph: function(paragraphId, newIndex) { - websocketEvents.notebookServer.sendNewEvent({ op: 'MOVE_PARAGRAPH', data : {id: paragraphId, index: newIndex}}); + websocketEvents.sendNewEvent({ op: 'MOVE_PARAGRAPH', data : {id: paragraphId, index: newIndex}}, 'notebookServer'); }, insertParagraph: function(newIndex) { - websocketEvents.notebookServer.sendNewEvent({ op: 'INSERT_PARAGRAPH', data : {index: newIndex}}); + websocketEvents.sendNewEvent({ op: 'INSERT_PARAGRAPH', data : {index: newIndex}}, 'notebookServer'); }, updateAngularObject: function(noteId, paragraphId, name, value, interpreterGroupId) { - websocketEvents.notebookServer.sendNewEvent({ + websocketEvents.sendNewEvent({ op: 'ANGULAR_OBJECT_UPDATED', data: { noteId: noteId, @@ -67,11 +67,11 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, value: value, interpreterGroupId: interpreterGroupId } - }); + }, 'notebookServer'); }, clientBindAngularObject: function(noteId, name, value, paragraphId) { - websocketEvents.notebookServer.sendNewEvent({ + websocketEvents.sendNewEvent({ op: 'ANGULAR_OBJECT_CLIENT_BIND', data: { noteId: noteId, @@ -79,26 +79,26 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, value: value, paragraphId: paragraphId } - }); + }, 'notebookServer'); }, clientUnbindAngularObject: function(noteId, name, paragraphId) { - websocketEvents.notebookServer.sendNewEvent({ + websocketEvents.sendNewEvent({ op: 'ANGULAR_OBJECT_CLIENT_UNBIND', data: { noteId: noteId, name: name, paragraphId: paragraphId } - }); + }, 'notebookServer'); }, cancelParagraphRun: function(paragraphId) { - websocketEvents.notebookServer.sendNewEvent({op: 'CANCEL_PARAGRAPH', data: {id: paragraphId}}); + websocketEvents.sendNewEvent({op: 'CANCEL_PARAGRAPH', data: {id: paragraphId}}, 'notebookServer'); }, runParagraph: function(paragraphId, paragraphTitle, paragraphData, paragraphConfig, paragraphParams) { - websocketEvents.notebookServer.sendNewEvent({ + websocketEvents.sendNewEvent({ op: 'RUN_PARAGRAPH', data: { id: paragraphId, @@ -107,30 +107,30 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, config: paragraphConfig, params: paragraphParams } - }); + }, 'notebookServer'); }, removeParagraph: function(paragraphId) { - websocketEvents.notebookServer.sendNewEvent({op: 'PARAGRAPH_REMOVE', data: {id: paragraphId}}); + websocketEvents.sendNewEvent({op: 'PARAGRAPH_REMOVE', data: {id: paragraphId}}, 'notebookServer'); }, clearParagraphOutput: function(paragraphId) { - websocketEvents.notebookServer.sendNewEvent({op: 'PARAGRAPH_CLEAR_OUTPUT', data: {id: paragraphId}}); + websocketEvents.sendNewEvent({op: 'PARAGRAPH_CLEAR_OUTPUT', data: {id: paragraphId}}, 'notebookServer'); }, completion: function(paragraphId, buf, cursor) { - websocketEvents.notebookServer.sendNewEvent({ + websocketEvents.sendNewEvent({ op : 'COMPLETION', data : { id : paragraphId, buf : buf, cursor : cursor } - }); + }, 'notebookServer'); }, commitParagraph: function(paragraphId, paragraphTitle, paragraphData, paragraphConfig, paragraphParams) { - websocketEvents.notebookServer.sendNewEvent({ + websocketEvents.sendNewEvent({ op: 'COMMIT_PARAGRAPH', data: { id: paragraphId, @@ -139,39 +139,40 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, config: paragraphConfig, params: paragraphParams } - }); + }, 'notebookServer'); }, importNotebook: function(notebook) { - websocketEvents.notebookServer.sendNewEvent({ + websocketEvents.sendNewEvent({ op: 'IMPORT_NOTE', data: { notebook: notebook } - }); + }, 'notebookServer'); }, checkpointNotebook: function(noteId, commitMessage) { - websocketEvents.notebookServer.sendNewEvent({ + websocketEvents.sendNewEvent({ op: 'CHECKPOINT_NOTEBOOK', data: { noteId: noteId, commitMessage: commitMessage } - }); + }, 'notebookServer'); }, isConnected: function(){ - return websocketEvents.notebookServer.isConnected(); + return websocketEvents.isConnected(); }, getNotebookJobsList: function() { - websocketEvents.jobManagerServer.sendNewEvent({op: 'LIST_NOTEBOOK_JOBS'}); + websocketEvents.sendNewEvent({op: 'LIST_NOTEBOOK_JOBS'}, 'jobManagerServer'); }, getUpdateNotebookJobsList: function(lastUpdateServerUnixTime) { - websocketEvents.jobManagerServer.sendNewEvent( - {op: 'LIST_UPDATE_NOTEBOOK_JOBS', data : {lastUpdateUnixTime : lastUpdateServerUnixTime*1}} + websocketEvents.sendNewEvent( + {op: 'LIST_UPDATE_NOTEBOOK_JOBS', data : {lastUpdateUnixTime : lastUpdateServerUnixTime*1}}, + 'jobManagerServer' ); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java index 74a0b9dc063..40cdebd922b 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java @@ -121,6 +121,7 @@ public static enum OP { } public OP op; + public String target; public Map data = new HashMap(); public String ticket = "anonymous"; public String principal = "anonymous"; From 80165f51eb00898b025bebcd0b43a1c2c9cf407f Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Thu, 2 Jun 2016 02:28:01 +0900 Subject: [PATCH 17/25] add 'check undefined logic' in jobmanager.controller --- zeppelin-web/src/app/jobmanager/jobmanager.controller.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 87fbd127101..b2e07f22d97 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -132,7 +132,9 @@ angular.module('zeppelinWebApp') websocketMsgSrv.getNotebookJobsList(); var refreshObj = $interval(function () { - websocketMsgSrv.getUpdateNotebookJobsList($scope.lastJobServerUnixTime); + if ($scope.lastJobServerUnixTime !== undefined) { + websocketMsgSrv.getUpdateNotebookJobsList($scope.lastJobServerUnixTime); + } }, 1000); $scope.$on('$destroy', function() { $interval.cancel(refreshObj); From 2d37f891544eba359b10c640145f4d682677fe78 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Thu, 2 Jun 2016 19:01:54 +0900 Subject: [PATCH 18/25] implement observer in notebook --- .../zeppelin/server/ZeppelinServer.java | 32 +++++---- .../zeppelin/socket/JobMangerServer.java | 38 ++++------ .../org/apache/zeppelin/notebook/Note.java | 18 ++++- .../apache/zeppelin/notebook/Notebook.java | 23 +++++- .../notebook/NotebookEventObserver.java | 71 +++++++++++++++++++ .../zeppelin/search/LuceneSearchTest.java | 3 +- 6 files changed, 144 insertions(+), 41 deletions(-) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventObserver.java diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index f554acaeeba..5439bafafa6 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -113,8 +113,9 @@ public static void main(String[] args) throws InterruptedException { mainWsServer.setSubWebSocketServer("notebookServer", notebookWsServer); // Job Manager WS server - WebSocketServer jobManagerServer = setupJobManagerServer(webApp, conf); - mainWsServer.setSubWebSocketServer("jobManagerServer", jobManagerServer); +// JobMangerServer jobManagerServer = setupJobManagerServer(webApp, conf); +// mainWsServer.setSubWebSocketServer("jobManagerServer", jobManagerServer); + //Below is commented since zeppelin-docs module is removed. //final WebAppContext webAppSwagg = setupWebAppSwagger(conf); @@ -128,6 +129,9 @@ public static void main(String[] args) throws InterruptedException { } LOG.info("Done, zeppelin server started"); + // register observer for job manager. +// notebook.getNotebookEventObserver().addObserver(jobManagerServer); + Runtime.getRuntime().addShutdownHook(new Thread(){ @Override public void run() { LOG.info("Shutting down Zeppelin Server ... "); @@ -217,24 +221,24 @@ private static AppMainServer setupWSAppMainServer(WebAppContext webapp, private static NotebookServer setupNotebookServer(WebAppContext webapp, ZeppelinConfiguration conf) { NotebookServer notebookWsServer = new NotebookServer(); - String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); - final ServletHolder servletHolder = new ServletHolder(notebookWsServer); - servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); - final ServletContextHandler cxfContext = new ServletContextHandler( - ServletContextHandler.SESSIONS); - webapp.addServlet(servletHolder, "/notebook/*"); +// String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); +// final ServletHolder servletHolder = new ServletHolder(notebookWsServer); +// servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); +// final ServletContextHandler cxfContext = new ServletContextHandler( +// ServletContextHandler.SESSIONS); +// webapp.addServlet(servletHolder, "/notebook/*"); return notebookWsServer; } private static JobMangerServer setupJobManagerServer(WebAppContext webapp, ZeppelinConfiguration conf) { JobMangerServer jobMangerWsServer = new JobMangerServer(); - String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); - final ServletHolder servletHolder = new ServletHolder(jobMangerWsServer); - servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); - final ServletContextHandler cxfContext = new ServletContextHandler( - ServletContextHandler.SESSIONS); - webapp.addServlet(servletHolder, "/jobmanager/*"); +// String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); +// final ServletHolder servletHolder = new ServletHolder(jobMangerWsServer); +// servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); +// final ServletContextHandler cxfContext = new ServletContextHandler( +// ServletContextHandler.SESSIONS); +// webapp.addServlet(servletHolder, "/jobmanager/*"); return jobMangerWsServer; } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java index 7391b6501f1..6cc2d811427 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java @@ -16,47 +16,25 @@ */ package org.apache.zeppelin.socket; -import com.google.common.base.Strings; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import org.apache.zeppelin.display.AngularObject; -import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.display.AngularObjectRegistryListener; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterOutput; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterSetting; -import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; import org.apache.zeppelin.notebook.*; import org.apache.zeppelin.notebook.socket.Message; import org.apache.zeppelin.notebook.socket.Message.OP; -import org.apache.zeppelin.scheduler.Job; -import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.notebook.NotebookEventObserver.NotebookChnagedEvent; import org.apache.zeppelin.server.ZeppelinServer; import org.apache.zeppelin.ticket.TicketContainer; -import org.apache.zeppelin.user.AuthenticationInfo; -import org.apache.zeppelin.utils.SecurityUtils; -import org.eclipse.jetty.websocket.servlet.WebSocketServlet; -import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; -import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServletRequest; import java.io.IOException; -import java.net.URISyntaxException; -import java.net.UnknownHostException; import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; /** * Zeppelin websocket service. */ -public class JobMangerServer extends AppMainServer implements WebSocketServer { +public class JobMangerServer extends AppMainServer implements WebSocketServer, Observer{ private static final Logger LOG = LoggerFactory.getLogger(JobMangerServer.class); private Notebook notebook() { @@ -383,5 +361,17 @@ public List> generateUpdateNotebooksJobInfo( return notesInfo; } + /** + * Notebook Observer Event Listener + */ + @Override + public void update(Observable observer, Object notebookChnagedEvent) { + + NotebookChnagedEvent noteEvent = (NotebookChnagedEvent) notebookChnagedEvent; + LOG.info("event note {}", noteEvent.getNoteId()); + + + } + } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index a73aad95a8f..4022ff395d4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -70,6 +70,7 @@ public class Note implements Serializable, JobListener { private transient NotebookRepo repo; private transient SearchService index; private transient ScheduledFuture delayedPersist; + private transient NotebookEventObserver notebookEventObserver; /** * note configurations. @@ -89,11 +90,13 @@ public class Note implements Serializable, JobListener { public Note() {} public Note(NotebookRepo repo, NoteInterpreterLoader replLoader, - JobListenerFactory jlFactory, SearchService noteIndex) { + JobListenerFactory jlFactory, SearchService noteIndex, + NotebookEventObserver notebookEventObserver) { this.repo = repo; this.replLoader = replLoader; this.jobListenerFactory = jlFactory; this.index = noteIndex; + this.notebookEventObserver = notebookEventObserver; generateId(); } @@ -115,6 +118,7 @@ public String getName() { public void setName(String name) { this.name = name; + notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.CHNAGED_NOTE_NAME); } public NoteInterpreterLoader getNoteReplLoader() { @@ -123,6 +127,7 @@ public NoteInterpreterLoader getNoteReplLoader() { public void setReplLoader(NoteInterpreterLoader replLoader) { this.replLoader = replLoader; + notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.BIND_INTERPRETER); } public JobListenerFactory getJobListenerFactory() { @@ -159,6 +164,9 @@ public Paragraph addParagraph() { synchronized (paragraphs) { paragraphs.add(p); } + + notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.ADD_PARAGRAPH); + return p; } @@ -190,6 +198,8 @@ public void addCloneParagraph(Paragraph srcParagraph) { synchronized (paragraphs) { paragraphs.add(newParagraph); } + + notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.ADD_PARAGRAPH); } /** @@ -202,6 +212,9 @@ public Paragraph insertParagraph(int index) { synchronized (paragraphs) { paragraphs.add(index, p); } + + notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.ADD_PARAGRAPH); + return p; } @@ -293,6 +306,7 @@ public void moveParagraph(String paragraphId, int index, boolean throwWhenIndexI if (p != null) { paragraphs.add(index, p); + notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.MOVED_PARAGRAPH); } } } @@ -386,6 +400,8 @@ public void run(String paragraphId) { if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) { intp.getScheduler().submit(p); } + + notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.RUN_PARAGRAPH); } public List completion(String paragraphId, String buffer, int cursor) { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 986753c9cc4..d91577f88bd 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -78,6 +78,7 @@ public class Notebook { private NotebookRepo notebookRepo; private SearchService notebookIndex; private NotebookAuthorization notebookAuthorization; + private NotebookEventObserver notebookEventObserver; /** * Main constructor \w manual Dependency Injection @@ -109,6 +110,7 @@ public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo, quartzSched = quertzSchedFact.getScheduler(); quartzSched.start(); CronJob.notebook = this; + notebookEventObserver = new NotebookEventObserver(); loadAllNotes(); if (this.notebookIndex != null) { @@ -121,6 +123,10 @@ public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo, } + public NotebookEventObserver getNotebookEventObserver() { + return notebookEventObserver; + } + /** * Create new note. * @@ -135,6 +141,7 @@ public Note createNote() throws IOException { note = createNote(null); } notebookIndex.addIndexDoc(note); + notebookEventObserver.notifyChanged(note.id(), NotebookEventObserver.ACTIONS.CREATE); return note; } @@ -146,7 +153,8 @@ public Note createNote() throws IOException { */ public Note createNote(List interpreterIds) throws IOException { NoteInterpreterLoader intpLoader = new NoteInterpreterLoader(replFactory); - Note note = new Note(notebookRepo, intpLoader, jobListenerFactory, notebookIndex); + Note note = new Note(notebookRepo, intpLoader, + jobListenerFactory, notebookIndex, notebookEventObserver); intpLoader.setNoteId(note.id()); synchronized (notes) { notes.put(note.id(), note); @@ -157,6 +165,7 @@ public Note createNote(List interpreterIds) throws IOException { notebookIndex.addIndexDoc(note); note.persist(); + notebookEventObserver.notifyChanged(note.id(), NotebookEventObserver.ACTIONS.CREATE); return note; } @@ -204,6 +213,7 @@ public Note importNote(String sourceJson, String noteName) throws IOException { } newNote.persist(); + notebookEventObserver.notifyChanged(newNote.id(), NotebookEventObserver.ACTIONS.CREATE); } catch (IOException e) { logger.error(e.toString(), e); throw e; @@ -241,6 +251,7 @@ public Note cloneNote(String sourceNoteID, String newNoteName) throws notebookIndex.addIndexDoc(newNote); newNote.persist(); + notebookEventObserver.notifyChanged(newNote.id(), NotebookEventObserver.ACTIONS.CREATE); return newNote; } @@ -251,6 +262,8 @@ public void bindInterpretersToNote(String id, note.getNoteReplLoader().setInterpreters(interpreterSettingIds); // comment out while note.getNoteReplLoader().setInterpreters(...) do the same // replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds); + notebookEventObserver.notifyChanged(note.id(), + NotebookEventObserver.ACTIONS.BIND_INTERPRETER); } } @@ -284,6 +297,8 @@ public void removeNote(String id) { synchronized (notes) { note = notes.remove(id); } + notebookEventObserver.notifyChanged(note.id(), NotebookEventObserver.ACTIONS.REMOVED); + replFactory.removeNoteInterpreterSettingBinding(id); notebookIndex.deleteIndexDocs(note); notebookAuthorization.removeNote(id); @@ -495,6 +510,10 @@ public void execute(JobExecutionContext context) throws JobExecutionException { String noteId = context.getJobDetail().getJobDataMap().getString("noteId"); Note note = notebook.getNote(noteId); + + NotebookEventObserver notebookEventObserver = notebook.getNotebookEventObserver(); + notebookEventObserver.notifyChanged(note.id(), NotebookEventObserver.ACTIONS.RUN); + note.runAll(); while (!note.getLastParagraph().isTerminated()) { @@ -569,6 +588,7 @@ public void refreshCron(String id) { info.put("cron", "Scheduler Exception"); } } + notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.CHANGED_CONFIG); } private void removeCron(String id) { @@ -577,6 +597,7 @@ private void removeCron(String id) { } catch (SchedulerException e) { logger.error("Can't remove quertz " + id, e); } + notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.CHANGED_CONFIG); } public InterpreterFactory getInterpreterFactory() { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventObserver.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventObserver.java new file mode 100644 index 00000000000..a33c2f3e4f3 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventObserver.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook; + +import java.util.Observable; + +/** + * Notebook event observer + */ +public class NotebookEventObserver extends Observable { + + /** + * Notebook event Enum + */ + public static enum ACTIONS { + REMOVED, CREATE, CLONE, RUN, MOVE, + BIND_INTERPRETER, CHANGED_CONFIG, CHNAGED_NOTE_NAME, ADD_PARAGRAPH, MOVED_PARAGRAPH, + RUN_PARAGRAPH + } + + void notifyChanged(String noteId, ACTIONS action) { + setChanged(); + NotebookChnagedEvent event = new NotebookChnagedEvent(noteId, action); + notifyObservers(event); + } + + /** + * Notebook Event Model + */ + public class NotebookChnagedEvent { + private String noteId; + private ACTIONS action; + + public NotebookChnagedEvent(String noteId, ACTIONS action) { + this.noteId = noteId; + this.action = action; + } + + public String getNoteId() { + return noteId; + } + + public void setNoteId(String noteId) { + this.noteId = noteId; + } + + public ACTIONS getAction() { + return action; + } + + public void setAction(ACTIONS action) { + this.action = action; + } + } + +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java index c744267ec22..a3cb3ba34c5 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java @@ -28,6 +28,7 @@ import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInterpreterLoader; +import org.apache.zeppelin.notebook.NotebookEventObserver; import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.junit.After; @@ -286,7 +287,7 @@ private Paragraph addParagraphWithTextAndTitle(Note note, String text, String ti } private Note newNote(String name) { - Note note = new Note(notebookRepoMock, replLoaderMock, null, notebookIndex); + Note note = new Note(notebookRepoMock, replLoaderMock, null, notebookIndex, new NotebookEventObserver()); note.setName(name); return note; } From 86bd17c6cc1a3e50404792d5dd891f15c9528f6c Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Fri, 3 Jun 2016 01:57:16 +0900 Subject: [PATCH 19/25] changed observer noti in note class --- .../zeppelin/server/ZeppelinServer.java | 34 +++++++++---------- .../org/apache/zeppelin/notebook/Note.java | 21 ++++++++---- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 5439bafafa6..525d5ea6c83 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -85,8 +85,8 @@ public ZeppelinServer() throws Exception { this.notebookIndex = new LuceneSearch(); this.notebookAuthorization = new NotebookAuthorization(conf); notebook = new Notebook(conf, - notebookRepo, schedulerFactory, replFactory, notebookWsServer, - notebookIndex, notebookAuthorization); + notebookRepo, schedulerFactory, replFactory, notebookWsServer, + notebookIndex, notebookAuthorization); } public static void main(String[] args) throws InterruptedException { @@ -113,8 +113,8 @@ public static void main(String[] args) throws InterruptedException { mainWsServer.setSubWebSocketServer("notebookServer", notebookWsServer); // Job Manager WS server -// JobMangerServer jobManagerServer = setupJobManagerServer(webApp, conf); -// mainWsServer.setSubWebSocketServer("jobManagerServer", jobManagerServer); + JobMangerServer jobManagerServer = setupJobManagerServer(webApp, conf); + mainWsServer.setSubWebSocketServer("jobManagerServer", jobManagerServer); //Below is commented since zeppelin-docs module is removed. @@ -130,7 +130,7 @@ public static void main(String[] args) throws InterruptedException { LOG.info("Done, zeppelin server started"); // register observer for job manager. -// notebook.getNotebookEventObserver().addObserver(jobManagerServer); + notebook.getNotebookEventObserver().addObserver(jobManagerServer); Runtime.getRuntime().addShutdownHook(new Thread(){ @Override public void run() { @@ -221,24 +221,24 @@ private static AppMainServer setupWSAppMainServer(WebAppContext webapp, private static NotebookServer setupNotebookServer(WebAppContext webapp, ZeppelinConfiguration conf) { NotebookServer notebookWsServer = new NotebookServer(); -// String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); -// final ServletHolder servletHolder = new ServletHolder(notebookWsServer); -// servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); -// final ServletContextHandler cxfContext = new ServletContextHandler( -// ServletContextHandler.SESSIONS); -// webapp.addServlet(servletHolder, "/notebook/*"); + String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); + final ServletHolder servletHolder = new ServletHolder(notebookWsServer); + servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); + final ServletContextHandler cxfContext = new ServletContextHandler( + ServletContextHandler.SESSIONS); + webapp.addServlet(servletHolder, "/notebook/*"); return notebookWsServer; } private static JobMangerServer setupJobManagerServer(WebAppContext webapp, ZeppelinConfiguration conf) { JobMangerServer jobMangerWsServer = new JobMangerServer(); -// String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); -// final ServletHolder servletHolder = new ServletHolder(jobMangerWsServer); -// servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); -// final ServletContextHandler cxfContext = new ServletContextHandler( -// ServletContextHandler.SESSIONS); -// webapp.addServlet(servletHolder, "/jobmanager/*"); + String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); + final ServletHolder servletHolder = new ServletHolder(jobMangerWsServer); + servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); + final ServletContextHandler cxfContext = new ServletContextHandler( + ServletContextHandler.SESSIONS); + webapp.addServlet(servletHolder, "/jobmanager/*"); return jobMangerWsServer; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 4022ff395d4..0a79d38045d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -31,6 +31,7 @@ import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.notebook.repo.NotebookRepo; +import org.apache.zeppelin.notebook.NotebookEventObserver.ACTIONS; import org.apache.zeppelin.notebook.utility.IdHashes; import org.apache.zeppelin.resource.ResourcePoolUtils; import org.apache.zeppelin.scheduler.Job; @@ -100,6 +101,12 @@ public Note(NotebookRepo repo, NoteInterpreterLoader replLoader, generateId(); } + private void notifyChanged(ACTIONS event) { + if (notebookEventObserver != null) { + notebookEventObserver.notifyChanged(id, event); + } + } + private void generateId() { id = IdHashes.encode(System.currentTimeMillis() + new Random().nextInt()); } @@ -118,7 +125,7 @@ public String getName() { public void setName(String name) { this.name = name; - notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.CHNAGED_NOTE_NAME); + notifyChanged(NotebookEventObserver.ACTIONS.CHNAGED_NOTE_NAME); } public NoteInterpreterLoader getNoteReplLoader() { @@ -127,7 +134,7 @@ public NoteInterpreterLoader getNoteReplLoader() { public void setReplLoader(NoteInterpreterLoader replLoader) { this.replLoader = replLoader; - notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.BIND_INTERPRETER); + notifyChanged(NotebookEventObserver.ACTIONS.BIND_INTERPRETER); } public JobListenerFactory getJobListenerFactory() { @@ -165,7 +172,7 @@ public Paragraph addParagraph() { paragraphs.add(p); } - notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.ADD_PARAGRAPH); + notifyChanged(NotebookEventObserver.ACTIONS.ADD_PARAGRAPH); return p; } @@ -199,7 +206,7 @@ public void addCloneParagraph(Paragraph srcParagraph) { paragraphs.add(newParagraph); } - notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.ADD_PARAGRAPH); + notifyChanged(NotebookEventObserver.ACTIONS.ADD_PARAGRAPH); } /** @@ -213,7 +220,7 @@ public Paragraph insertParagraph(int index) { paragraphs.add(index, p); } - notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.ADD_PARAGRAPH); + notifyChanged(NotebookEventObserver.ACTIONS.ADD_PARAGRAPH); return p; } @@ -306,7 +313,7 @@ public void moveParagraph(String paragraphId, int index, boolean throwWhenIndexI if (p != null) { paragraphs.add(index, p); - notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.MOVED_PARAGRAPH); + notifyChanged(NotebookEventObserver.ACTIONS.MOVED_PARAGRAPH); } } } @@ -401,7 +408,7 @@ public void run(String paragraphId) { intp.getScheduler().submit(p); } - notebookEventObserver.notifyChanged(id, NotebookEventObserver.ACTIONS.RUN_PARAGRAPH); + notifyChanged(NotebookEventObserver.ACTIONS.RUN_PARAGRAPH); } public List completion(String paragraphId, String buffer, int cursor) { From a40c734b0da40d0bb6d9d1e73ff6b6e5fdd487e4 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Sat, 4 Jun 2016 07:36:22 +0900 Subject: [PATCH 20/25] job manager bug fix and improve --- .../apache/zeppelin/socket/AppMainServer.java | 5 +- .../zeppelin/socket/JobMangerServer.java | 264 +++++++++++++----- .../zeppelin/socket/NotebookServer.java | 1 - .../app/jobmanager/jobmanager.controller.js | 94 +++++-- .../src/app/jobmanager/jobmanager.html | 64 +++-- zeppelin-web/src/app/jobmanager/jobs/job.html | 34 +-- .../websocketEvents/websocketMsg.service.js | 6 +- .../org/apache/zeppelin/notebook/Note.java | 9 +- .../apache/zeppelin/notebook/Notebook.java | 3 +- .../notebook/NotebookEventObserver.java | 2 +- .../zeppelin/notebook/socket/Message.java | 3 +- 11 files changed, 339 insertions(+), 146 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/AppMainServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/AppMainServer.java index 60559d908b8..714033b0b64 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/AppMainServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/AppMainServer.java @@ -102,10 +102,10 @@ public void onMessage(WebAppSocket conn, String msg) { WebSocketServer processServer = subWebSocketServer.get(messagereceived.target); if (processServer != null) { - LOG.info("target {} received.", messagereceived.target); + LOG.debug("server {} received.", messagereceived.target); processServer.onMessage(conn, msg); } else { - LOG.info("target {} received.", AppMainServer.class.toString()); + LOG.debug("server {} received.", AppMainServer.class.toString()); } /** Lets be elegant here */ @@ -202,6 +202,7 @@ protected void broadcast(String key, Message m) { conn.send(serializeMessage(m)); } catch (IOException e) { LOG.error("socket error", e); + removeConnectionFromAllKey(conn); } } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java index 6cc2d811427..be53ef2a96e 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java @@ -35,6 +35,21 @@ * Zeppelin websocket service. */ public class JobMangerServer extends AppMainServer implements WebSocketServer, Observer{ + + /** + * Job manager service type + */ + protected enum JOB_MANAGER_SERVICE { + JOB_MANAGER_PAGE("JOB_MANAGER_PAGE"); + private String serviceTypeKey; + JOB_MANAGER_SERVICE(String serviceType) { + this.serviceTypeKey = serviceType; + } + String getKey() { + return this.serviceTypeKey; + } + } + private static final Logger LOG = LoggerFactory.getLogger(JobMangerServer.class); private Notebook notebook() { @@ -75,14 +90,17 @@ public void onMessage(WebAppSocket conn, String msg) { userAndRoles.addAll(roles); } } - LOG.info("lcs notebook received"); + /** Lets be elegant here */ switch (messagereceived.op) { case LIST_NOTEBOOK_JOBS: - unicastNotebookJobInfo(conn); + sendNotebookJobInfo(conn); break; case LIST_UPDATE_NOTEBOOK_JOBS: - unicastUpdateNotebookJobInfo(conn, messagereceived); + sendUpdateNotebookJobInfo(conn, messagereceived); + break; + case UNSUBSCRIBE_JOBMANAGER: + unsubscribeJobManager(conn); break; default: break; @@ -92,6 +110,15 @@ public void onMessage(WebAppSocket conn, String msg) { } } + + private void addConnectionToJobManagerService(String serviceType, WebAppSocket socket) { + addConnectionToKey(serviceType, socket); + } + + private void removeConnectionToJobManagerService(WebAppSocket socket) { + removeConnectionFromAllKey(socket); + } + protected Message deserializeMessage(String msg) { return gson.fromJson(msg, Message.class); } @@ -100,19 +127,25 @@ protected String serializeMessage(Message m) { return gson.toJson(m); } + public void unsubscribeJobManager(WebAppSocket conn) { + LOG.info("unsubscribe"); + removeConnectionToJobManagerService(conn); + } + + public void sendNotebookJobInfo(WebAppSocket conn) throws IOException { - public void unicastNotebookJobInfo(WebAppSocket conn) { List> notebookJobs = generateNotebooksJobInfo(false); Map response = new HashMap<>(); response.put("lastResponseUnixTime", System.currentTimeMillis()); response.put("jobs", notebookJobs); - unicast(new Message(OP.LIST_NOTEBOOK_JOBS).put("notebookJobs", response), conn); + conn.send(serializeMessage(new Message(OP.LIST_NOTEBOOK_JOBS) + .put("notebookJobs", response))); } - public void unicastUpdateNotebookJobInfo(WebAppSocket conn, Message fromMessage) { - LOG.info("update time {}", fromMessage); + public void sendUpdateNotebookJobInfo(WebAppSocket conn, Message fromMessage) + throws IOException { double lastUpdateUnixTimeRaw = (double) fromMessage.get("lastUpdateUnixTime"); long lastUpdateUnixTime = new Double(lastUpdateUnixTimeRaw).longValue(); List> notebookJobs; @@ -122,10 +155,53 @@ public void unicastUpdateNotebookJobInfo(WebAppSocket conn, Message fromMessage) response.put("lastResponseUnixTime", System.currentTimeMillis()); response.put("jobs", notebookJobs); - unicast(new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS) - .put("notebookRunningJobs", response), conn); + conn.send(serializeMessage(new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS) + .put("notebookRunningJobs", response))); + } + + private Map getParagraphPacketItem(Paragraph paragraph) { + Map paragraphItem = new HashMap<>(); + + // set paragraph id + paragraphItem.put("id", paragraph.getId()); + + // set paragraph name + String paragraphName = paragraph.getTitle(); + if (paragraphName != null) { + paragraphItem.put("name", paragraphName); + } else { + paragraphItem.put("name", paragraph.getId()); + } + + // set status for paragraph. + paragraphItem.put("status", paragraph.getStatus().toString()); + + return paragraphItem; } + private long getUnixTimeLastRunParagraph(Paragraph paragraph) { + + Date lastRunningDate = null; + long lastRunningUnixTime = 0; + + Date paragaraphDate = paragraph.getDateStarted(); + if (paragaraphDate == null) { + paragaraphDate = paragraph.getDateCreated(); + } + + // set last update unixtime(ms). + if (lastRunningDate == null) { + lastRunningDate = paragaraphDate; + } else { + if (lastRunningDate.after(paragaraphDate) == true) { + lastRunningDate = paragaraphDate; + } + } + + lastRunningUnixTime = lastRunningDate.getTime(); + + return lastRunningUnixTime; + } public List> generateNotebooksJobInfo(boolean needsReload) { Notebook notebook = notebook(); @@ -159,7 +235,7 @@ public List> generateNotebooksJobInfo(boolean needsReload) { if (notebookName != null) { info.put("notebookName", note.getName()); } else { - info.put("notebookName", note.id()); + info.put("notebookName", "Note " + note.id()); } if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true @@ -170,7 +246,6 @@ public List> generateNotebooksJobInfo(boolean needsReload) { info.put("notebookType", "normal"); } - Date lastRunningDate = null; long lastRunningUnixTime = 0; List> paragraphsInfo = new LinkedList<>(); @@ -178,36 +253,10 @@ public List> generateNotebooksJobInfo(boolean needsReload) { if (paragraph.getStatus().isRunning() == true) { isNotebookRunning = true; } - Map paragraphItem = new HashMap<>(); - // set paragraph id - paragraphItem.put("id", paragraph.getId()); - - // set paragraph name - String paragraphName = paragraph.getTitle(); - if (paragraphName != null) { - paragraphItem.put("name", paragraphName); - } else { - paragraphItem.put("name", paragraph.getId()); - } - - // set status for paragraph. - paragraphItem.put("status", paragraph.getStatus().toString()); - // get last update time. - Date paragaraphDate = paragraph.getDateStarted(); - if (paragaraphDate == null) { - paragaraphDate = paragraph.getDateCreated(); - } - if (lastRunningDate == null) { - lastRunningDate = paragaraphDate; - } else { - if (lastRunningDate.after(paragaraphDate) == true) { - lastRunningDate = paragaraphDate; - } - } + Map paragraphItem = getParagraphPacketItem(paragraph); - // convert date to unixtime(ms). - lastRunningUnixTime = lastRunningDate.getTime(); + lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph); paragraphsInfo.add(paragraphItem); } @@ -278,7 +327,6 @@ public List> generateUpdateNotebooksJobInfo( info.put("notebookType", "normal"); } - Date lastRunningDate = null; long lastRunningUnixTime = 0; List> paragraphsInfo = new LinkedList<>(); @@ -299,36 +347,10 @@ public List> generateUpdateNotebooksJobInfo( isUpdateNotebook = true; } - Map paragraphItem = new HashMap<>(); - - // set paragraph id - paragraphItem.put("id", paragraph.getId()); + Map paragraphItem = getParagraphPacketItem(paragraph); - // set paragraph name - String paragraphName = paragraph.getTitle(); - if (paragraphName != null) { - paragraphItem.put("name", paragraphName); - } else { - paragraphItem.put("name", paragraph.getId()); - } + lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph); - // set status for paragraph - paragraphItem.put("status", paragraph.getStatus().toString()); - - Date paragaraphDate = startedDate; - if (paragaraphDate == null) { - paragaraphDate = createdDate; - } - - // set last update unixtime(ms). - if (lastRunningDate == null) { - lastRunningDate = paragaraphDate; - } else { - if (lastRunningDate.after(paragaraphDate) == true) { - lastRunningDate = paragaraphDate; - } - } - lastRunningUnixTime = lastRunningDate.getTime(); if (paragraph.getStatus().isRunning() == true) { isNotebookRunning = true; isUpdateNotebook = true; @@ -361,16 +383,116 @@ public List> generateUpdateNotebooksJobInfo( return notesInfo; } + public boolean broadUpdateNote(String noteId) { + + Note note = notebook().getNote(noteId); + + if (note == null) { + LOG.info("broadUpdateNote - not found note"); + return false; + } + + List> notesList = new LinkedList<>(); + + Map noteItem = new HashMap<>(); + + noteItem.put("notebookId", note.id()); + String notebookName = note.getName(); + if (notebookName != null) { + noteItem.put("notebookName", note.getName()); + } else { + noteItem.put("notebookName", note.id()); + } + + // set const keyword for cron type + String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; + if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true + && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { + noteItem.put("notebookType", "cron"); + } + else { + noteItem.put("notebookType", "normal"); + } + + + long lastRunningUnixTime = 0; + boolean isNotebookRunning = false; + List> paragraphsInfo = new LinkedList<>(); + for (Paragraph paragraph : note.getParagraphs()) { + Map paragraphItem = getParagraphPacketItem(paragraph); + + if (paragraph.getStatus().isRunning() == true) { + isNotebookRunning = true; + } + + lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph); + + paragraphsInfo.add(paragraphItem); + } + + // Interpreter is set does not exist. + String interpreterGroupName = null; + if (note.getNoteReplLoader().getInterpreterSettings() != null + && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { + interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); + } + + // set notebook root information. + noteItem.put("interpreter", interpreterGroupName); + noteItem.put("isRunningJob", isNotebookRunning); + noteItem.put("unixTimeLastRun", lastRunningUnixTime); + noteItem.put("paragraphs", paragraphsInfo); + notesList.add(noteItem); + + Map response = new HashMap<>(); + + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", notesList); + + broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), + new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response)); + + return true; + } + + public boolean broadRemovedNote(String noteId) { + + List> notesList = new LinkedList<>(); + Map noteItem = new HashMap<>(); + + // set notebook root information. + noteItem.put("notebookId", noteId); + noteItem.put("isRemoved", true); + notesList.add(noteItem); + + Map response = new HashMap<>(); + + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", notesList); + + broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), + new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response)); + + return true; + } + /** * Notebook Observer Event Listener */ @Override public void update(Observable observer, Object notebookChnagedEvent) { - NotebookChnagedEvent noteEvent = (NotebookChnagedEvent) notebookChnagedEvent; - LOG.info("event note {}", noteEvent.getNoteId()); - + LOG.info("event note {}-{}", noteEvent.getNoteId(), noteEvent.getAction().name()); + try { + if (noteEvent.getAction() == NotebookEventObserver.ACTIONS.REMOVED) { + broadRemovedNote(noteEvent.getNoteId()); + } else { + broadUpdateNote(noteEvent.getNoteId()); + } + } catch (Exception e) { + LOG.info("socket error job {}", e.getMessage()); + } } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 151249083d7..5b571b8223e 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -100,7 +100,6 @@ public void onMessage(WebAppSocket conn, String msg) { userAndRoles.addAll(roles); } } - LOG.info("lcs notebook received"); /** Lets be elegant here */ switch (messagereceived.op) { case LIST_NOTES: diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index b2e07f22d97..ca325d7f54d 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -21,17 +21,10 @@ angular.module('zeppelinWebApp') var FILTER_VALUE_INTERPRETER = filterConfig.FILTER_VALUE_INTERPRETER; var FILTER_VALUE_NOTEBOOK_NAME = filterConfig.FILTER_VALUE_NOTEBOOK_NAME; var RUNNING_ALWAYS_TOP = filterConfig.RUNNING_ALWAYS_TOP; + var SORT_BY_ASC = filterConfig.SORT_BY_ASC; var filterItems = jobItems; - if (RUNNING_ALWAYS_TOP === true) { - var runningJobList = _.where(filterItems, {isRunningJob : true}); - filterItems = _.reject(filterItems, {isRunningJob : true}); - runningJobList.map(function (runningJob) { - filterItems.splice(0,0, runningJob); - }); - } - if (FILTER_VALUE_INTERPRETER === undefined) { filterItems = _.filter(filterItems, function (jobItem) { return jobItem.interpreter === undefined? true : false; @@ -47,17 +40,36 @@ angular.module('zeppelinWebApp') return lowerNotebookName.match(new RegExp('.*'+lowerFilterValue+'.*')); }); } + + if (SORT_BY_ASC === true) { + filterItems = _.sortBy(filterItems, function (sortItem) { + return sortItem.notebookName; + }); + } else { + filterItems = _.sortBy(filterItems, function (sortItem) { + return sortItem.notebookName; + }); + filterItems = filterItems.reverse(); + } + + if (RUNNING_ALWAYS_TOP === true) { + var runningJobList = _.where(filterItems, {isRunningJob : true}); + filterItems = _.reject(filterItems, {isRunningJob : true}); + runningJobList.map(function (runningJob) { + filterItems.splice(0,0, runningJob); + }); + } + return filterItems; } return filterContext; }) .controller('JobmanagerCtrl', - function($scope, $route, $routeParams, $location, $rootScope, $http, + function($scope, $route, $routeParams, $location, $rootScope, $http, $q, websocketMsgSrv, baseUrlSrv, $interval, $timeout, SaveAsService, myJobFilter) { $scope.$on('setNotebookJobs', function(event, responseData) { $scope.lastJobServerUnixTime = responseData.lastResponseUnixTime; - console.log('recev time ', $scope.lastJobServerUnixTime); $scope.jobInfomations = responseData.jobs; $scope.jobInfomationsIndexs = $scope.jobInfomations? _.indexBy($scope.jobInfomations, 'notebookId') : {}; $scope.JobInfomationsByFilter = $scope.jobTypeFilter($scope.jobInfomations, $scope.filterConfig); @@ -74,6 +86,7 @@ angular.module('zeppelinWebApp') value : interpreterLists[index] }); } + $scope.doFiltering($scope.jobInfomations, $scope.filterConfig); }); $scope.$on('setUpdateNotebookJobs', function(event, responseData) { @@ -86,34 +99,57 @@ angular.module('zeppelinWebApp') var newItem = angular.copy(changedItem); jobInfomations.push(newItem); indexStore[changedItem.notebookId] = newItem; - console.log('new Item Notebook Job ', newItem); } else { var changeOriginTarget = indexStore[changedItem.notebookId]; - changeOriginTarget.isRunningJob = changedItem.isRunningJob; - changeOriginTarget.notebookName = changedItem.notebookName; - changeOriginTarget.notebookType = changedItem.notebookType; - changeOriginTarget.unixTimeLastRun = changedItem.unixTimeLastRun; - changeOriginTarget.paragraphs = changedItem.paragraphs; - console.log('new Event for Notebook job ', changedItem.notebookName); + + if (changedItem.isRemoved !== undefined && changedItem.isRemoved === true) { + // remove Item. + delete indexStore[changedItem.notebookId]; + var removeIndex = _.findIndex(indexStore, changedItem.notebookId); + indexStore.splice(removeIndex, 1); + removeIndex = _.findIndex(jobInfomations, { 'notebookId' : changedItem.notebookId}); + indexStore.splice(removeIndex, 1); + } else { + // change value for item. + changeOriginTarget.isRunningJob = changedItem.isRunningJob; + changeOriginTarget.notebookName = changedItem.notebookName; + changeOriginTarget.notebookType = changedItem.notebookType; + changeOriginTarget.interpreter = changedItem.interpreter; + changeOriginTarget.unixTimeLastRun = changedItem.unixTimeLastRun; + changeOriginTarget.paragraphs = changedItem.paragraphs; + } } $scope.doFiltering(jobInfomations, $scope.filterConfig); }); }); $scope.doFiltering = function (jobInfomations, filterConfig) { - $scope.JobInfomationsByFilter = $scope.jobTypeFilter(jobInfomations, filterConfig); + asyncNotebookJobFilter(jobInfomations, filterConfig).then( + function () { + // success + $scope.isLoadingFilter = false; + }, + function (){ + // failed + }); }; - $scope.onChangeRunJobToAlwaysTopToggle = function (jobInfomations, filterConfig) { - $scope.isRunJobToAlwaysTopToggle = !$scope.isRunJobToAlwaysTopToggle; - $scope.doFiltering(jobInfomations, filterConfig); + $scope.onChangeRunJobToAlwaysTopToggle = function () { + $scope.filterConfig.RUNNING_ALWAYS_TOP = !$scope.filterConfig.RUNNING_ALWAYS_TOP; + $scope.doFiltering($scope.jobInfomations, $scope.filterConfig); + }; + + $scope.onChangeSortAsc = function () { + $scope.filterConfig.SORT_BY_ASC = !$scope.filterConfig.SORT_BY_ASC; + $scope.doFiltering($scope.jobInfomations, $scope.filterConfig); }; $scope.doFilterInputTyping = function (keyEvent, jobInfomations, filterConfig) { var returnKey = 13; - console.log('key ', keyEvent.which); $timeout.cancel($scope.dofilterTimeoutObject); - $scope.dofilterTimeoutObject = $timeout(function(){$scope.doFiltering(jobInfomations, filterConfig);}, 1000); + $scope.dofilterTimeoutObject = $timeout(function(){ + $scope.doFiltering(jobInfomations, filterConfig); + }, 1000); if (keyEvent.which === returnKey) { $timeout.cancel($scope.dofilterTimeoutObject); $scope.doFiltering(jobInfomations, filterConfig); @@ -121,10 +157,12 @@ angular.module('zeppelinWebApp') }; $scope.init = function () { + $scope.isLoadingFilter = true; $scope.filterConfig = { RUNNING_ALWAYS_TOP : true, FILTER_VALUE_NOTEBOOK_NAME : '', - FILTER_VALUE_INTERPRETER : '*' + FILTER_VALUE_INTERPRETER : '*', + SORT_BY_ASC : true }; $scope.jobTypeFilter = myJobFilter; $scope.jobInfomations = []; @@ -136,9 +174,17 @@ angular.module('zeppelinWebApp') websocketMsgSrv.getUpdateNotebookJobsList($scope.lastJobServerUnixTime); } }, 1000); + $scope.$on('$destroy', function() { $interval.cancel(refreshObj); + websocketMsgSrv.unsubscribeJobManager(); }); + }; + var asyncNotebookJobFilter = function (jobInfomations, filterConfig) { + return $q(function(resolve, reject) { + $scope.JobInfomationsByFilter = $scope.jobTypeFilter(jobInfomations, filterConfig); + resolve($scope.JobInfomationsByFilter); + }); }; }); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.html b/zeppelin-web/src/app/jobmanager/jobmanager.html index 443f7a8ff8d..75371cc637b 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.html +++ b/zeppelin-web/src/app/jobmanager/jobmanager.html @@ -32,22 +32,37 @@

    -
    -
    - - -
    +
    +
    + + + + + + +
    -
    -
    - - Running the notebook always on top - - -
    -
    +
    +
    +  Loading... +
    +
    diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.html b/zeppelin-web/src/app/jobmanager/jobs/job.html index f6bd74c783b..7bc8f1547fb 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job.html +++ b/zeppelin-web/src/app/jobmanager/jobs/job.html @@ -23,9 +23,9 @@   - + - {{::notebookJob.notebookName}} + {{notebookJob.notebookName}}  -  @@ -35,7 +35,7 @@ Interpreter is not set - {{::notebookJob.interpreter}} + {{notebookJob.interpreter}} @@ -48,51 +48,51 @@ ng-switch="paragraphJob.status"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{paragraphJob.id}}"> + tooltip="{{paragraphJob.name}} is READY"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{paragraphJob.id}}"> + tooltip="{{paragraphJob.name}} is FINISHED"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{paragraphJob.id}}"> + tooltip="{{paragraphJob.name}} is ABORT"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{paragraphJob.id}}"> + tooltip="{{paragraphJob.name}} is ERROR"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{paragraphJob.id}}"> + tooltip="{{paragraphJob.name}} is PENDING"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{paragraphJob.id}}"> + ng-href="#/notebook/{{notebookJob.notebookId}}/paragraph/{{paragraphJob.id}}"> + tooltip="{{paragraphJob.name}} is {{paragraphJob.status}}"> diff --git a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js index 8042910f598..3b908a0004b 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js +++ b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js @@ -174,7 +174,11 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, {op: 'LIST_UPDATE_NOTEBOOK_JOBS', data : {lastUpdateUnixTime : lastUpdateServerUnixTime*1}}, 'jobManagerServer' ); - } + }, + + unsubscribeJobManager: function() { + websocketEvents.sendNewEvent({op: 'UNSUBSCRIBE_JOBMANAGER'}, 'jobManagerServer'); + }, }; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 0a79d38045d..a96c5568cec 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -128,6 +128,14 @@ public void setName(String name) { notifyChanged(NotebookEventObserver.ACTIONS.CHNAGED_NOTE_NAME); } + public NotebookEventObserver getNotebookEventObserver() { + return notebookEventObserver; + } + + public void setNotebookEventObserver(NotebookEventObserver notebookEventObserver) { + this.notebookEventObserver = notebookEventObserver; + } + public NoteInterpreterLoader getNoteReplLoader() { return replLoader; } @@ -408,7 +416,6 @@ public void run(String paragraphId) { intp.getScheduler().submit(p); } - notifyChanged(NotebookEventObserver.ACTIONS.RUN_PARAGRAPH); } public List completion(String paragraphId, String buffer, int cursor) { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index d91577f88bd..14e3d807c9d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -357,7 +357,7 @@ private Note loadNoteFromRepo(String id) { note.setJobListenerFactory(jobListenerFactory); note.setNotebookRepo(notebookRepo); - + note.setNotebookEventObserver(notebookEventObserver); Map angularObjectSnapshot = new HashMap<>(); // restore angular object -------------- @@ -512,7 +512,6 @@ public void execute(JobExecutionContext context) throws JobExecutionException { Note note = notebook.getNote(noteId); NotebookEventObserver notebookEventObserver = notebook.getNotebookEventObserver(); - notebookEventObserver.notifyChanged(note.id(), NotebookEventObserver.ACTIONS.RUN); note.runAll(); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventObserver.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventObserver.java index a33c2f3e4f3..e1230e48379 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventObserver.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventObserver.java @@ -28,7 +28,7 @@ public class NotebookEventObserver extends Observable { * Notebook event Enum */ public static enum ACTIONS { - REMOVED, CREATE, CLONE, RUN, MOVE, + REMOVED, CREATE, RUN, BIND_INTERPRETER, CHANGED_CONFIG, CHNAGED_NOTE_NAME, ADD_PARAGRAPH, MOVED_PARAGRAPH, RUN_PARAGRAPH } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java index 40cdebd922b..9fbca710c9c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java @@ -115,8 +115,9 @@ public static enum OP { // @param noteId // @param checkpointName LIST_NOTEBOOK_JOBS, // [c-s] get notebook job management infomations - LIST_UPDATE_NOTEBOOK_JOBS // [c-s] get job management informations for until unixtime + LIST_UPDATE_NOTEBOOK_JOBS, // [c-s] get job management informations for until unixtime // @param unixTime + UNSUBSCRIBE_JOBMANAGER // [c-s] Unsubscribe notification for job manger. } From 97e8bbea32ba9b0c15d20c1299e176f83fe0340c Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Sat, 4 Jun 2016 11:22:57 +0900 Subject: [PATCH 21/25] remove class job manager server --- bin/derby.log | 4604 +++++++++++++++++ .../zeppelin/server/ZeppelinServer.java | 43 +- .../zeppelin/socket/JobMangerServer.java | 499 -- .../zeppelin/socket/NotebookServer.java | 389 +- .../app/jobmanager/jobmanager.controller.js | 12 +- 5 files changed, 5004 insertions(+), 543 deletions(-) create mode 100644 bin/derby.log delete mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java diff --git a/bin/derby.log b/bin/derby.log new file mode 100644 index 00000000000..5c9a5ff754d --- /dev/null +++ b/bin/derby.log @@ -0,0 +1,4604 @@ +Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Cleanup action completed +Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 100 more +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + ... 97 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 97 more +============= begin nested exception, level (1) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 102 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 97 more +============= end nested exception, level (1) =========== +============= begin nested exception, level (2) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 97 more +============= end nested exception, level (2) =========== +============= begin nested exception, level (3) =========== +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +============= end nested exception, level (3) =========== +Cleanup action completed +Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Cleanup action completed +Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 100 more +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + ... 97 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 97 more +============= begin nested exception, level (1) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 102 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 97 more +============= end nested exception, level (1) =========== +============= begin nested exception, level (2) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 97 more +============= end nested exception, level (2) =========== +============= begin nested exception, level (3) =========== +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +============= end nested exception, level (3) =========== +Cleanup action completed +Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Cleanup action completed +Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 100 more +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + ... 97 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 97 more +============= begin nested exception, level (1) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 102 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 97 more +============= end nested exception, level (1) =========== +============= begin nested exception, level (2) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 97 more +============= end nested exception, level (2) =========== +============= begin nested exception, level (3) =========== +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +============= end nested exception, level (3) =========== +Cleanup action completed +Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Cleanup action completed +Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 100 more +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + ... 97 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 97 more +============= begin nested exception, level (1) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 102 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 97 more +============= end nested exception, level (1) =========== +============= begin nested exception, level (2) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 97 more +============= end nested exception, level (2) =========== +============= begin nested exception, level (3) =========== +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) + at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) + at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +============= end nested exception, level (3) =========== +Cleanup action completed +Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Cleanup action completed +Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 97 more +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + ... 94 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 94 more +============= begin nested exception, level (1) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 99 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 94 more +============= end nested exception, level (1) =========== +============= begin nested exception, level (2) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 94 more +============= end nested exception, level (2) =========== +============= begin nested exception, level (3) =========== +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +============= end nested exception, level (3) =========== +Cleanup action completed +Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Cleanup action completed +Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 97 more +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + ... 94 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 94 more +============= begin nested exception, level (1) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 99 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 94 more +============= end nested exception, level (1) =========== +============= begin nested exception, level (2) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 94 more +============= end nested exception, level (2) =========== +============= begin nested exception, level (3) =========== +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +============= end nested exception, level (3) =========== +Cleanup action completed +Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Cleanup action completed +Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 97 more +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + ... 94 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 94 more +============= begin nested exception, level (1) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 99 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 94 more +============= end nested exception, level (1) =========== +============= begin nested exception, level (2) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 94 more +============= end nested exception, level (2) =========== +============= begin nested exception, level (3) =========== +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +============= end nested exception, level (3) =========== +Cleanup action completed +Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Cleanup action completed +Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting +java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 97 more +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + ... 94 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 94 more +============= begin nested exception, level (1) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + ... 99 more +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 94 more +============= end nested exception, level (1) =========== +============= begin nested exception, level (2) =========== +java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) + at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) + at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + ... 94 more +============= end nested exception, level (2) =========== +============= begin nested exception, level (3) =========== +ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. + at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) + at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) + at java.security.AccessController.doPrivileged(Native Method) + at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) + at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) + at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) + at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) + at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) + at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) + at org.apache.derby.jdbc.Driver20.connect(Unknown Source) + at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) + at java.sql.DriverManager.getConnection(DriverManager.java:664) + at java.sql.DriverManager.getConnection(DriverManager.java:208) + at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) + at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) + at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) + at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) + at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) + at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) + at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) + at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) + at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:497) + at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) + at java.security.AccessController.doPrivileged(Native Method) + at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) + at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) + at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) + at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) + at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) + at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) + at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) + at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) + at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) + at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) + at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) + at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) + at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) + at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) + at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) + at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) + at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) + at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) + at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) + at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) + at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) + at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) + at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) + at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) + at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) + at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) + at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.lang.reflect.Constructor.newInstance(Constructor.java:422) + at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) + at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) + at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) + at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) + at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) + at org.apache.zeppelin.scheduler.Job.run(Job.java:176) + at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) + at java.util.concurrent.FutureTask.run(FutureTask.java:266) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +============= end nested exception, level (3) =========== +Cleanup action completed diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index f70e31465e9..c097fef82af 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -31,9 +31,7 @@ import org.apache.zeppelin.search.LuceneSearch; import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.socket.AppMainServer; -import org.apache.zeppelin.socket.JobMangerServer; import org.apache.zeppelin.socket.NotebookServer; -import org.apache.zeppelin.socket.WebSocketServer; import org.apache.zeppelin.user.Credentials; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.server.*; @@ -49,7 +47,6 @@ import org.slf4j.LoggerFactory; import javax.servlet.DispatcherType; -import javax.servlet.Servlet; import javax.ws.rs.core.Application; import java.io.File; import java.io.IOException; @@ -108,17 +105,8 @@ public static void main(String[] args) throws InterruptedException { // REST api setupRestApiContextHandler(webApp, conf); - // Main WS server - AppMainServer mainWsServer = setupWSAppMainServer(webApp, conf); - - // Notebook WS server + // Main Notebook WS server notebookWsServer = setupNotebookServer(webApp, conf); - mainWsServer.setSubWebSocketServer("notebookServer", notebookWsServer); - - // Job Manager WS server - JobMangerServer jobManagerServer = setupJobManagerServer(webApp, conf); - mainWsServer.setSubWebSocketServer("jobManagerServer", jobManagerServer); - //Below is commented since zeppelin-docs module is removed. //final WebAppContext webAppSwagg = setupWebAppSwagger(conf); @@ -133,7 +121,7 @@ public static void main(String[] args) throws InterruptedException { LOG.info("Done, zeppelin server started"); // register observer for job manager. - notebook.getNotebookEventObserver().addObserver(jobManagerServer); + notebook.getNotebookEventObserver().addObserver(notebookWsServer); Runtime.getRuntime().addShutdownHook(new Thread(){ @Override public void run() { @@ -208,19 +196,6 @@ private static Server setupJettyServer(ZeppelinConfiguration conf) { return server; } - private static AppMainServer setupWSAppMainServer(WebAppContext webapp, - ZeppelinConfiguration conf) { - AppMainServer appMainWsServer = new AppMainServer(); - String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); - final ServletHolder servletHolder = new ServletHolder(appMainWsServer); - servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); - final ServletContextHandler cxfContext = new ServletContextHandler( - ServletContextHandler.SESSIONS); - webapp.addServlet(servletHolder, "/ws/*"); - - return appMainWsServer; - } - private static NotebookServer setupNotebookServer(WebAppContext webapp, ZeppelinConfiguration conf) { NotebookServer notebookWsServer = new NotebookServer(); @@ -229,22 +204,10 @@ private static NotebookServer setupNotebookServer(WebAppContext webapp, servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); final ServletContextHandler cxfContext = new ServletContextHandler( ServletContextHandler.SESSIONS); - webapp.addServlet(servletHolder, "/notebook/*"); + webapp.addServlet(servletHolder, "/ws/*"); return notebookWsServer; } - private static JobMangerServer setupJobManagerServer(WebAppContext webapp, - ZeppelinConfiguration conf) { - JobMangerServer jobMangerWsServer = new JobMangerServer(); - String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize(); - final ServletHolder servletHolder = new ServletHolder(jobMangerWsServer); - servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); - final ServletContextHandler cxfContext = new ServletContextHandler( - ServletContextHandler.SESSIONS); - webapp.addServlet(servletHolder, "/jobmanager/*"); - return jobMangerWsServer; - } - private static SslContextFactory getSslContextFactory(ZeppelinConfiguration conf) { SslContextFactory sslContextFactory = new SslContextFactory(); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java deleted file mode 100644 index be53ef2a96e..00000000000 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/JobMangerServer.java +++ /dev/null @@ -1,499 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.zeppelin.socket; - -import com.google.gson.reflect.TypeToken; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import org.apache.zeppelin.notebook.*; -import org.apache.zeppelin.notebook.socket.Message; -import org.apache.zeppelin.notebook.socket.Message.OP; -import org.apache.zeppelin.notebook.NotebookEventObserver.NotebookChnagedEvent; -import org.apache.zeppelin.server.ZeppelinServer; -import org.apache.zeppelin.ticket.TicketContainer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.*; - -/** - * Zeppelin websocket service. - */ -public class JobMangerServer extends AppMainServer implements WebSocketServer, Observer{ - - /** - * Job manager service type - */ - protected enum JOB_MANAGER_SERVICE { - JOB_MANAGER_PAGE("JOB_MANAGER_PAGE"); - private String serviceTypeKey; - JOB_MANAGER_SERVICE(String serviceType) { - this.serviceTypeKey = serviceType; - } - String getKey() { - return this.serviceTypeKey; - } - } - - private static final Logger LOG = LoggerFactory.getLogger(JobMangerServer.class); - - private Notebook notebook() { - return ZeppelinServer.notebook; - } - - @Override - public void onMessage(WebAppSocket conn, String msg) { - Notebook notebook = notebook(); - try { - Message messagereceived = deserializeMessage(msg); - LOG.debug("RECEIVE << " + messagereceived.op); - LOG.debug("RECEIVE PRINCIPAL << " + messagereceived.principal); - LOG.debug("RECEIVE TICKET << " + messagereceived.ticket); - LOG.debug("RECEIVE ROLES << " + messagereceived.roles); - - if (LOG.isTraceEnabled()) { - LOG.trace("RECEIVE MSG = " + messagereceived); - } - - String ticket = TicketContainer.instance.getTicket(messagereceived.principal); - if (ticket != null && !ticket.equals(messagereceived.ticket)) - throw new Exception("Invalid ticket " + messagereceived.ticket + " != " + ticket); - - ZeppelinConfiguration conf = ZeppelinConfiguration.create(); - boolean allowAnonymous = conf. - getBoolean(ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED); - if (!allowAnonymous && messagereceived.principal.equals("anonymous")) { - throw new Exception("Anonymous access not allowed "); - } - - HashSet userAndRoles = new HashSet(); - userAndRoles.add(messagereceived.principal); - if (!messagereceived.roles.equals("")) { - HashSet roles = gson.fromJson(messagereceived.roles, - new TypeToken>(){}.getType()); - if (roles != null) { - userAndRoles.addAll(roles); - } - } - - /** Lets be elegant here */ - switch (messagereceived.op) { - case LIST_NOTEBOOK_JOBS: - sendNotebookJobInfo(conn); - break; - case LIST_UPDATE_NOTEBOOK_JOBS: - sendUpdateNotebookJobInfo(conn, messagereceived); - break; - case UNSUBSCRIBE_JOBMANAGER: - unsubscribeJobManager(conn); - break; - default: - break; - } - } catch (Exception e) { - LOG.error("Can't handle message", e); - } - } - - - private void addConnectionToJobManagerService(String serviceType, WebAppSocket socket) { - addConnectionToKey(serviceType, socket); - } - - private void removeConnectionToJobManagerService(WebAppSocket socket) { - removeConnectionFromAllKey(socket); - } - - protected Message deserializeMessage(String msg) { - return gson.fromJson(msg, Message.class); - } - - protected String serializeMessage(Message m) { - return gson.toJson(m); - } - - public void unsubscribeJobManager(WebAppSocket conn) { - LOG.info("unsubscribe"); - removeConnectionToJobManagerService(conn); - } - - public void sendNotebookJobInfo(WebAppSocket conn) throws IOException { - - List> notebookJobs = generateNotebooksJobInfo(false); - Map response = new HashMap<>(); - - response.put("lastResponseUnixTime", System.currentTimeMillis()); - response.put("jobs", notebookJobs); - - conn.send(serializeMessage(new Message(OP.LIST_NOTEBOOK_JOBS) - .put("notebookJobs", response))); - } - - public void sendUpdateNotebookJobInfo(WebAppSocket conn, Message fromMessage) - throws IOException { - double lastUpdateUnixTimeRaw = (double) fromMessage.get("lastUpdateUnixTime"); - long lastUpdateUnixTime = new Double(lastUpdateUnixTimeRaw).longValue(); - List> notebookJobs; - notebookJobs = generateUpdateNotebooksJobInfo(false, lastUpdateUnixTime); - Map response = new HashMap<>(); - - response.put("lastResponseUnixTime", System.currentTimeMillis()); - response.put("jobs", notebookJobs); - - conn.send(serializeMessage(new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS) - .put("notebookRunningJobs", response))); - } - - private Map getParagraphPacketItem(Paragraph paragraph) { - Map paragraphItem = new HashMap<>(); - - // set paragraph id - paragraphItem.put("id", paragraph.getId()); - - // set paragraph name - String paragraphName = paragraph.getTitle(); - if (paragraphName != null) { - paragraphItem.put("name", paragraphName); - } else { - paragraphItem.put("name", paragraph.getId()); - } - - // set status for paragraph. - paragraphItem.put("status", paragraph.getStatus().toString()); - - return paragraphItem; - } - - private long getUnixTimeLastRunParagraph(Paragraph paragraph) { - - Date lastRunningDate = null; - long lastRunningUnixTime = 0; - - Date paragaraphDate = paragraph.getDateStarted(); - if (paragaraphDate == null) { - paragaraphDate = paragraph.getDateCreated(); - } - - // set last update unixtime(ms). - if (lastRunningDate == null) { - lastRunningDate = paragaraphDate; - } else { - if (lastRunningDate.after(paragaraphDate) == true) { - lastRunningDate = paragaraphDate; - } - } - - lastRunningUnixTime = lastRunningDate.getTime(); - - return lastRunningUnixTime; - } - - public List> generateNotebooksJobInfo(boolean needsReload) { - Notebook notebook = notebook(); - - ZeppelinConfiguration conf = notebook.getConf(); - String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); - boolean hideHomeScreenNotebookFromList = conf - .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); - - if (needsReload) { - try { - notebook.reloadAllNotes(); - } catch (IOException e) { - LOG.error("Fail to reload notes from repository"); - } - } - - List notes = notebook.getAllNotes(); - List> notesInfo = new LinkedList<>(); - for (Note note : notes) { - boolean isNotebookRunning = false; - Map info = new HashMap<>(); - - if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { - continue; - } - - String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; - info.put("notebookId", note.id()); - String notebookName = note.getName(); - if (notebookName != null) { - info.put("notebookName", note.getName()); - } else { - info.put("notebookName", "Note " + note.id()); - } - - if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true - && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { - info.put("notebookType", "cron"); - } - else { - info.put("notebookType", "normal"); - } - - long lastRunningUnixTime = 0; - - List> paragraphsInfo = new LinkedList<>(); - for (Paragraph paragraph : note.getParagraphs()) { - if (paragraph.getStatus().isRunning() == true) { - isNotebookRunning = true; - } - - Map paragraphItem = getParagraphPacketItem(paragraph); - - lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph); - - paragraphsInfo.add(paragraphItem); - } - - // Interpreter is set does not exist. - String interpreterGroupName = null; - if (note.getNoteReplLoader().getInterpreterSettings() != null - && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { - interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); - } - - // notebook json object root information. - info.put("interpreter", interpreterGroupName); - info.put("isRunningJob", isNotebookRunning); - info.put("unixTimeLastRun", lastRunningUnixTime); - info.put("paragraphs", paragraphsInfo); - notesInfo.add(info); - } - return notesInfo; - } - - - public List> generateUpdateNotebooksJobInfo( - boolean needsReload, long lastUpdateServerUnixTime) { - Notebook notebook = notebook(); - - ZeppelinConfiguration conf = notebook.getConf(); - String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); - boolean hideHomeScreenNotebookFromList = conf - .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); - - if (needsReload) { - try { - notebook.reloadAllNotes(); - } catch (IOException e) { - LOG.error("Fail to reload notes from repository"); - } - } - - List notes = notebook.getAllNotes(); - List> notesInfo = new LinkedList<>(); - for (Note note : notes) { - boolean isNotebookRunning = false; - boolean isUpdateNotebook = false; - - Map info = new HashMap<>(); - - if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { - continue; - } - - // set const keyword for cron type - String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; - info.put("notebookId", note.id()); - String notebookName = note.getName(); - if (notebookName != null) { - info.put("notebookName", note.getName()); - } else { - info.put("notebookName", note.id()); - } - - - if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true - && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { - info.put("notebookType", "cron"); - } - else { - info.put("notebookType", "normal"); - } - - long lastRunningUnixTime = 0; - - List> paragraphsInfo = new LinkedList<>(); - for (Paragraph paragraph : note.getParagraphs()) { - - // check date for update time. - Date startedDate = paragraph.getDateStarted(); - Date createdDate = paragraph.getDateCreated(); - Date finishedDate = paragraph.getDateFinished(); - - if (startedDate != null && startedDate.getTime() > lastUpdateServerUnixTime) { - isUpdateNotebook = true; - } - if (createdDate != null && createdDate.getTime() > lastUpdateServerUnixTime) { - isUpdateNotebook = true; - } - if (finishedDate != null && finishedDate.getTime() > lastUpdateServerUnixTime) { - isUpdateNotebook = true; - } - - Map paragraphItem = getParagraphPacketItem(paragraph); - - lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph); - - if (paragraph.getStatus().isRunning() == true) { - isNotebookRunning = true; - isUpdateNotebook = true; - } - paragraphsInfo.add(paragraphItem); - } - - // Insert only data that has changed. - if (isUpdateNotebook != true) { - continue; - } - - // Interpreter is set does not exist. - String interpreterGroupName = null; - if (note.getNoteReplLoader().getInterpreterSettings() != null - && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { - interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); - } - - // set notebook root information. - info.put("interpreter", interpreterGroupName); - info.put("isRunningJob", isNotebookRunning); - info.put("unixTimeLastRun", lastRunningUnixTime); - info.put("paragraphs", paragraphsInfo); - notesInfo.add(info); - } - if (notesInfo.size() > 0) { - LOG.info("update count {}", notesInfo.size()); - } - return notesInfo; - } - - public boolean broadUpdateNote(String noteId) { - - Note note = notebook().getNote(noteId); - - if (note == null) { - LOG.info("broadUpdateNote - not found note"); - return false; - } - - List> notesList = new LinkedList<>(); - - Map noteItem = new HashMap<>(); - - noteItem.put("notebookId", note.id()); - String notebookName = note.getName(); - if (notebookName != null) { - noteItem.put("notebookName", note.getName()); - } else { - noteItem.put("notebookName", note.id()); - } - - // set const keyword for cron type - String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; - if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true - && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { - noteItem.put("notebookType", "cron"); - } - else { - noteItem.put("notebookType", "normal"); - } - - - long lastRunningUnixTime = 0; - boolean isNotebookRunning = false; - List> paragraphsInfo = new LinkedList<>(); - for (Paragraph paragraph : note.getParagraphs()) { - Map paragraphItem = getParagraphPacketItem(paragraph); - - if (paragraph.getStatus().isRunning() == true) { - isNotebookRunning = true; - } - - lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph); - - paragraphsInfo.add(paragraphItem); - } - - // Interpreter is set does not exist. - String interpreterGroupName = null; - if (note.getNoteReplLoader().getInterpreterSettings() != null - && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { - interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); - } - - // set notebook root information. - noteItem.put("interpreter", interpreterGroupName); - noteItem.put("isRunningJob", isNotebookRunning); - noteItem.put("unixTimeLastRun", lastRunningUnixTime); - noteItem.put("paragraphs", paragraphsInfo); - notesList.add(noteItem); - - Map response = new HashMap<>(); - - response.put("lastResponseUnixTime", System.currentTimeMillis()); - response.put("jobs", notesList); - - broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), - new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response)); - - return true; - } - - public boolean broadRemovedNote(String noteId) { - - List> notesList = new LinkedList<>(); - Map noteItem = new HashMap<>(); - - // set notebook root information. - noteItem.put("notebookId", noteId); - noteItem.put("isRemoved", true); - notesList.add(noteItem); - - Map response = new HashMap<>(); - - response.put("lastResponseUnixTime", System.currentTimeMillis()); - response.put("jobs", notesList); - - broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), - new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response)); - - return true; - } - - /** - * Notebook Observer Event Listener - */ - @Override - public void update(Observable observer, Object notebookChnagedEvent) { - NotebookChnagedEvent noteEvent = (NotebookChnagedEvent) notebookChnagedEvent; - LOG.info("event note {}-{}", noteEvent.getNoteId(), noteEvent.getAction().name()); - try { - if (noteEvent.getAction() == NotebookEventObserver.ACTIONS.REMOVED) { - broadRemovedNote(noteEvent.getNoteId()); - } else { - broadUpdateNote(noteEvent.getNoteId()); - } - - } catch (Exception e) { - LOG.info("socket error job {}", e.getMessage()); - } - } - -} - diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 5b571b8223e..b456da0e208 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -34,6 +34,7 @@ import org.apache.zeppelin.notebook.*; import org.apache.zeppelin.notebook.socket.Message; import org.apache.zeppelin.notebook.socket.Message.OP; +import org.apache.zeppelin.notebook.NotebookEventObserver.*; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.server.ZeppelinServer; @@ -57,9 +58,23 @@ * Zeppelin websocket service. */ public class NotebookServer extends AppMainServer implements JobListenerFactory, - AngularObjectRegistryListener, RemoteInterpreterProcessListener { + AngularObjectRegistryListener, RemoteInterpreterProcessListener, Observer{ private static final Logger LOG = LoggerFactory.getLogger(NotebookServer.class); + /** + * Job manager service type + */ + protected enum JOB_MANAGER_SERVICE { + JOB_MANAGER_PAGE("JOB_MANAGER_PAGE"); + private String serviceTypeKey; + JOB_MANAGER_SERVICE(String serviceType) { + this.serviceTypeKey = serviceType; + } + String getKey() { + return this.serviceTypeKey; + } + } + final Queue connectedSockets = new ConcurrentLinkedQueue<>(); private Notebook notebook() { @@ -170,6 +185,15 @@ public void onMessage(WebAppSocket conn, String msg) { case CHECKPOINT_NOTEBOOK: checkpointNotebook(conn, notebook, messagereceived); break; + case LIST_NOTEBOOK_JOBS: + sendNotebookJobInfo(conn); + break; + case LIST_UPDATE_NOTEBOOK_JOBS: + sendUpdateNotebookJobInfo(conn, messagereceived); + break; + case UNSUBSCRIBE_JOBMANAGER: + unsubscribeJobManager(conn); + break; default: break; } @@ -263,6 +287,369 @@ public void broadcastReloadedNoteList() { broadcastAll(new Message(OP.NOTES_INFO).put("notes", notesInfo)); } + public void unsubscribeJobManager(WebAppSocket conn) { + LOG.info("unsubscribe"); + removeConnectionFromAllKey(conn); + } + + public void sendNotebookJobInfo(WebAppSocket conn) throws IOException { + + addConnectionToKey(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), conn); + List> notebookJobs = generateNotebooksJobInfo(false); + Map response = new HashMap<>(); + + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", notebookJobs); + + conn.send(serializeMessage(new Message(OP.LIST_NOTEBOOK_JOBS) + .put("notebookJobs", response))); + } + + public void sendUpdateNotebookJobInfo(WebAppSocket conn, Message fromMessage) + throws IOException { + double lastUpdateUnixTimeRaw = (double) fromMessage.get("lastUpdateUnixTime"); + long lastUpdateUnixTime = new Double(lastUpdateUnixTimeRaw).longValue(); + List> notebookJobs; + notebookJobs = generateUpdateNotebooksJobInfo(false, lastUpdateUnixTime); + Map response = new HashMap<>(); + + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", notebookJobs); + + conn.send(serializeMessage(new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS) + .put("notebookRunningJobs", response))); + } + + private Map getParagraphPacketItem(Paragraph paragraph) { + Map paragraphItem = new HashMap<>(); + + // set paragraph id + paragraphItem.put("id", paragraph.getId()); + + // set paragraph name + String paragraphName = paragraph.getTitle(); + if (paragraphName != null) { + paragraphItem.put("name", paragraphName); + } else { + paragraphItem.put("name", paragraph.getId()); + } + + // set status for paragraph. + paragraphItem.put("status", paragraph.getStatus().toString()); + + return paragraphItem; + } + + private long getUnixTimeLastRunParagraph(Paragraph paragraph) { + + Date lastRunningDate = null; + long lastRunningUnixTime = 0; + + Date paragaraphDate = paragraph.getDateStarted(); + if (paragaraphDate == null) { + paragaraphDate = paragraph.getDateCreated(); + } + + // set last update unixtime(ms). + if (lastRunningDate == null) { + lastRunningDate = paragaraphDate; + } else { + if (lastRunningDate.after(paragaraphDate) == true) { + lastRunningDate = paragaraphDate; + } + } + + lastRunningUnixTime = lastRunningDate.getTime(); + + return lastRunningUnixTime; + } + + public List> generateNotebooksJobInfo(boolean needsReload) { + Notebook notebook = notebook(); + + ZeppelinConfiguration conf = notebook.getConf(); + String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); + boolean hideHomeScreenNotebookFromList = conf + .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); + + if (needsReload) { + try { + notebook.reloadAllNotes(); + } catch (IOException e) { + LOG.error("Fail to reload notes from repository"); + } + } + + List notes = notebook.getAllNotes(); + List> notesInfo = new LinkedList<>(); + for (Note note : notes) { + boolean isNotebookRunning = false; + Map info = new HashMap<>(); + + if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { + continue; + } + + String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; + info.put("notebookId", note.id()); + String notebookName = note.getName(); + if (notebookName != null) { + info.put("notebookName", note.getName()); + } else { + info.put("notebookName", "Note " + note.id()); + } + + if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true + && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { + info.put("notebookType", "cron"); + } + else { + info.put("notebookType", "normal"); + } + + long lastRunningUnixTime = 0; + + List> paragraphsInfo = new LinkedList<>(); + for (Paragraph paragraph : note.getParagraphs()) { + if (paragraph.getStatus().isRunning() == true) { + isNotebookRunning = true; + } + + Map paragraphItem = getParagraphPacketItem(paragraph); + + lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph); + + paragraphsInfo.add(paragraphItem); + } + + // Interpreter is set does not exist. + String interpreterGroupName = null; + if (note.getNoteReplLoader().getInterpreterSettings() != null + && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { + interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); + } + + // notebook json object root information. + info.put("interpreter", interpreterGroupName); + info.put("isRunningJob", isNotebookRunning); + info.put("unixTimeLastRun", lastRunningUnixTime); + info.put("paragraphs", paragraphsInfo); + notesInfo.add(info); + } + return notesInfo; + } + + + public List> generateUpdateNotebooksJobInfo( + boolean needsReload, long lastUpdateServerUnixTime) { + Notebook notebook = notebook(); + + ZeppelinConfiguration conf = notebook.getConf(); + String homescreenNotebookId = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN); + boolean hideHomeScreenNotebookFromList = conf + .getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE); + + if (needsReload) { + try { + notebook.reloadAllNotes(); + } catch (IOException e) { + LOG.error("Fail to reload notes from repository"); + } + } + + List notes = notebook.getAllNotes(); + List> notesInfo = new LinkedList<>(); + for (Note note : notes) { + boolean isNotebookRunning = false; + boolean isUpdateNotebook = false; + + Map info = new HashMap<>(); + + if (hideHomeScreenNotebookFromList && note.id().equals(homescreenNotebookId)) { + continue; + } + + // set const keyword for cron type + String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; + info.put("notebookId", note.id()); + String notebookName = note.getName(); + if (notebookName != null) { + info.put("notebookName", note.getName()); + } else { + info.put("notebookName", note.id()); + } + + if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true + && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { + info.put("notebookType", "cron"); + } + else { + info.put("notebookType", "normal"); + } + + long lastRunningUnixTime = 0; + + List> paragraphsInfo = new LinkedList<>(); + for (Paragraph paragraph : note.getParagraphs()) { + + // check date for update time. + Date startedDate = paragraph.getDateStarted(); + Date createdDate = paragraph.getDateCreated(); + Date finishedDate = paragraph.getDateFinished(); + + if (startedDate != null && startedDate.getTime() > lastUpdateServerUnixTime) { + isUpdateNotebook = true; + } + if (createdDate != null && createdDate.getTime() > lastUpdateServerUnixTime) { + isUpdateNotebook = true; + } + if (finishedDate != null && finishedDate.getTime() > lastUpdateServerUnixTime) { + isUpdateNotebook = true; + } + + Map paragraphItem = getParagraphPacketItem(paragraph); + + lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph); + + if (paragraph.getStatus().isRunning() == true) { + isNotebookRunning = true; + isUpdateNotebook = true; + } + paragraphsInfo.add(paragraphItem); + } + + // Insert only data that has changed. + if (isUpdateNotebook != true) { + continue; + } + + // Interpreter is set does not exist. + String interpreterGroupName = null; + if (note.getNoteReplLoader().getInterpreterSettings() != null + && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { + interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); + } + + // set notebook root information. + info.put("interpreter", interpreterGroupName); + info.put("isRunningJob", isNotebookRunning); + info.put("unixTimeLastRun", lastRunningUnixTime); + info.put("paragraphs", paragraphsInfo); + notesInfo.add(info); + } + return notesInfo; + } + + public boolean broadUpdateNote(String noteId) { + + Note note = notebook().getNote(noteId); + + if (note == null) { + LOG.info("broadUpdateNote - not found note"); + return false; + } + + List> notesList = new LinkedList<>(); + + Map noteItem = new HashMap<>(); + + noteItem.put("notebookId", note.id()); + String notebookName = note.getName(); + if (notebookName != null) { + noteItem.put("notebookName", note.getName()); + } else { + noteItem.put("notebookName", note.id()); + } + + // set const keyword for cron type + String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; + if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true + && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { + noteItem.put("notebookType", "cron"); + } + else { + noteItem.put("notebookType", "normal"); + } + + + long lastRunningUnixTime = 0; + boolean isNotebookRunning = false; + List> paragraphsInfo = new LinkedList<>(); + for (Paragraph paragraph : note.getParagraphs()) { + Map paragraphItem = getParagraphPacketItem(paragraph); + + if (paragraph.getStatus().isRunning() == true) { + isNotebookRunning = true; + } + + lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph); + + paragraphsInfo.add(paragraphItem); + } + + // Interpreter is set does not exist. + String interpreterGroupName = null; + if (note.getNoteReplLoader().getInterpreterSettings() != null + && note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { + interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); + } + + // set notebook root information. + noteItem.put("interpreter", interpreterGroupName); + noteItem.put("isRunningJob", isNotebookRunning); + noteItem.put("unixTimeLastRun", lastRunningUnixTime); + noteItem.put("paragraphs", paragraphsInfo); + notesList.add(noteItem); + + Map response = new HashMap<>(); + + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", notesList); + + broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), + new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response)); + + return true; + } + + public boolean broadRemovedNote(String noteId) { + + List> notesList = new LinkedList<>(); + Map noteItem = new HashMap<>(); + + // set notebook root information. + noteItem.put("notebookId", noteId); + noteItem.put("isRemoved", true); + notesList.add(noteItem); + + Map response = new HashMap<>(); + + response.put("lastResponseUnixTime", System.currentTimeMillis()); + response.put("jobs", notesList); + + broadcast(JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), + new Message(OP.LIST_UPDATE_NOTEBOOK_JOBS).put("notebookRunningJobs", response)); + + return true; + } + + /** + * Notebook Observer Event Listener + */ + @Override + public void update(Observable observer, Object notebookChnagedEvent) { + NotebookChnagedEvent noteEvent = (NotebookChnagedEvent) notebookChnagedEvent; + try { + if (noteEvent.getAction() == NotebookEventObserver.ACTIONS.REMOVED) { + broadRemovedNote(noteEvent.getNoteId()); + } else { + broadUpdateNote(noteEvent.getNoteId()); + } + } catch (Exception e) { + LOG.info("socket error job {}", e.getMessage()); + } + } + void permissionError(WebAppSocket conn, String op, Set userAndRoles, Set allowed) throws IOException { LOG.info("Cannot {}. Connection readers {}. Allowed readers {}", diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index ca325d7f54d..5db234423ac 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -103,12 +103,18 @@ angular.module('zeppelinWebApp') var changeOriginTarget = indexStore[changedItem.notebookId]; if (changedItem.isRemoved !== undefined && changedItem.isRemoved === true) { + // remove Item. - delete indexStore[changedItem.notebookId]; var removeIndex = _.findIndex(indexStore, changedItem.notebookId); - indexStore.splice(removeIndex, 1); + if (removeIndex > -1) { + indexStore.splice(removeIndex, 1); + } + removeIndex = _.findIndex(jobInfomations, { 'notebookId' : changedItem.notebookId}); - indexStore.splice(removeIndex, 1); + if (removeIndex) { + jobInfomations.splice(removeIndex, 1); + } + } else { // change value for item. changeOriginTarget.isRunningJob = changedItem.isRunningJob; From d3c40ecf8992359d77c900183185292a2539dbf8 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Sat, 4 Jun 2016 11:34:11 +0900 Subject: [PATCH 22/25] remove notebook target op for job manager --- .../components/websocketEvents/websocketEvents.factory.js | 8 +------- .../components/websocketEvents/websocketMsg.service.js | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js index 4ee94a39903..0ecac157334 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js +++ b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js @@ -27,7 +27,7 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, }, 10000); }); - websocketCalls.sendNewEvent = function(data, targetServer) { + websocketCalls.sendNewEvent = function(data) { if ($rootScope.ticket !== undefined) { data.principal = $rootScope.ticket.principal; data.ticket = $rootScope.ticket.ticket; @@ -38,12 +38,6 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, data.roles = ''; } - if (targetServer !== undefined) { - data.target = targetServer; - } else { - data.target = ''; - } - console.log('Send Notebook Server >> %o, %o, %o, %o, %o', data.op, data.principal, data.ticket, data.roles, data); websocketCalls.ws.send(JSON.stringify(data)); }; diff --git a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js index 3b908a0004b..661a9f10617 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js +++ b/zeppelin-web/src/components/websocketEvents/websocketMsg.service.js @@ -166,7 +166,7 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, }, getNotebookJobsList: function() { - websocketEvents.sendNewEvent({op: 'LIST_NOTEBOOK_JOBS'}, 'jobManagerServer'); + websocketEvents.sendNewEvent({op: 'LIST_NOTEBOOK_JOBS'}); }, getUpdateNotebookJobsList: function(lastUpdateServerUnixTime) { @@ -177,7 +177,7 @@ angular.module('zeppelinWebApp').service('websocketMsgSrv', function($rootScope, }, unsubscribeJobManager: function() { - websocketEvents.sendNewEvent({op: 'UNSUBSCRIBE_JOBMANAGER'}, 'jobManagerServer'); + websocketEvents.sendNewEvent({op: 'UNSUBSCRIBE_JOBMANAGER'}); }, }; From 7960e2877fd1994a908f8a2c2584f1f89e6c89ba Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Sun, 5 Jun 2016 00:42:59 +0900 Subject: [PATCH 23/25] changed ui - filter, select, and button size - job manager --- .../app/jobmanager/jobmanager.controller.js | 15 ++++++++++ .../src/app/jobmanager/jobmanager.css | 8 +++++ .../src/app/jobmanager/jobmanager.html | 29 +++++++++++++------ 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js index 5db234423ac..bf20af18a2c 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobmanager.controller.js @@ -140,6 +140,21 @@ angular.module('zeppelinWebApp') }); }; + $scope.filterValueToName = function (filterValue) { + var index = _.findIndex($scope.ACTIVE_INTERPRETERS, {value : filterValue}); + + if ($scope.ACTIVE_INTERPRETERS[index].name !== undefined) { + return $scope.ACTIVE_INTERPRETERS[index].name; + } else { + return 'undefined'; + } + }; + + $scope.setFilterValue = function (filterValue) { + $scope.filterConfig.FILTER_VALUE_INTERPRETER = filterValue; + $scope.doFiltering($scope.jobInfomations, $scope.filterConfig); + }; + $scope.onChangeRunJobToAlwaysTopToggle = function () { $scope.filterConfig.RUNNING_ALWAYS_TOP = !$scope.filterConfig.RUNNING_ALWAYS_TOP; $scope.doFiltering($scope.jobInfomations, $scope.filterConfig); diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.css b/zeppelin-web/src/app/jobmanager/jobmanager.css index a09f91dce87..0e60d568485 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.css +++ b/zeppelin-web/src/app/jobmanager/jobmanager.css @@ -33,3 +33,11 @@ .jobManagerHead .header { font-family: 'Roboto', sans-serif; } + +.job-note-name-query { + padding: 6px; + color: #000; + height: 22px; + width: 200px; + font: normal normal normal 14px/1 FontAwesome; +} diff --git a/zeppelin-web/src/app/jobmanager/jobmanager.html b/zeppelin-web/src/app/jobmanager/jobmanager.html index 75371cc637b..ead19b64224 100644 --- a/zeppelin-web/src/app/jobmanager/jobmanager.html +++ b/zeppelin-web/src/app/jobmanager/jobmanager.html @@ -36,27 +36,38 @@

    - + + + +
    From 5590b66965ffc35c3bb38741fca1edc56f885b7e Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Sun, 5 Jun 2016 03:20:40 +0900 Subject: [PATCH 24/25] remove bin/derby.log --- bin/derby.log | 4604 ------------------------------------------------- 1 file changed, 4604 deletions(-) delete mode 100644 bin/derby.log diff --git a/bin/derby.log b/bin/derby.log deleted file mode 100644 index 5c9a5ff754d..00000000000 --- a/bin/derby.log +++ /dev/null @@ -1,4604 +0,0 @@ -Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Cleanup action completed -Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 100 more -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - ... 97 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 97 more -============= begin nested exception, level (1) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 102 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 97 more -============= end nested exception, level (1) =========== -============= begin nested exception, level (2) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 97 more -============= end nested exception, level (2) =========== -============= begin nested exception, level (3) =========== -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -============= end nested exception, level (3) =========== -Cleanup action completed -Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Cleanup action completed -Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 100 more -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - ... 97 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 97 more -============= begin nested exception, level (1) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 102 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 97 more -============= end nested exception, level (1) =========== -============= begin nested exception, level (2) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 97 more -============= end nested exception, level (2) =========== -============= begin nested exception, level (3) =========== -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -============= end nested exception, level (3) =========== -Cleanup action completed -Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Cleanup action completed -Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 100 more -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - ... 97 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 97 more -============= begin nested exception, level (1) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 102 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 97 more -============= end nested exception, level (1) =========== -============= begin nested exception, level (2) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 97 more -============= end nested exception, level (2) =========== -============= begin nested exception, level (3) =========== -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -============= end nested exception, level (3) =========== -Cleanup action completed -Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Cleanup action completed -Sat Jun 04 11:10:44 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 100 more -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - ... 97 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 97 more -============= begin nested exception, level (1) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 102 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 97 more -============= end nested exception, level (1) =========== -============= begin nested exception, level (2) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 97 more -============= end nested exception, level (2) =========== -============= begin nested exception, level (3) =========== -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) - at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) - at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:166) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -============= end nested exception, level (3) =========== -Cleanup action completed -Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Cleanup action completed -Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 97 more -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - ... 94 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 94 more -============= begin nested exception, level (1) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 99 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 94 more -============= end nested exception, level (1) =========== -============= begin nested exception, level (2) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 94 more -============= end nested exception, level (2) =========== -============= begin nested exception, level (3) =========== -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -============= end nested exception, level (3) =========== -Cleanup action completed -Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Cleanup action completed -Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 97 more -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - ... 94 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 94 more -============= begin nested exception, level (1) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 99 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 94 more -============= end nested exception, level (1) =========== -============= begin nested exception, level (2) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 94 more -============= end nested exception, level (2) =========== -============= begin nested exception, level (3) =========== -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:620) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -============= end nested exception, level (3) =========== -Cleanup action completed -Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Cleanup action completed -Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 97 more -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - ... 94 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 94 more -============= begin nested exception, level (1) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 99 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 94 more -============= end nested exception, level (1) =========== -============= begin nested exception, level (2) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 94 more -============= end nested exception, level (2) =========== -============= begin nested exception, level (3) =========== -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:349) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -============= end nested exception, level (3) =========== -Cleanup action completed -Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Cleanup action completed -Sat Jun 04 11:10:45 KST 2016 Thread[pool-2-thread-3,5,main] Cleanup action starting -java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.seeNextException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: Failed to create database 'metastore_db', see the next exception for details. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 97 more -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - ... 94 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 94 more -============= begin nested exception, level (1) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - ... 99 more -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 94 more -============= end nested exception, level (1) =========== -============= begin nested exception, level (2) =========== -java.sql.SQLException: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown Source) - at org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source) - at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -Caused by: ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - ... 94 more -============= end nested exception, level (2) =========== -============= begin nested exception, level (3) =========== -ERROR XBM0A: The database directory '/Users/ringo/Documents/Source/nflabs/incubator-zeppelin/bin/metastore_db' exists. However, it does not contain the expected 'service.properties' file. Perhaps Derby was brought down in the middle of creating this database. You may want to delete this directory and try creating the database again. - at org.apache.derby.iapi.error.StandardException.newException(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.vetService(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService.access$900(Unknown Source) - at org.apache.derby.impl.services.monitor.StorageFactoryService$10.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at org.apache.derby.impl.services.monitor.StorageFactoryService.createServiceRoot(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source) - at org.apache.derby.impl.services.monitor.BaseMonitor.createPersistentService(Unknown Source) - at org.apache.derby.iapi.services.monitor.Monitor.createPersistentService(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.createDatabase(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection.(Unknown Source) - at org.apache.derby.impl.jdbc.EmbedConnection40.(Unknown Source) - at org.apache.derby.jdbc.Driver40.getNewEmbedConnection(Unknown Source) - at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source) - at org.apache.derby.jdbc.Driver20.connect(Unknown Source) - at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source) - at java.sql.DriverManager.getConnection(DriverManager.java:664) - at java.sql.DriverManager.getConnection(DriverManager.java:208) - at com.jolbox.bonecp.BoneCP.obtainRawInternalConnection(BoneCP.java:361) - at com.jolbox.bonecp.BoneCP.(BoneCP.java:416) - at com.jolbox.bonecp.BoneCPDataSource.getConnection(BoneCPDataSource.java:120) - at org.datanucleus.store.rdbms.ConnectionFactoryImpl$ManagedConnectionImpl.getConnection(ConnectionFactoryImpl.java:501) - at org.datanucleus.store.rdbms.RDBMSStoreManager.(RDBMSStoreManager.java:298) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.datanucleus.plugin.NonManagedPluginRegistry.createExecutableExtension(NonManagedPluginRegistry.java:631) - at org.datanucleus.plugin.PluginManager.createExecutableExtension(PluginManager.java:301) - at org.datanucleus.NucleusContext.createStoreManagerForProperties(NucleusContext.java:1187) - at org.datanucleus.NucleusContext.initialise(NucleusContext.java:356) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.freezeConfiguration(JDOPersistenceManagerFactory.java:775) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.createPersistenceManagerFactory(JDOPersistenceManagerFactory.java:333) - at org.datanucleus.api.jdo.JDOPersistenceManagerFactory.getPersistenceManagerFactory(JDOPersistenceManagerFactory.java:202) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.lang.reflect.Method.invoke(Method.java:497) - at javax.jdo.JDOHelper$16.run(JDOHelper.java:1965) - at java.security.AccessController.doPrivileged(Native Method) - at javax.jdo.JDOHelper.invoke(JDOHelper.java:1960) - at javax.jdo.JDOHelper.invokeGetPersistenceManagerFactoryOnImplementation(JDOHelper.java:1166) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:808) - at javax.jdo.JDOHelper.getPersistenceManagerFactory(JDOHelper.java:701) - at org.apache.hadoop.hive.metastore.ObjectStore.getPMF(ObjectStore.java:365) - at org.apache.hadoop.hive.metastore.ObjectStore.getPersistenceManager(ObjectStore.java:394) - at org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:291) - at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:258) - at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:73) - at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:133) - at org.apache.hadoop.hive.metastore.RawStoreProxy.(RawStoreProxy.java:57) - at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:66) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:593) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:571) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:624) - at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:461) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.(RetryingHMSHandler.java:66) - at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) - at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) - at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.(HiveMetaStoreClient.java:199) - at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.(SessionHiveMetaStoreClient.java:74) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.(RetryingMetaStoreClient.java:86) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) - at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) - at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) - at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) - at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) - at org.apache.spark.sql.hive.client.ClientWrapper.(ClientWrapper.scala:204) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) - at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) - at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) - at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) - at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:229) - at org.apache.spark.sql.hive.HiveContext.(HiveContext.scala:101) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) - at java.lang.reflect.Constructor.newInstance(Constructor.java:422) - at org.apache.zeppelin.spark.SparkInterpreter.getSQLContext(SparkInterpreter.java:184) - at org.apache.zeppelin.spark.SparkInterpreter.open(SparkInterpreter.java:510) - at org.apache.zeppelin.interpreter.ClassloaderInterpreter.open(ClassloaderInterpreter.java:74) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.open(LazyOpenInterpreter.java:68) - at org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:92) - at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:345) - at org.apache.zeppelin.scheduler.Job.run(Job.java:176) - at org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) - at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) - at java.util.concurrent.FutureTask.run(FutureTask.java:266) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) - at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) - at java.lang.Thread.run(Thread.java:745) -============= end nested exception, level (3) =========== -Cleanup action completed From bc443da2aa860a7a31672d6761c859b8793a1c7f Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Sun, 5 Jun 2016 03:28:02 +0900 Subject: [PATCH 25/25] remove head tag - job manager. --- .../main/java/org/apache/zeppelin/rest/NotebookRestApi.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index a14cd6a78eb..48cc575be38 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -32,11 +32,8 @@ import javax.ws.rs.core.Response.Status; import org.apache.commons.lang3.StringUtils; -<<<<<<< HEAD import org.apache.zeppelin.interpreter.InterpreterException; -======= import org.apache.zeppelin.annotation.ZeppelinApi; ->>>>>>> master import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Notebook;