New RestServer Architecture: RestServer -> DB -> ApiServer; P0 Items

.github/workflows/continuous-integration.yml

@@ -131,8 +131,10 @@ jobs:
     - name: yarn install and test
       run: |
         cd src/rest-server
+        cp -rf ../database-controller/sdk openpaidbsdk
         yarn config set ignore-engines true
         yarn install --frozen-lockfiles
+        rm -rf openpaidbsdk
         yarn test
 
   webportal:

.travis.yml

@@ -91,8 +91,10 @@ matrix:
       env: NODE_ENV=test
       before_install:
         - cd src/rest-server
+        - cp -rf ../database-controller/sdk openpaidbsdk
       install:
         - yarn install
+        - rm -rf openpaidbsdk
       script:
         - npm test
         - npm run coveralls
@@ -102,8 +104,10 @@ matrix:
       env: NODE_ENV=test
       before_install:
         - cd src/rest-server
+        - cp -rf ../database-controller/sdk openpaidbsdk
       install:
         - yarn install --ignore-engines
+        - rm -rf openpaidbsdk
       script:
         - npm test
 

contrib/kubespray/quick-start/services-configuration.yaml.template

@@ -48,12 +48,6 @@ rest-server:
 webportal:
   server-port: 9286
 
-internal-storage:
-  enable: true
-
-postgresql:
-  enable: true
-
 #If you want to customize the scheduling config, such add more virtual clusters or more gpu types, check:
 #https://github.com/microsoft/pai/blob/master/docs/hivedscheduler/devops.md
 hivedscheduler:

src/database-controller/.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true

src/database-controller/.gitignore

@@ -0,0 +1,106 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# Next.js build output
+.next
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and *not* Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+version/

src/database-controller/README.md

@@ -0,0 +1,8 @@
+## Database Controller
+
+### Development
+
+**Environment:** Node.js 8.17.0, use `yarn install` to install all dependencies under `src/` or `sdk/`. To set environmental variables, create a `.env` file under `src`.
+
+
+**Lint:** Use `npm install standard --global` to isntall Standard.js globally. Then run `npm run lint` under `src/` or `sdk/`.

src/database-controller/build/build-pre.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+pushd $(dirname "$0") > /dev/null
+
+mkdir -m 777 -p "../version"
+cp -arf "../../../version/PAI.VERSION" "../version/"
+echo `git rev-parse HEAD` > "../version/COMMIT.VERSION"
+
+popd > /dev/null

src/database-controller/build/database-controller.common.dockerfile

@@ -0,0 +1,35 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+FROM node:carbon
+
+WORKDIR /database-controller
+
+COPY ./src ./src
+COPY ./sdk ./sdk
+COPY ./version ./version
+
+WORKDIR src
+
+RUN yarn install
+
+RUN npm install json -g
+RUN json -I -f package.json -e "this.paiVersion=\"`cat ../version/PAI.VERSION`\""
+RUN json -I -f package.json -e "this.paiCommitVersion=\"`cat ../version/COMMIT.VERSION`\""
+
+
+CMD ["sleep", "infinity"]

src/database-controller/config/database-controller.yaml

@@ -0,0 +1,32 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+service_type: "common"
+
+# general settings
+log-level: info
+recovery-mode: false
+k8s-connection-timeout-second: 120
+write-merger-connection-timeout-second: 120
+
+# write merger
+write-merger-port: 9748
+write-merger-max-db-connection: 50
+
+# db poller
+db-poller-interval-second: 120
+db-poller-max-db-connection: 10

src/database-controller/config/database_controller.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+#
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import copy
+import logging
+
+
+class DatabaseController(object):
+    def __init__(self, cluster_conf, service_conf, default_service_conf):
+        self.cluster_conf = cluster_conf
+        self.service_conf = self.merge_service_configuration(service_conf, default_service_conf)
+        self.logger = logging.getLogger(__name__)
+
+    @staticmethod
+    def merge_service_configuration(overwrite_srv_cfg, default_srv_cfg):
+        if overwrite_srv_cfg is None:
+            return default_srv_cfg
+        srv_cfg = default_srv_cfg.copy()
+        for k in overwrite_srv_cfg:
+            srv_cfg[k] = overwrite_srv_cfg[k]
+        return srv_cfg
+
+    def get_master_ip(self):
+        for host_conf in self.cluster_conf["machine-list"]:
+            if "pai-master" in host_conf and host_conf["pai-master"] == "true":
+                return host_conf["hostip"]
+
+    def validation_pre(self):
+        return True, None
+
+    def run(self):
+        result = copy.deepcopy(self.service_conf)
+        result['write-merger-url'] = 'http://{}:{}'.format(self.get_master_ip(), result['write-merger-port'])
+        return result
+
+    def validation_post(self, conf):
+        return True, None