[v7r3][WIP] OAuth2/OIDC AuthN/AuthZ mechanism (Full version)

ConfigurationSystem/Client/Helpers/Resources.py

@@ -465,42 +465,68 @@ def getFilterConfig(filterID):
   return gConfig.getOptionsDict('Resources/LogFilters/%s' % filterID)
 
 
-def getInfoAboutProviders(of=None, providerName=None, option='', section=''):
-  """ Get the information about providers
-
-      :param basestring of: provider of what(Id, Proxy or etc.) need to look,
-             None, "all" to get list of instance of what this providers
-      :param basestring providerName: provider name,
-             None, "all" to get list of providers names
-      :param basestring option: option name that need to get,
-             None, "all" to get all options in a section
-      :param basestring section: section path in root section of provider,
-             "all" to get options in all sections
-
-      :return: S_OK()/S_ERROR()
+def getProvidersForInstance(instance, providerType=None):
+  """ Get providers for instance
+
+      :param str instance: instance of what this providers
+      :param str providerType: provider type
+
+      :return: S_OK(list)/S_ERROR()
+  """
+  result = gConfig.getSections('%s/%sProviders' % (gBaseResourcesSection, instance))
+  if not result['OK'] or not result['Value'] or (result['OK'] and not providerType):
+    return result
+  data = []
+  for prov in result['Value']:
+    if providerType == gConfig.getValue('%s/%sProviders/%s/ProviderType' %
+                                        (gBaseResourcesSection, instance, prov)):
+      data.append(prov)
+  return S_OK(data)
+
+
+def getProviderByAlias(alias, instance=None):
+  """ Find provider name by alias
+
+      :param str alias: other registered provider name
+      :param str instance: provider of what
+
+      :return: S_OK(str)/S_ERROR()
   """
-  if not of or of == "all":
+  instances = [instance] or []
+  if not instances:
     result = gConfig.getSections(gBaseResourcesSection)
     if not result['OK']:
       return result
-    return S_OK([i.replace('Providers', '') for i in result['Value']])
-  if not providerName or providerName == "all":
-    return gConfig.getSections('%s/%sProviders' % (gBaseResourcesSection, of))
-  if not option or option == 'all':
-    if not section:
-      return gConfig.getOptionsDict("%s/%sProviders/%s" % (gBaseResourcesSection, of, providerName))
-    elif section == "all":
-      resDict = {}
-      relPath = "%s/%sProviders/%s/" % (gBaseResourcesSection, of, providerName)
-      result = gConfig.getConfigurationTree(relPath)
+    for section in result['Value']:
+      if section[-9:] == 'Providers':
+        instances.append(section[:-9])
+  for instance in instances:
+    result = getProvidersForInstance(instance)
+    if not result['OK']:
+      return result
+    for provider in result['Value']:
+      if alias in gConfig.getValue("%s/%sProviders/%s/Aliases" % (gBaseResourcesSection,
+                                                                  instance, provider), []):
+        return S_OK(provider)
+  return S_ERROR('No found any provider for %s' % alias)
+
+
+def getProviderInfo(provider):
+  """ Get provider info
+
+      :param str provider: provider
+
+      :return: S_OK(dict)/S_ERROR()
+  """
+  result = gConfig.getSections(gBaseResourcesSection)
+  if not result['OK']:
+    return result
+  for section in result['Value']:
+    if section[-9:] == 'Providers':
+      result = getProvidersForInstance(section[:-9])
       if not result['OK']:
         return result
-      for key, value in result['Value'].items():  # can be an iterator
-        if value:
-          resDict[key.replace(relPath, '')] = value
-      return S_OK(resDict)
-    else:
-      return gConfig.getSections('%s/%sProviders/%s/%s/' % (gBaseResourcesSection, of, providerName, section))
-  else:
-    return S_OK(gConfig.getValue('%s/%sProviders/%s/%s/%s' % (gBaseResourcesSection, of, providerName,
-                                                              section, option)))
+      if provider in result['Value']:
+        return gConfig.getOptionsDictRecursively("%s/%s/%s/" % (gBaseResourcesSection,
+                                                                section, provider))
+  return S_ERROR('%s provider not found.' % provider)

ConfigurationSystem/Client/Utilities.py

@@ -701,14 +701,14 @@ def getElasticDBParameters(fullname):
   return S_OK(parameters)
 
 
-def getOAuthAPI(instance='Production'):
-  """ Get OAuth API url
+def getAuthAPI(instance='Production'):
+  """ Get Auth REST API url
 
       :param str instance: instance
 
       :return: str
   """
-  return gConfig.getValue("/Systems/Framework/%s/URLs/OAuthAPI" % instance)
+  return gConfig.getValue("/Systems/Framework/%s/URLs/AuthAPI" % instance)
 
 
 def getDIRACGOCDictionary():

ConfigurationSystem/Utilities/ConfigurationHandler.py

@@ -0,0 +1,96 @@
+""" HTTP API of the DIRAC configuration data, rewrite from the RESTDIRAC project
+"""
+import re
+import json
+
+from tornado import web, gen
+from tornado.template import Template
+
+from DIRAC import S_OK, S_ERROR, gConfig, gLogger
+from DIRAC.ConfigurationSystem.Client.Helpers import Resources, Registry
+from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
+from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
+
+from DIRAC.Core.Web.WebHandler import WebHandler, asyncGen, WErr
+
+__RCSID__ = "$Id$"
+
+
+class ConfigurationHandler(WebHandler):
+  OVERPATH = True
+  AUTH_PROPS = "all"
+  LOCATION = "/"
+
+  def initialize(self):
+    super(ConfigurationHandler, self).initialize()
+    self.args = {}
+    for arg in self.request.arguments:
+      if len(self.request.arguments[arg]) > 1:
+        self.args[arg] = self.request.arguments[arg]
+      else:
+        self.args[arg] = self.request.arguments[arg][0] or ''
+    return S_OK()
+
+  @asyncGen
+  def web_conf(self):
+    """ REST endpoint for configuration system:
+
+        **GET** /conf/<key>?<options> -- get configuration information
+
+          Options:
+            * *path* -- path in the configuration structure, by default it's "/". 
+            * *version* -- the configuration version of the requester, if *version* is newer
+                           than the one present on the server, an empty result will be returned 
+
+          Response:
+            +-----------+---------------------------------------+------------------------+
+            | *key*     | Description                           | Type                   |
+            +-----------+---------------------------------------+------------------------+
+            | dump      | Current CFG()                         | encoded in json format |
+            +-----------+---------------------------------------+------------------------+
+            | option    | Option value                          | text                   |
+            +-----------+---------------------------------------+------------------------+
+            | options   | Options list in a section             | encoded in json format |
+            +-----------+---------------------------------------+------------------------+
+            | dict      | Options with values in a section      | encoded in json format |
+            +-----------+---------------------------------------+------------------------+
+            | sections  | Sections list in a section            | text                   |
+            +-----------+---------------------------------------+------------------------+
+    """
+    self.log.notice('Request configuration information')
+    optns = self.overpath.strip('/').split('/')
+    path = self.args.get('path', '/')
+    if not optns or len(optns) > 1:
+      raise WErr(404, "You forgot to set attribute.")
+
+    result = S_ERROR('%s request unsuported' % optns[0])
+    if 'version' in self.args and (self.args.get('version') or '0') >= gConfigurationData.getVersion():
+      self.finish()
+    if optns[0] == 'dump':
+      remoteCFG = yield self.threadTask(gConfigurationData.getRemoteCFG)
+      result['Value'] = str(remoteCFG)
+    elif optns[0] == 'option':
+      result = yield self.threadTask(gConfig.getOption, path)
+    elif optns[0] == 'dict':
+      result = yield self.threadTask(gConfig.getOptionsDict, path)
+    elif optns[0] == 'options':
+      result = yield self.threadTask(gConfig.getOptions, path)
+    elif optns[0] == 'sections':
+      result = yield self.threadTask(gConfig.getSections, path)
+    elif optns[0] == 'getGroupsStatusByUsername':
+      result = yield self.threadTask(gProxyManager.getGroupsStatusByUsername, **self.args)
+    elif any([optns[0] == m and re.match('^[a-z][A-z]+', m) for m in dir(Registry)]) and self.isRegisteredUser():
+      result = yield self.threadTask(getattr(Registry, optns[0]), **self.args)
+    else:
+      raise WErr(500, '%s request unsuported' % optns[0])
+      # result = yield self.threadTask(getattr(Registry, optns[0]), **self.args)
+
+    if not result['OK']:
+      raise WErr(404, result['Message'])
+    self.finishJEncode(result['Value'])
+
+  @asyncGen
+  def post(self):
+    """ Post method
+    """
+    pass

ConfigurationSystem/Utilities/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+# $HeadURL$
+__RCSID__ = "$Id$"

ConfigurationSystem/test/Test_agentOptions.py

@@ -75,7 +75,7 @@
           ('DIRAC.WorkloadManagementSystem.Agent.StatesAccountingAgent', {}),
           ('DIRAC.WorkloadManagementSystem.Agent.StatesMonitoringAgent', {}),
           ('DIRAC.WorkloadManagementSystem.Agent.SiteDirector',
-           {'SpecialMocks': {'findGenericPilotCredentials': S_OK(('a', 'b'))}}),
+           {'SpecialMocks': {'findGenericPilotCredentials': S_OK(('a', 'b', 'c'))}}),
           # ('DIRAC.WorkloadManagementSystem.Agent.MultiProcessorSiteDirector', {}),  # not inheriting from AgentModule
           ]
 

Core/Base/API.py

@@ -9,7 +9,7 @@
 
 from DIRAC import gLogger, gConfig, S_OK, S_ERROR
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo, formatProxyInfoAsString
-from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername
+from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNsForUsername
 from DIRAC.Core.Utilities.Version import getCurrentVersion
 
 __RCSID__ = '$Id$'
@@ -141,9 +141,11 @@ def _getCurrentUser(self):
     gLogger.debug(formatProxyInfoAsString(proxyInfo))
     if 'group' not in proxyInfo:
       return self._errorReport('Proxy information does not contain the group', res['Message'])
-    res = getDNForUsername(proxyInfo['username'])
-    if not res['OK']:
-      return self._errorReport('Failed to get proxies for user', res['Message'])
+    result = getDNsForUsername(proxyInfo['username'])
+    if not result['OK']:
+      return self._errorReport('Failed to get proxies for user', result['Message'])
+    if not result['Value']:
+      return self._errorReport('Failed to get proxies for user', "No DNs found for %s" % proxyInfo['username'])
     return S_OK(proxyInfo['username'])
 
   #############################################################################

Core/Base/DB.py

@@ -5,7 +5,7 @@
 from __future__ import division
 from __future__ import print_function
 
-from DIRAC import gLogger, gConfig
+from DIRAC import gLogger, gConfig, S_OK, S_ERROR
 from DIRAC.Core.Utilities.MySQL import MySQL
 from DIRAC.ConfigurationSystem.Client.Utilities import getDBParameters
 from DIRAC.ConfigurationSystem.Client.PathFinder import getDatabaseSection
@@ -18,9 +18,16 @@ class DB(MySQL):
   """
 
   def __init__(self, dbname, fullname, debug=False):
+    """ C'or
 
+        :param str dbname: database name
+        :param str fullname: full name
+        :param bool debug: debug mode
+    """
+    self.versionDB = 0
     self.fullname = fullname
     database_name = dbname
+    self.versionTable = '%s_Version' % database_name
     self.log = gLogger.getSubLogger(database_name)
 
     result = getDBParameters(fullname)
@@ -44,6 +51,23 @@ def __init__(self, dbname, fullname, debug=False):
     if not self._connected:
       raise RuntimeError("Can not connect to DB '%s', exiting..." % self.dbName)
 
+    # Initialize version
+    result = self._query("show tables")
+    if result['OK']:
+      if self.versionTable not in [t[0] for t in result['Value']]:
+        result = self._createTables({self.versionTable: {'Fields': {'Version': 'INTEGER NOT NULL'},
+                                                         'PrimaryKey': 'Version'}})
+    if not result['OK']:
+      raise RuntimeError("Can not initialize %s version: %s" % (self.dbName, result['Message']))
+    result = self._query("SELECT Version FROM `%s`" % self.versionTable)
+    if result['OK']:
+      if len(result['Value']) > 0:
+        self.versionDB = result['Value'][0][0]
+      else:
+        result = self._update("INSERT INTO `%s` (Version) VALUES (%s)" % (self.versionTable, self.versionDB))
+    if not result['OK']:
+      raise RuntimeError("Can not initialize %s version: %s" % (self.dbName, result['Message']))
+
     self.log.info("===================== MySQL ======================")
     self.log.info("User:           " + self.dbUser)
     self.log.info("Host:           " + self.dbHost)
@@ -54,5 +78,27 @@ def __init__(self, dbname, fullname, debug=False):
 
 #############################################################################
   def getCSOption(self, optionName, defaultValue=None):
+    """ Get option from CS
+
+        :param str optionName: option name
+        :param defaultValue: default value
+
+        :return: value that inherits the defaultValue type
+    """
     cs_path = getDatabaseSection(self.fullname)
     return gConfig.getValue("/%s/%s" % (cs_path, optionName), defaultValue)
+
+  def updateDBVersion(self, version):
+    """ Update DB version
+
+        :param int version: version number
+
+        :return: S_OK()/S_ERROR()
+    """
+    result = self._query('DELETE FROM `%s_Version`' % self.dbName)
+    if result['OK']:
+      result = self._update("INSERT INTO `%s_Version` (Version) VALUES (%s)" % (self.dbName, version))
+    if not result['OK']:
+      return S_ERROR("Can not initialize %s version: %s" % (self.dbName, result['Message']))
+    self.versionDB = version
+    return S_OK()