diff --git a/ConfigurationSystem/Client/Helpers/Registry.py b/ConfigurationSystem/Client/Helpers/Registry.py index 92098ac0776..425b73a2036 100644 --- a/ConfigurationSystem/Client/Helpers/Registry.py +++ b/ConfigurationSystem/Client/Helpers/Registry.py @@ -8,16 +8,29 @@ from DIRAC import S_OK, S_ERROR from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Utilities.Decorators import deprecated from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO -__RCSID__ = "$Id$" +# Registry use cached data from AuthManager and ProxyManager services +from DIRAC.FrameworkSystem.Client.ProxyManagerData import gProxyManagerData +from DIRAC.FrameworkSystem.Client.AuthManagerData import gAuthManagerData -# pylint: disable=missing-docstring +__RCSID__ = "$Id$" gBaseRegistrySection = "/Registry" +def getVOMSInfo(vo=None, dn=None): + """ Get cached information from VOMS API + + :param list dn: requested DN + + :return: S_OK(dict)/S_ERROR() + """ + return gProxyManagerData.getActualVOMSesDNs(voList=[vo] if vo else vo, dnList=[dn] if dn else dn) + + def getUsernameForDN(dn, usersList=None): """ Find DIRAC user for DN @@ -35,18 +48,30 @@ def getUsernameForDN(dn, usersList=None): for username in usersList: if dn in gConfig.getValue("%s/Users/%s/DN" % (gBaseRegistrySection, username), []): return S_OK(username) - return S_ERROR("No username found for dn %s" % dn) + # Get users profiles from session manager cache + result = gAuthManagerData.getIDsForDN(dn) + if result['OK']: + for uid in result['Value']: + result = getUsernameForID(uid) + if result['OK']: + return result + + return S_ERROR("No username found for dn %s" % dn) +@deprecated("Use getDNsForUsername or getDNsForUsernameFromCS if need information only from CS") def getDNForUsername(username): - """ Get user DN for user + dnList = getDNsForUsernameFromSC(username) + return S_OK(dnList) if dnList else S_ERROR("No DN found for user %s" % username) - :param str username: user name +def getDNsForUsernameFromCS(username): + """ Find DNs for DIRAC user from CS - :return: S_OK(str)/S_ERROR() + :param str username: DIRAC user + + :return: list -- contain DNs """ - dnList = gConfig.getValue("%s/Users/%s/DN" % (gBaseRegistrySection, username), []) - return S_OK(dnList) if dnList else S_ERROR("No DN found for user %s" % username) + return gConfig.getValue("%s/Users/%s/DN" % (gBaseRegistrySection, username), []) def getDNForHost(host): @@ -60,18 +85,53 @@ def getDNForHost(host): return S_OK(dnList) if dnList else S_ERROR("No DN found for host %s" % host) -def getGroupsForDN(dn): +def getGroupsForDN(dn, groupsList=None): """ Get all possible groups for DN :param str dn: user DN + :param list groupsList: group list where need to search :return: S_OK(list)/S_ERROR() -- contain list of groups """ dn = dn.strip() + groups = [] + if not groupsList: + result = gConfig.getSections("%s/Groups" % gBaseRegistrySection) + if not result['OK']: + return result + groupsList = result['Value'] + result = getUsernameForDN(dn) if not result['OK']: return result - return getGroupsForUser(result['Value']) + user = result['Value'] + + # Get VOMS information cache + result = getVOMSInfo(dn=dn) + if not result['OK']: + return result + vomsData = result['Value'] + + result = getVOsWithVOMS() + if not result['OK']: + return result + vomsVOs = result['Value'] + + for group in groupsList: + if user in getGroupOption(group, 'Users', []): + vo = getGroupOption(group, 'VO') + # Is VOMS VO? + if vo in vomsVOs and vomsData.get(vo) and vomsData[vo]['OK'] and vomsData[vo]['Value']: + voData = vomsData[vo]['Value'] + role = getGroupOption(group, 'VOMSRole') + if not role or role in voData[dn]['VOMSRoles']: + groups.append(group) + else: + # If it's not VOMS VO or cannot get information from VOMS + groups.append(group) + groups = list(set(groups)) + groups.sort() + return S_OK(groups) if groups else S_ERROR('No groups found for %s' % dn) def __getGroupsWithAttr(attrName, value): @@ -94,14 +154,27 @@ def __getGroupsWithAttr(attrName, value): return S_OK(groups) if groups else S_ERROR("No groups found for %s=%s" % (attrName, value)) -def getGroupsForUser(username): - """ Find groups for user +def getGroupsForUser(username, groupsList=None): + """ Find groups for user or if set reseachedGroup check it for user :param str username: user name + :param list groupsList: groups - :return: S_OK(list)/S_ERROR() -- contain list of groups + :return: S_OK(list or bool)/S_ERROR() -- contain list of groups or status group for user """ - return __getGroupsWithAttr('Users', username) + if not groupsList: + retVal = gConfig.getSections("%s/Groups" % gBaseRegistrySection) + if not retVal['OK']: + return retVal + groupsList = retVal['Value'] + + groups = [] + for group in groupsList: + if username in getGroupOption(group, 'Users', []): + groups.append(group) + groups = list(set(groups)) + groups.sort() + return S_OK(groups) if groups else S_ERROR('No groups found for %s user' % username) def getGroupsForVO(vo): @@ -203,7 +276,7 @@ def getAllGroups(): return result['Value'] if result['OK'] else [] -def getUsersInGroup(groupName, defaultValue=None): +def getUsersInGroup(group, defaultValue=None): """ Find all users for group :param str group: group name @@ -211,8 +284,9 @@ def getUsersInGroup(groupName, defaultValue=None): :return: list """ - option = "%s/Groups/%s/Users" % (gBaseRegistrySection, groupName) - return gConfig.getValue(option, [] if defaultValue is None else defaultValue) + users = list(set(getGroupOption(group, 'Users', []))) + users.sort() + return users or [] if defaultValue is None else defaultValue def getUsersInVO(vo, defaultValue=None): @@ -223,45 +297,56 @@ def getUsersInVO(vo, defaultValue=None): :return: list """ + users = [] result = getGroupsForVO(vo) - if not result['OK'] or not result['Value']: - return [] if defaultValue is None else defaultValue - groups = result['Value'] + if result['OK'] and result['Value']: + for group in result['Value']: + users += getUsersInGroup(group) + users = list(set(users)) + users.sort() + return users or [] if defaultValue is None else defaultValue - userList = [] - for group in groups: - userList += getUsersInGroup(group) - return userList +def getDNsInGroup(group, checkStatus=False): + """ Find user DNs for DIRAC group -def getDNsInVO(vo): - """ Get all DNs that have a VO users - - :param str vo: VO name + :param str group: group name + :param bool checkStatus: don't add suspended DNs :return: list """ - DNs = [] - for user in getUsersInVO(vo): - result = getDNForUsername(user) - if result['OK']: - DNs.extend(result['Value']) - return DNs - - -def getDNsInGroup(groupName): - """ Find all DNs for DIRAC group + vomsData = {} + vo = getGroupOption(group, 'VO') - :param str groupName: group name + # Get VOMS information for VO, if it's VOMS VO + result = getVOsWithVOMS([vo]) + if not result['OK']: + return result + if result['Value']: + result = getVOMSInfo(vo=vo) + if not result['OK']: + return result + vomsData = result['Value'] - :return: list - """ DNs = [] - for user in getUsersInGroup(groupName): - result = getDNForUsername(user) - if result['OK']: - DNs.extend(result['Value']) - return DNs + for username in getGroupOption(group, 'Users', []): + if checkStatus and vo in getUserOption(username, 'Suspended', []): + continue + result = getDNsForUsername(username) + if not result['OK']: + return result + userDNs = result['Value'] + if vomsData.get(vo) and vomsData[vo]['OK']: + voData = vomsData[vo]['Value'] + role = getGroupOption(group, 'VOMSRole') + for dn in userDNs: + if dn in voData and (not checkStatus or not voData[dn]['Suspended']): + if not role or role in voData[dn]['ActiveRoles' if checkStatus else 'VOMSRoles']: + DNs.append(dn) + else: + DNs += userDNs + + return list(set(DNs)) def getPropertiesForGroup(groupName, defaultValue=None): @@ -298,8 +383,6 @@ def getPropertiesForEntity(group, name="", dn="", defaultValue=None): :return: defaultValue or list """ - if defaultValue is None: - defaultValue = [] if group == 'hosts': if not name: result = getHostnameForDN(dn) @@ -466,15 +549,16 @@ def getVOMSVOForGroup(group): return vomsVO -def getGroupsWithVOMSAttribute(vomsAttr): +def getGroupsWithVOMSAttribute(vomsAttr, groupsList=None): """ Search groups with VOMS attribute :param str vomsAttr: VOMS attribute + :param list groupsList: groups where need to search :return: list """ groups = [] - for group in gConfig.getSections("%s/Groups" % (gBaseRegistrySection)).get('Value', []): + for group in groupsList or getAllGroups(): if vomsAttr == gConfig.getValue("%s/Groups/%s/VOMSRole" % (gBaseRegistrySection, group), ""): groups.append(group) return groups @@ -569,10 +653,10 @@ def getVOMSRoleGroupMapping(vo=''): def getUsernameForID(ID, usersList=None): """ Get DIRAC user name by ID - :param basestring ID: user ID + :param str ID: user ID :param list usersList: list of DIRAC user names - :return: S_OK(basestring)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ if not usersList: result = gConfig.getSections("%s/Users" % gBaseRegistrySection) @@ -585,72 +669,30 @@ def getUsernameForID(ID, usersList=None): return S_ERROR("No username found for ID %s" % ID) -def getCAForUsername(username): - """ Get CA option by user name - - :param str username: user name - - :return: S_OK(str)/S_ERROR() - """ - dnList = gConfig.getValue("%s/Users/%s/CA" % (gBaseRegistrySection, username), []) - return S_OK(dnList) if dnList else S_ERROR("No CA found for user %s" % username) +def getDNProperty(dn, prop, defaultValue=None, username=None): + """ Get user DN property - -def getDNProperty(userDN, value, defaultValue=None): - """ Get property from DNProperties section by user DN - - :param str userDN: user DN - :param str value: option that need to get + :param str dn: user DN + :param str prop: property name :param defaultValue: default value + :param str username: username - :return: S_OK()/S_ERROR() -- str or list that contain option value - """ - result = getUsernameForDN(userDN) - if not result['OK']: - return result - pathDNProperties = "%s/Users/%s/DNProperties" % (gBaseRegistrySection, result['Value']) - result = gConfig.getSections(pathDNProperties) - if result['OK']: - for section in result['Value']: - if userDN == gConfig.getValue("%s/%s/DN" % (pathDNProperties, section)): - return S_OK(gConfig.getValue("%s/%s/%s" % (pathDNProperties, section, value), defaultValue)) - return S_OK(defaultValue) - - -def getProxyProvidersForDN(userDN): - """ Get proxy providers by user DN - - :param str userDN: user DN - - :return: S_OK(list)/S_ERROR() + :return: S_OK()/S_ERROR() """ - return getDNProperty(userDN, 'ProxyProviders', []) - - -def getDNFromProxyProviderForUserID(proxyProvider, userID): - """ Get groups by user DN in DNProperties - - :param str proxyProvider: proxy provider name - :param str userID: user identificator + if not username: + result = getUsernameForDN(dn) + if not result['OK']: + return result + username = result['Value'] - :return: S_OK(str)/S_ERROR() - """ - # Get user name - result = getUsernameForID(userID) + root = "%s/Users/%s/DNProperties" % (gBaseRegistrySection, username) + result = gConfig.getSections(root) if not result['OK']: return result - # Get DNs from user - result = getDNForUsername(result['Value']) - if not result['OK']: - return result - for DN in result['Value']: - result = getProxyProvidersForDN(DN) - if not result['OK']: - return result - if proxyProvider in result['Value']: - return S_OK(DN) - return S_ERROR(errno.ENODATA, - "No DN found for %s proxy provider for user ID %s" % (proxyProvider, userID)) + for section in result['Value']: + if dn == gConfig.getValue("%s/%s/DN" % (root, section)): + return S_OK(gConfig.getValue("%s/%s/%s" % (root, section, prop), defaultValue)) + return S_OK(defaultValue) def isDownloadableGroup(groupName): @@ -665,24 +707,6 @@ def isDownloadableGroup(groupName): return True -def getUserDict(username): - """ Get full information from user section - - :param str username: DIRAC user name - - :return: S_OK()/S_ERROR() - """ - resDict = {} - relPath = '%s/Users/%s/' % (gBaseRegistrySection, username) - result = gConfig.getConfigurationTree(relPath) - if not result['OK']: - return result - for key, value in result['Value'].items(): - if value: - resDict[key.replace(relPath, '')] = value - return S_OK(resDict) - - def getEmailsForGroup(groupName): """ Get email list of users in group @@ -695,3 +719,104 @@ def getEmailsForGroup(groupName): email = getUserOption(username, 'Email', []) emails.append(email) return emails + + +def getIDsForUsername(username): + """ Return IDs for DIRAC user + + :param str username: DIRAC user + + :return: list -- contain IDs + """ + return gConfig.getValue("%s/Users/%s/ID" % (gBaseRegistrySection, username), []) + + +def getVOsWithVOMS(voList=None): + """ Get all the configured VOMS VOs + + :param list voList: VOs where to look + + :return: S_OK(list)/S_ERROR() + """ + vos = [] + if not voList: + result = getVOs() + if result['OK']: + voList = result['Value'] + for vo in voList or []: + if getVOOption(vo, 'VOMSName'): + vos.append(vo) + return S_OK(vos) + + +def getDNsForUsername(username): + """ Find all DNs for DIRAC user + + :param str username: DIRAC user + + :return: S_OK(list)/S_ERROR() -- contain DNs + """ + userDNs = getDNsForUsernameFromCS(username) + for uid in getIDsForUsername(username): + result = gAuthManagerData.getDNsForID(uid) + if result['OK']: + userDNs += result['Value'] + return S_OK(list(set(userDNs))) + + +def getDNForUsernameInGroup(username, group, checkStatus=False): + """ Get user DN for user in group + + :param str username: user name + :param str group: group name + :param bool checkStatus: don't add suspended DNs + + :return: S_OK(str)/S_ERROR() + """ + result = getDNsForUsernameInGroup(username, group, checkStatus) + return S_OK(result['Value'][0]) if result['OK'] else result + + +def getDNsForUsernameInGroup(username, group, checkStatus=False): + """ Get user DN for user in group + + :param str username: user name + :param str group: group name + :param bool checkStatus: don't add suspended DNs + + :return: S_OK(str)/S_ERROR() + """ + if username not in getGroupOption(group, 'Users', []): + return S_ERROR('%s group not have %s user.' % (group, username)) + DNs = [] + result = getDNsForUsername(username) + if not result['OK']: + return result + userDNs = result['Value'] + + vo = getGroupOption(group, 'VO') + if checkStatus and vo in getUserOption(username, 'Suspended', []): + return S_ERROR('%s marked as suspended for %s VO.' % (username, vo)) + result = getVOsWithVOMS([vo]) + if not result['OK']: + return result + if result['Value']: + result = getVOMSInfo(vo=vo) + if not result['OK']: + return result + vomsData = result['Value'] + if vomsData.get(vo) and vomsData[vo]['OK']: + voData = vomsData[vo]['Value'] + role = getGroupOption(group, 'VOMSRole') + for dn in userDNs: + if dn in voData and (not checkStatus or not voData[dn]['Suspended']): + if not role or role in voData[dn]['ActiveRoles' if checkStatus else 'VOMSRoles']: + DNs.append(dn) + else: + DNs += userDNs + else: + DNs += userDNs + + if DNs: + return S_OK(list(set(DNs))) + return S_ERROR('For %s@%s not found DN%s.' % (username, group, ' or it suspended' if checkStatus else '')) diff --git a/ConfigurationSystem/Client/Helpers/Resources.py b/ConfigurationSystem/Client/Helpers/Resources.py index 28d88a719de..788d93e42ad 100644 --- a/ConfigurationSystem/Client/Helpers/Resources.py +++ b/ConfigurationSystem/Client/Helpers/Resources.py @@ -465,42 +465,68 @@ def getFilterConfig(filterID): return gConfig.getOptionsDict('Resources/LogFilters/%s' % filterID) -def getInfoAboutProviders(of=None, providerName=None, option='', section=''): - """ Get the information about providers - - :param basestring of: provider of what(Id, Proxy or etc.) need to look, - None, "all" to get list of instance of what this providers - :param basestring providerName: provider name, - None, "all" to get list of providers names - :param basestring option: option name that need to get, - None, "all" to get all options in a section - :param basestring section: section path in root section of provider, - "all" to get options in all sections - - :return: S_OK()/S_ERROR() +def getProvidersForInstance(instance, providerType=None): + """ Get providers for instance + + :param str instance: instance of what this providers + :param str providerType: provider type + + :return: S_OK(list)/S_ERROR() + """ + result = gConfig.getSections('%s/%sProviders' % (gBaseResourcesSection, instance)) + if not result['OK'] or not result['Value'] or (result['OK'] and not providerType): + return result + data = [] + for prov in result['Value']: + if providerType == gConfig.getValue('%s/%sProviders/%s/ProviderType' % + (gBaseResourcesSection, instance, prov)): + data.append(prov) + return S_OK(data) + + +def getProviderByAlias(alias, instance=None): + """ Find provider name by alias + + :param str alias: other registered provider name + :param str instance: provider of what + + :return: S_OK(str)/S_ERROR() """ - if not of or of == "all": + instances = [instance] or [] + if not instances: result = gConfig.getSections(gBaseResourcesSection) if not result['OK']: return result - return S_OK([i.replace('Providers', '') for i in result['Value']]) - if not providerName or providerName == "all": - return gConfig.getSections('%s/%sProviders' % (gBaseResourcesSection, of)) - if not option or option == 'all': - if not section: - return gConfig.getOptionsDict("%s/%sProviders/%s" % (gBaseResourcesSection, of, providerName)) - elif section == "all": - resDict = {} - relPath = "%s/%sProviders/%s/" % (gBaseResourcesSection, of, providerName) - result = gConfig.getConfigurationTree(relPath) + for section in result['Value']: + if section[-9:] == 'Providers': + instances.append(section[:-9]) + for instance in instances: + result = getProvidersForInstance(instance) + if not result['OK']: + return result + for provider in result['Value']: + if alias in gConfig.getValue("%s/%sProviders/%s/Aliases" % (gBaseResourcesSection, + instance, provider), []): + return S_OK(provider) + return S_ERROR('No found any provider for %s' % alias) + + +def getProviderInfo(provider): + """ Get provider info + + :param str provider: provider + + :return: S_OK(dict)/S_ERROR() + """ + result = gConfig.getSections(gBaseResourcesSection) + if not result['OK']: + return result + for section in result['Value']: + if section[-9:] == 'Providers': + result = getProvidersForInstance(section[:-9]) if not result['OK']: return result - for key, value in result['Value'].items(): # can be an iterator - if value: - resDict[key.replace(relPath, '')] = value - return S_OK(resDict) - else: - return gConfig.getSections('%s/%sProviders/%s/%s/' % (gBaseResourcesSection, of, providerName, section)) - else: - return S_OK(gConfig.getValue('%s/%sProviders/%s/%s/%s' % (gBaseResourcesSection, of, providerName, - section, option))) + if provider in result['Value']: + return gConfig.getOptionsDictRecursively("%s/%s/%s/" % (gBaseResourcesSection, + section, provider)) + return S_ERROR('%s provider not found.' % provider) diff --git a/ConfigurationSystem/Client/Utilities.py b/ConfigurationSystem/Client/Utilities.py index a8f788cea1a..2f7f8b98c64 100644 --- a/ConfigurationSystem/Client/Utilities.py +++ b/ConfigurationSystem/Client/Utilities.py @@ -701,14 +701,14 @@ def getElasticDBParameters(fullname): return S_OK(parameters) -def getOAuthAPI(instance='Production'): - """ Get OAuth API url +def getAuthAPI(instance='Production'): + """ Get Auth REST API url :param str instance: instance :return: str """ - return gConfig.getValue("/Systems/Framework/%s/URLs/OAuthAPI" % instance) + return gConfig.getValue("/Systems/Framework/%s/URLs/AuthAPI" % instance) def getDIRACGOCDictionary(): diff --git a/ConfigurationSystem/Utilities/ConfigurationHandler.py b/ConfigurationSystem/Utilities/ConfigurationHandler.py new file mode 100644 index 00000000000..b131ac36e09 --- /dev/null +++ b/ConfigurationSystem/Utilities/ConfigurationHandler.py @@ -0,0 +1,96 @@ +""" HTTP API of the DIRAC configuration data, rewrite from the RESTDIRAC project +""" +import re +import json + +from tornado import web, gen +from tornado.template import Template + +from DIRAC import S_OK, S_ERROR, gConfig, gLogger +from DIRAC.ConfigurationSystem.Client.Helpers import Resources, Registry +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager + +from DIRAC.Core.Web.WebHandler import WebHandler, asyncGen, WErr + +__RCSID__ = "$Id$" + + +class ConfigurationHandler(WebHandler): + OVERPATH = True + AUTH_PROPS = "all" + LOCATION = "/" + + def initialize(self): + super(ConfigurationHandler, self).initialize() + self.args = {} + for arg in self.request.arguments: + if len(self.request.arguments[arg]) > 1: + self.args[arg] = self.request.arguments[arg] + else: + self.args[arg] = self.request.arguments[arg][0] or '' + return S_OK() + + @asyncGen + def web_conf(self): + """ REST endpoint for configuration system: + + **GET** /conf/? -- get configuration information + + Options: + * *path* -- path in the configuration structure, by default it's "/". + * *version* -- the configuration version of the requester, if *version* is newer + than the one present on the server, an empty result will be returned + + Response: + +-----------+---------------------------------------+------------------------+ + | *key* | Description | Type | + +-----------+---------------------------------------+------------------------+ + | dump | Current CFG() | encoded in json format | + +-----------+---------------------------------------+------------------------+ + | option | Option value | text | + +-----------+---------------------------------------+------------------------+ + | options | Options list in a section | encoded in json format | + +-----------+---------------------------------------+------------------------+ + | dict | Options with values in a section | encoded in json format | + +-----------+---------------------------------------+------------------------+ + | sections | Sections list in a section | text | + +-----------+---------------------------------------+------------------------+ + """ + self.log.notice('Request configuration information') + optns = self.overpath.strip('/').split('/') + path = self.args.get('path', '/') + if not optns or len(optns) > 1: + raise WErr(404, "You forgot to set attribute.") + + result = S_ERROR('%s request unsuported' % optns[0]) + if 'version' in self.args and (self.args.get('version') or '0') >= gConfigurationData.getVersion(): + self.finish() + if optns[0] == 'dump': + remoteCFG = yield self.threadTask(gConfigurationData.getRemoteCFG) + result['Value'] = str(remoteCFG) + elif optns[0] == 'option': + result = yield self.threadTask(gConfig.getOption, path) + elif optns[0] == 'dict': + result = yield self.threadTask(gConfig.getOptionsDict, path) + elif optns[0] == 'options': + result = yield self.threadTask(gConfig.getOptions, path) + elif optns[0] == 'sections': + result = yield self.threadTask(gConfig.getSections, path) + elif optns[0] == 'getGroupsStatusByUsername': + result = yield self.threadTask(gProxyManager.getGroupsStatusByUsername, **self.args) + elif any([optns[0] == m and re.match('^[a-z][A-z]+', m) for m in dir(Registry)]) and self.isRegisteredUser(): + result = yield self.threadTask(getattr(Registry, optns[0]), **self.args) + else: + raise WErr(500, '%s request unsuported' % optns[0]) + # result = yield self.threadTask(getattr(Registry, optns[0]), **self.args) + + if not result['OK']: + raise WErr(404, result['Message']) + self.finishJEncode(result['Value']) + + @asyncGen + def post(self): + """ Post method + """ + pass diff --git a/ConfigurationSystem/Utilities/__init__.py b/ConfigurationSystem/Utilities/__init__.py new file mode 100755 index 00000000000..2492dd6ec85 --- /dev/null +++ b/ConfigurationSystem/Utilities/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +# $HeadURL$ +__RCSID__ = "$Id$" diff --git a/ConfigurationSystem/test/Test_agentOptions.py b/ConfigurationSystem/test/Test_agentOptions.py index 4b3e8c2ebf3..5706348df8a 100644 --- a/ConfigurationSystem/test/Test_agentOptions.py +++ b/ConfigurationSystem/test/Test_agentOptions.py @@ -75,7 +75,7 @@ ('DIRAC.WorkloadManagementSystem.Agent.StatesAccountingAgent', {}), ('DIRAC.WorkloadManagementSystem.Agent.StatesMonitoringAgent', {}), ('DIRAC.WorkloadManagementSystem.Agent.SiteDirector', - {'SpecialMocks': {'findGenericPilotCredentials': S_OK(('a', 'b'))}}), + {'SpecialMocks': {'findGenericPilotCredentials': S_OK(('a', 'b', 'c'))}}), # ('DIRAC.WorkloadManagementSystem.Agent.MultiProcessorSiteDirector', {}), # not inheriting from AgentModule ] diff --git a/Core/Base/API.py b/Core/Base/API.py index 7d86ebe5d28..37adb98602d 100644 --- a/Core/Base/API.py +++ b/Core/Base/API.py @@ -9,7 +9,7 @@ from DIRAC import gLogger, gConfig, S_OK, S_ERROR from DIRAC.Core.Security.ProxyInfo import getProxyInfo, formatProxyInfoAsString -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNsForUsername from DIRAC.Core.Utilities.Version import getCurrentVersion __RCSID__ = '$Id$' @@ -141,9 +141,11 @@ def _getCurrentUser(self): gLogger.debug(formatProxyInfoAsString(proxyInfo)) if 'group' not in proxyInfo: return self._errorReport('Proxy information does not contain the group', res['Message']) - res = getDNForUsername(proxyInfo['username']) - if not res['OK']: - return self._errorReport('Failed to get proxies for user', res['Message']) + result = getDNsForUsername(proxyInfo['username']) + if not result['OK']: + return self._errorReport('Failed to get proxies for user', result['Message']) + if not result['Value']: + return self._errorReport('Failed to get proxies for user', "No DNs found for %s" % proxyInfo['username']) return S_OK(proxyInfo['username']) ############################################################################# diff --git a/Core/Base/DB.py b/Core/Base/DB.py index 363bd493826..b196b5e04aa 100755 --- a/Core/Base/DB.py +++ b/Core/Base/DB.py @@ -5,7 +5,7 @@ from __future__ import division from __future__ import print_function -from DIRAC import gLogger, gConfig +from DIRAC import gLogger, gConfig, S_OK, S_ERROR from DIRAC.Core.Utilities.MySQL import MySQL from DIRAC.ConfigurationSystem.Client.Utilities import getDBParameters from DIRAC.ConfigurationSystem.Client.PathFinder import getDatabaseSection @@ -18,9 +18,16 @@ class DB(MySQL): """ def __init__(self, dbname, fullname, debug=False): + """ C'or + :param str dbname: database name + :param str fullname: full name + :param bool debug: debug mode + """ + self.versionDB = 0 self.fullname = fullname database_name = dbname + self.versionTable = '%s_Version' % database_name self.log = gLogger.getSubLogger(database_name) result = getDBParameters(fullname) @@ -44,6 +51,23 @@ def __init__(self, dbname, fullname, debug=False): if not self._connected: raise RuntimeError("Can not connect to DB '%s', exiting..." % self.dbName) + # Initialize version + result = self._query("show tables") + if result['OK']: + if self.versionTable not in [t[0] for t in result['Value']]: + result = self._createTables({self.versionTable: {'Fields': {'Version': 'INTEGER NOT NULL'}, + 'PrimaryKey': 'Version'}}) + if not result['OK']: + raise RuntimeError("Can not initialize %s version: %s" % (self.dbName, result['Message'])) + result = self._query("SELECT Version FROM `%s`" % self.versionTable) + if result['OK']: + if len(result['Value']) > 0: + self.versionDB = result['Value'][0][0] + else: + result = self._update("INSERT INTO `%s` (Version) VALUES (%s)" % (self.versionTable, self.versionDB)) + if not result['OK']: + raise RuntimeError("Can not initialize %s version: %s" % (self.dbName, result['Message'])) + self.log.info("===================== MySQL ======================") self.log.info("User: " + self.dbUser) self.log.info("Host: " + self.dbHost) @@ -54,5 +78,27 @@ def __init__(self, dbname, fullname, debug=False): ############################################################################# def getCSOption(self, optionName, defaultValue=None): + """ Get option from CS + + :param str optionName: option name + :param defaultValue: default value + + :return: value that inherits the defaultValue type + """ cs_path = getDatabaseSection(self.fullname) return gConfig.getValue("/%s/%s" % (cs_path, optionName), defaultValue) + + def updateDBVersion(self, version): + """ Update DB version + + :param int version: version number + + :return: S_OK()/S_ERROR() + """ + result = self._query('DELETE FROM `%s_Version`' % self.dbName) + if result['OK']: + result = self._update("INSERT INTO `%s_Version` (Version) VALUES (%s)" % (self.dbName, version)) + if not result['OK']: + return S_ERROR("Can not initialize %s version: %s" % (self.dbName, result['Message'])) + self.versionDB = version + return S_OK() diff --git a/Core/DISET/AuthManager.py b/Core/DISET/AuthManager.py index 847043cc987..e2bfa523c8b 100755 --- a/Core/DISET/AuthManager.py +++ b/Core/DISET/AuthManager.py @@ -5,180 +5,253 @@ from __future__ import print_function import six + +from DIRAC.Core.Utilities import List +from DIRAC.Core.Security import Properties +from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.ConfigurationSystem.Client.Helpers import Registry -from DIRAC.FrameworkSystem.Client.Logger import gLogger -from DIRAC.Core.Security import Properties -from DIRAC.Core.Utilities import List __RCSID__ = "$Id$" +KW_ID = 'ID' +KW_DN = 'DN' +KW_GROUP = 'group' +KW_USERNAME = 'username' +KW_HOSTS_GROUP = 'hosts' +KW_PROPERTIES = 'properties' +KW_EXTRA_CREDENTIALS = 'extraCredentials' + + +def forwardingCredentials(credDict, logObj=gLogger): + """ Check whether the credentials are being forwarded by a valid source and extract it + + :param dict credDict: Credentials to ckeck + :param object logObj: logger + + :return: bool + """ + if isinstance(credDict.get(KW_EXTRA_CREDENTIALS), (tuple, list)): + retVal = Registry.getHostnameForDN(credDict.get(KW_DN)) + if not retVal['OK']: + logObj.debug("The credentials forwarded not by a host:", credDict.get(KW_DN)) + return False + hostname = retVal['Value'] + if Properties.TRUSTED_HOST not in Registry.getPropertiesForHost(hostname, []): + logObj.debug("The credentials forwarded by a %s host, but it is not a trusted one" % hostname) + return False + if credDict[KW_EXTRA_CREDENTIALS][0][0] == '/': + credDict[KW_DN] = credDict[KW_EXTRA_CREDENTIALS][0] + credDict[KW_ID] = None + else: + credDict[KW_ID] = credDict[KW_EXTRA_CREDENTIALS][0] + credDict[KW_DN] = None + credDict[KW_GROUP] = credDict[KW_EXTRA_CREDENTIALS][1] + del credDict[KW_EXTRA_CREDENTIALS] + return True + return False + + +def initializationOfSession(credDict, logObj=gLogger): + """ Discover the username associated to the authentication session. It will check if the selected group is valid. + The username will be included in the credentials dictionary. And will discover DN for group if last not set. + + :param dict credDict: Credentials to check + :param object logObj: logger + + :return: bool -- specifying whether the username was found + """ + # Find user + result = Registry.getUsernameForID(credDict[KW_ID]) + if not result['OK']: + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + credDict[KW_USERNAME] = result['Value'] + return True + + +def initializationOfCertificate(credDict, logObj=gLogger): + """ Discover the username associated to the certificate DN. It will check if the selected group is valid. + The username will be included in the credentials dictionary. + + :param dict credDict: Credentials to check + :param object logObj: logger + + :return: bool -- specifying whether the username was found + """ + # Search host + result = Registry.getHostnameForDN(credDict[KW_DN]) + if result['OK'] and result['Value']: + credDict[KW_USERNAME] = result['Value'] + credDict[KW_GROUP] = KW_HOSTS_GROUP + return True + elif credDict.get(KW_GROUP) == KW_HOSTS_GROUP: + logObj.warn("Cannot find hostname for DN %s: %s" % (credDict[KW_DN], result['Message'])) + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + + # Search user + result = Registry.getUsernameForDN(credDict[KW_DN]) + if not result['OK']: + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + credDict[KW_USERNAME] = result['Value'] + return True + + +def initializationOfGroup(credDict, logObj=gLogger): + """ Check/get default group + + :param dict credDict: Credentials to check + :param object logObj: logger + + :return: bool -- specifying whether the username was found + """ + # Find/check group + credDict[KW_PROPERTIES] = [] + if not credDict.get(KW_GROUP) or credDict[KW_GROUP] == 'visitor': + result = Registry.findDefaultGroupForUser(credDict[KW_USERNAME]) + if not result['OK']: + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + credDict[KW_GROUP] = result['Value'] + if credDict[KW_GROUP] == KW_HOSTS_GROUP: + credDict[KW_PROPERTIES] = Registry.getPropertiesForHost(credDict[KW_USERNAME], []) + return True + if not Registry.getGroupsForUser(credDict[KW_USERNAME], groupsList=[credDict[KW_GROUP]]).get('Value'): + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + # Get DN for user/group + result = Registry.getDNsForUsernameInGroup(credDict[KW_USERNAME], credDict[KW_GROUP], checkStatus=True) + if not result['OK']: + logObj.error(result['Message']) + credDict[KW_GROUP] = "visitor" + return False + # Set DN if authorization not througth certificate + if not credDict.get(KW_DN): + credDict[KW_DN] = result['Value'][0] + # Check if DN match for group + if credDict[KW_DN] not in result["Value"]: + logObj.error('%s DN is not match for %s group.' % (credDict[KW_DN], credDict[KW_GROUP])) + credDict[KW_GROUP] = "visitor" + return False + + # Fill group properties + credDict[KW_PROPERTIES] = Registry.getPropertiesForGroup(credDict[KW_GROUP], []) + return True + class AuthManager(object): """ Handle Service Authorization """ - __authLogger = gLogger.getSubLogger("Authorization") - KW_HOSTS_GROUP = 'hosts' - KW_DN = 'DN' - KW_GROUP = 'group' - KW_EXTRA_CREDENTIALS = 'extraCredentials' - KW_PROPERTIES = 'properties' - KW_USERNAME = 'username' def __init__(self, authSection): - """ - Constructor + """ Constructor - :type authSection: string - :param authSection: Section containing the authorization rules + :param str authSection: Section containing the authorization rules """ self.authSection = authSection def authQuery(self, methodQuery, credDict, defaultProperties=False): - """ - Check if the query is authorized for a credentials dictionary - - :type methodQuery: string - :param methodQuery: Method to test - :type credDict: dictionary - :param credDict: dictionary containing credentials for test. The dictionary can contain the DN - and selected group. - :return: Boolean result of test + """ Check if the query is authorized for a credentials dictionary + + :param str methodQuery: Method to test + :param dict credDict: dictionary containing credentials for test. The dictionary can contain the DN + and selected group. + :param defaultProperties: default properties + :type defaultProperties: list or tuple + + :return: bool -- result of test """ userString = "" - if self.KW_DN in credDict: - userString += "DN=%s" % credDict[self.KW_DN] - if self.KW_GROUP in credDict: - userString += " group=%s" % credDict[self.KW_GROUP] - if self.KW_EXTRA_CREDENTIALS in credDict: - userString += " extraCredentials=%s" % str(credDict[self.KW_EXTRA_CREDENTIALS]) + if KW_ID in credDict: + userString += " ID=%s" % credDict[KW_ID] + if KW_DN in credDict: + userString += " DN=%s" % credDict[KW_DN] + if credDict.get(KW_GROUP): + userString += " group=%s" % credDict[KW_GROUP] + if KW_EXTRA_CREDENTIALS in credDict: + userString += " extraCredentials=%s" % str(credDict[KW_EXTRA_CREDENTIALS]) self.__authLogger.debug("Trying to authenticate %s" % userString) + # Get properties requiredProperties = self.getValidPropertiesForMethod(methodQuery, defaultProperties) + lowerCaseProperties = [prop.lower() for prop in requiredProperties] or ['any'] + allowAll = "any" in lowerCaseProperties or "all" in lowerCaseProperties + # Extract valid groups validGroups = self.getValidGroups(requiredProperties) - lowerCaseProperties = [prop.lower() for prop in requiredProperties] - if not lowerCaseProperties: - lowerCaseProperties = ['any'] + self.__authLogger.info("validGroups: ", validGroups) - allowAll = "any" in lowerCaseProperties or "all" in lowerCaseProperties - # Set no properties by default - credDict[self.KW_PROPERTIES] = [] - # Check non secure backends - if self.KW_DN not in credDict or not credDict[self.KW_DN]: - if allowAll and not validGroups: - self.__authLogger.debug("Accepted request from unsecure transport") - return True + # Read extra credentials + if KW_EXTRA_CREDENTIALS in credDict: + # Is it a host? and HACK TO MAINTAIN COMPATIBILITY + if credDict.get(KW_EXTRA_CREDENTIALS) == KW_HOSTS_GROUP: + credDict[KW_GROUP] = credDict[KW_EXTRA_CREDENTIALS] + del credDict[KW_EXTRA_CREDENTIALS] + # Check if query comes though a gateway/web server + elif forwardingCredentials(credDict, logObj=self.__authLogger): + self.__authLogger.debug("Query comes from a gateway") + return self.authQuery(methodQuery, credDict, requiredProperties) else: - self.__authLogger.debug( - "Explicit property required and query seems to be coming through an unsecure transport") return False - # Check if query comes though a gateway/web server - if self.forwardedCredentials(credDict): - self.__authLogger.debug("Query comes from a gateway") - self.unpackForwardedCredentials(credDict) - return self.authQuery(methodQuery, credDict, requiredProperties) - # Get the properties - # Check for invalid forwarding - if self.KW_EXTRA_CREDENTIALS in credDict: - # Invalid forwarding? - if not isinstance(credDict[self.KW_EXTRA_CREDENTIALS], six.string_types): - self.__authLogger.debug("The credentials seem to be forwarded by a host, but it is not a trusted one") - return False - # Is it a host? - if self.KW_EXTRA_CREDENTIALS in credDict and credDict[self.KW_EXTRA_CREDENTIALS] == self.KW_HOSTS_GROUP: - # Get the nickname of the host - credDict[self.KW_GROUP] = credDict[self.KW_EXTRA_CREDENTIALS] - # HACK TO MAINTAIN COMPATIBILITY - else: - if self.KW_EXTRA_CREDENTIALS in credDict and self.KW_GROUP not in credDict: - credDict[self.KW_GROUP] = credDict[self.KW_EXTRA_CREDENTIALS] - # END OF HACK - # Get the username - if self.KW_DN in credDict and credDict[self.KW_DN]: - if self.KW_GROUP not in credDict: - result = Registry.findDefaultGroupForDN(credDict[self.KW_DN]) - if not result['OK']: - credDict[self.KW_USERNAME] = "anonymous" - credDict[self.KW_GROUP] = "visitor" - else: - credDict[self.KW_GROUP] = result['Value'] - if credDict[self.KW_GROUP] == self.KW_HOSTS_GROUP: - # For host - if not self.getHostNickName(credDict): - self.__authLogger.warn("Host is invalid") - if not allowAll: - return False - # If all, then set anon credentials - credDict[self.KW_USERNAME] = "anonymous" - credDict[self.KW_GROUP] = "visitor" - else: - # For users - username = self.getUsername(credDict) - suspended = self.isUserSuspended(credDict) - if not username: - self.__authLogger.warn("User is invalid or does not belong to the group it's saying") - if suspended: - self.__authLogger.warn("User is Suspended") - - if not username or suspended: - if not allowAll: - return False - # If all, then set anon credentials - credDict[self.KW_USERNAME] = "anonymous" - credDict[self.KW_GROUP] = "visitor" - else: - if not allowAll: - return False - credDict[self.KW_USERNAME] = "anonymous" - credDict[self.KW_GROUP] = "visitor" - # If any or all in the props, allow - allowGroup = not validGroups or credDict[self.KW_GROUP] in validGroups - if allowAll and allowGroup: - return True - # Check authorized groups - if "authenticated" in lowerCaseProperties and allowGroup: - return True - if not self.matchProperties(credDict, requiredProperties): - self.__authLogger.warn("Client is not authorized\nValid properties: %s\nClient: %s" % - (requiredProperties, credDict)) - return False - elif not allowGroup: - self.__authLogger.warn("Client is not authorized\nValid groups: %s\nClient: %s" % - (validGroups, credDict)) - return False - return True + # Check authorization + authorized = True + # Search user name + if not credDict.get(KW_USERNAME): + if credDict.get(KW_DN): + # With certificate + authorized = initializationOfCertificate(credDict, logObj=self.__authLogger) + elif credDict.get(KW_ID): + # With IdP session + authorized = initializationOfSession(credDict, logObj=self.__authLogger) + else: + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + authorized = False - def getHostNickName(self, credDict): - """ - Discover the host nickname associated to the DN. - The nickname will be included in the credentials dictionary. + # Search group + if authorized: + authorized = initializationOfGroup(credDict, logObj=self.__authLogger) - :type credDict: dictionary - :param credDict: Credentials to ckeck - :return: Boolean specifying whether the nickname was found - """ - if self.KW_DN not in credDict: - return True - if self.KW_GROUP not in credDict: - return False - retVal = Registry.getHostnameForDN(credDict[self.KW_DN]) - if not retVal['OK']: - gLogger.warn("Cannot find hostname for DN %s: %s" % (credDict[self.KW_DN], retVal['Message'])) - return False - credDict[self.KW_USERNAME] = retVal['Value'] - credDict[self.KW_PROPERTIES] = Registry.getPropertiesForHost(credDict[self.KW_USERNAME], []) - return True + # Authorize check + if allowAll or authorized: + # Properties check + if not self.matchProperties(credDict, list(set(requiredProperties) - set(['Any', 'any', + 'All', 'all', + 'authenticated', + 'Authenticated']))): + self.__authLogger.warn("Client is not authorized\nValid properties: %s\nClient: %s" % + (requiredProperties, credDict)) + return False + # Groups check + elif validGroups and credDict[KW_GROUP] not in validGroups: + self.__authLogger.warn("Client is not authorized\nValid groups: %s\nClient: %s" % + (validGroups, credDict)) + return False + else: + if not authorized: + self.__authLogger.debug("Accepted request from unsecure transport") + return True + else: + self.__authLogger.debug("User is invalid or does not belong to the group it's saying") + return False def getValidPropertiesForMethod(self, method, defaultProperties=False): - """ - Get all authorized groups for calling a method + """ Get all authorized groups for calling a method + + :param str method: Method to test + :param defaultProperties: default properties + :type defaultProperties: list or tuple - :type method: string - :param method: Method to test - :return: List containing the allowed groups + :return: list -- List containing the allowed groups """ authProps = gConfig.getValue("%s/%s" % (self.authSection, method), []) if authProps: @@ -197,11 +270,11 @@ def getValidPropertiesForMethod(self, method, defaultProperties=False): return [] def getValidGroups(self, rawProperties): - """ Get valid groups as specified in the method authorization rules + """ Get valid groups as specified in the method authorization rules + + :param list rawProperties: all method properties - :param rawProperties: all method properties - :type rawProperties: python:list - :return: list of allowed groups or [] + :return: list -- list of allowed groups """ validGroups = [] for prop in list(rawProperties): @@ -219,103 +292,26 @@ def getValidGroups(self, rawProperties): validGroups = list(set(validGroups)) return validGroups - def forwardedCredentials(self, credDict): - """ - Check whether the credentials are being forwarded by a valid source - - :type credDict: dictionary - :param credDict: Credentials to ckeck - :return: Boolean with the result - """ - if self.KW_EXTRA_CREDENTIALS in credDict and isinstance(credDict[self.KW_EXTRA_CREDENTIALS], (tuple, list)): - if self.KW_DN in credDict: - retVal = Registry.getHostnameForDN(credDict[self.KW_DN]) - if retVal['OK']: - hostname = retVal['Value'] - if Properties.TRUSTED_HOST in Registry.getPropertiesForHost(hostname, []): - return True - return False - - def unpackForwardedCredentials(self, credDict): - """ - Extract the forwarded credentials - - :type credDict: dictionary - :param credDict: Credentials to unpack - """ - credDict[self.KW_DN] = credDict[self.KW_EXTRA_CREDENTIALS][0] - credDict[self.KW_GROUP] = credDict[self.KW_EXTRA_CREDENTIALS][1] - del(credDict[self.KW_EXTRA_CREDENTIALS]) - - def getUsername(self, credDict): - """ - Discover the username associated to the DN. It will check if the selected group is valid. - The username will be included in the credentials dictionary. - - :type credDict: dictionary - :param credDict: Credentials to check - :return: Boolean specifying whether the username was found - """ - if self.KW_DN not in credDict: - return True - if self.KW_GROUP not in credDict: - result = Registry.findDefaultGroupForDN(credDict[self.KW_DN]) - if not result['OK']: - return False - credDict[self.KW_GROUP] = result['Value'] - credDict[self.KW_PROPERTIES] = Registry.getPropertiesForGroup(credDict[self.KW_GROUP], []) - usersInGroup = Registry.getUsersInGroup(credDict[self.KW_GROUP], []) - if not usersInGroup: - return False - retVal = Registry.getUsernameForDN(credDict[self.KW_DN], usersInGroup) - if retVal['OK']: - credDict[self.KW_USERNAME] = retVal['Value'] - return True - return False + def matchProperties(self, credDict, validProps, caseSensitive=False): + """ Return True if one or more properties are in the valid list of properties - def isUserSuspended(self, credDict): - """ Discover if the user is in Suspended status + :param dict credDict: credentials to match + :param list validProps: List of valid properties + :param bool caseSensitive: Map lower case properties to properties to make the check in + lowercase but return the proper case - :param dict credDict: Credentials to check - :return: Boolean True if user is Suspended + :return: bool -- specifying whether any property has matched the valid ones """ - # Update credDict if the username is not there - if self.KW_USERNAME not in credDict: - self.getUsername(credDict) - # If username or group is not known we can not judge if the user is suspended - # These cases are treated elsewhere anyway - if self.KW_USERNAME not in credDict or self.KW_GROUP not in credDict: - return False - suspendedVOList = Registry.getUserOption(credDict[self.KW_USERNAME], 'Suspended', []) - if not suspendedVOList: - return False - vo = Registry.getVOForGroup(credDict[self.KW_GROUP]) - if vo in suspendedVOList: + if not validProps: return True - return False - - def matchProperties(self, credDict, validProps, caseSensitive=False): - """ - Return True if one or more properties are in the valid list of properties - - :type props: list - :param props: List of properties to match - :type validProps: list - :param validProps: List of valid properties - :return: Boolean specifying whether any property has matched the valid ones - """ - - # HACK: Map lower case properties to properties to make the check in lowercase but return the proper case if not caseSensitive: validProps = dict((prop.lower(), prop) for prop in validProps) else: validProps = dict((prop, prop) for prop in validProps) - groupProperties = credDict[self.KW_PROPERTIES] foundProps = [] - for prop in groupProperties: + for prop in credDict[KW_PROPERTIES]: if not caseSensitive: prop = prop.lower() if prop in validProps: foundProps.append(validProps[prop]) - credDict[self.KW_PROPERTIES] = foundProps - return foundProps + return bool(foundProps) diff --git a/Core/DISET/RequestHandler.py b/Core/DISET/RequestHandler.py index 0d423b92ac1..a13046c4688 100755 --- a/Core/DISET/RequestHandler.py +++ b/Core/DISET/RequestHandler.py @@ -18,6 +18,8 @@ from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.Core.Security.Properties import CS_ADMINISTRATOR +from DIRAC.Core.DISET.AuthManager import initializationOfCertificate, initializationOfGroup,\ + forwardingCredentials, initializationOfSession def getServiceOption(serviceInfo, optionName, defaultValue): @@ -98,7 +100,14 @@ def getRemoteCredentials(self): :return: Credentials dictionary of remote peer. """ - return self.__trPool.get(self.__trid).getConnectingCredentials() + credDict = self.__trPool.get(self.__trid).getConnectingCredentials() + forwardingCredentials(credDict, logObj=self.log) + if credDict.get('ID'): + initializationOfSession(credDict, logObj=self.log) + else: + initializationOfCertificate(credDict, logObj=self.log) + initializationOfGroup(credDict, logObj=self.log) + return credDict @classmethod def getCSOption(cls, optionName, defaultValue=False): diff --git a/Core/DISET/ThreadConfig.py b/Core/DISET/ThreadConfig.py index bc86d58936a..8ca8e5dc13b 100644 --- a/Core/DISET/ThreadConfig.py +++ b/Core/DISET/ThreadConfig.py @@ -28,6 +28,7 @@ def reset(self): """ Reset extra information """ self.__DN = False + self.__ID = False self.__group = False self.__deco = False self.__setup = False @@ -74,21 +75,19 @@ def getGroup(self): """ return self.__group - def setID(self, DN, group): + def setID(self, ID): """ Set user ID - :param str DN: user DN - :param str group: user group + :param str ID: user ID """ - self.__DN = DN - self.__group = group + self.__ID = ID def getID(self): """ Return user ID - :return: tuple + :return: str """ - return (self.__DN, self.__group) + return self.__ID def setSetup(self, setup): """ Set setup name @@ -109,19 +108,17 @@ def dump(self): :return: tuple """ - return (self.__DN, self.__group, self.__setup) + return (self.__DN, self.__group, self.__setup, self.__ID) def load(self, tp): """ Save extra information - :param tuple tp: contain DN, group name, setup name + :param tuple tp: contain DN, group name, setup name, ID """ - if tp[0]: - self.__DN = tp[0] - if tp[1]: - self.__group = tp[1] - if tp[2]: - self.__setup = tp[2] + self.__ID = tp[3] or self.__ID + self.__DN = tp[0] or self.__DN + self.__group = tp[1] or self.__group + self.__setup = tp[2] or self.__setup def threadDeco(method): diff --git a/Core/DISET/private/BaseClient.py b/Core/DISET/private/BaseClient.py index 9eeae56ddfc..69474c973d4 100755 --- a/Core/DISET/private/BaseClient.py +++ b/Core/DISET/private/BaseClient.py @@ -13,11 +13,10 @@ import DIRAC from DIRAC.Core.DISET.private.Protocols import gProtocolDict from DIRAC.FrameworkSystem.Client.Logger import gLogger -from DIRAC.Core.Utilities import List, Network +from DIRAC.Core.Utilities import List, Network, DErrno from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL, getServiceFailoverURL -from DIRAC.ConfigurationSystem.Client.Helpers import Registry from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck from DIRAC.Core.DISET.private.TransportPool import getGlobalTransportPool from DIRAC.Core.DISET.ThreadConfig import ThreadConfig @@ -35,6 +34,7 @@ class BaseClient(object): KW_TIMEOUT = "timeout" KW_SETUP = "setup" KW_VO = "VO" + KW_DELEGATED_ID = "delegatedID" KW_DELEGATED_DN = "delegatedDN" KW_DELEGATED_GROUP = "delegatedGroup" KW_IGNORE_GATEWAYS = "ignoreGateways" @@ -256,17 +256,19 @@ def __discoverExtraCredentials(self): self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS] # Are we delegating something? + delegatedID = self.kwargs.get(self.KW_DELEGATED_ID) or self.__threadConfig.getID() delegatedDN = self.kwargs.get(self.KW_DELEGATED_DN) or self.__threadConfig.getDN() delegatedGroup = self.kwargs.get(self.KW_DELEGATED_GROUP) or self.__threadConfig.getGroup() + + if delegatedID: + self.kwargs[self.KW_DELEGATED_ID] = delegatedID if delegatedDN: self.kwargs[self.KW_DELEGATED_DN] = delegatedDN - if not delegatedGroup: - result = Registry.findDefaultGroupForDN(delegatedDN) - if not result['OK']: - return result - delegatedGroup = result['Value'] + if delegatedGroup: self.kwargs[self.KW_DELEGATED_GROUP] = delegatedGroup - self.__extraCredentials = (delegatedDN, delegatedGroup) + + if delegatedID or delegatedDN: + self.__extraCredentials = (delegatedID or delegatedDN, delegatedGroup) return S_OK() def __findServiceURL(self): @@ -496,10 +498,10 @@ def _connect(self): # try to reconnect return self._connect() else: - return retVal + return S_ERROR(DErrno.ECONNECT, retVal['Message']) except Exception as e: gLogger.exception(lException=True, lExcInfo=True) - return S_ERROR("Can't connect to %s: %s" % (self.serviceURL, repr(e))) + return S_ERROR(DErrno.ECONNECT, "Can't connect to %s: %s" % (self.serviceURL, repr(e))) # We add the connection to the transport pool gLogger.debug("Connected to: %s" % self.serviceURL) trid = getGlobalTransportPool().add(transport) diff --git a/Core/DISET/test/Test_AuthManager.py b/Core/DISET/test/Test_AuthManager.py index 56645994f00..814f6998b96 100644 --- a/Core/DISET/test/Test_AuthManager.py +++ b/Core/DISET/test/Test_AuthManager.py @@ -4,14 +4,44 @@ from __future__ import division from __future__ import print_function +import os +import mock +import pickle import unittest from diraccfg import CFG -from DIRAC import gConfig +from DIRAC import gConfig, rootPath, S_OK, S_ERROR from DIRAC.Core.DISET.AuthManager import AuthManager __RCSID__ = "$Id$" +workDir = os.path.join(gConfig.getValue('/LocalSite/InstancePath', rootPath), 'work/ProxyManager') + +voDict = { + 'testVO': S_OK({ + '/User/test/DN/CN=userS': { + 'Suspended': True, + 'VOMSRoles': [u'/testVO'], + 'ActiveRoles': [], + 'SuspendedRoles': [u'/testVO'] + }, + '/User/test/DN/CN=userA': { + 'Suspended': False, + 'VOMSRoles': [u'/testVO'], + 'ActiveRoles': [u'/testVO'], + 'SuspendedRoles': [] + } + }), + 'testVOOther': S_OK({ + '/User/test/DN/CN=userS': { + 'Suspended': False, + 'VOMSRoles': [u'/testVOOther'], + 'ActiveRoles': [u'/testVOOther'], + 'SuspendedRoles': [] + } + }) +} + testSystemsCFG = """ Systems { @@ -46,6 +76,7 @@ testVO { VOAdmin = userA + VOMSName = testVO } testVOBad { @@ -54,6 +85,7 @@ testVOOther { VOAdmin = userA + VOMSName = testVOOther } } Users @@ -91,6 +123,7 @@ { Users = userA, userS VO = testVO + VOMSRole = /testVO Properties = NormalUser } group_test_other @@ -110,9 +143,28 @@ """ +def sf_getVOMSInfo(vo=None, dn=None): + return S_OK(voDict) + + +@mock.patch('DIRAC.ConfigurationSystem.Client.Helpers.Registry.getVOMSInfo', + new=sf_getVOMSInfo) class AuthManagerTest(unittest.TestCase): """ Base class for the Modules test cases """ + @classmethod + def setUpClass(cls): + if not os.path.exists(workDir): + os.makedirs(workDir) + for vo, infoDict in voDict.items(): + with open(os.path.join(workDir, vo + '.pkl'), 'wb+') as f: + pickle.dump(infoDict, f, pickle.HIGHEST_PROTOCOL) + + @classmethod + def tearDownClass(cls): + for vo in voDict.keys(): + if os.path.exists(os.path.join(workDir, vo + '.pkl')): + os.remove(os.path.join(workDir, vo + '.pkl')) def setUp(self): self.authMgr = AuthManager('/Systems/Service/Authorization') diff --git a/Core/Security/VOMSService.py b/Core/Security/VOMSService.py index 5b34ea7db71..26100b21310 100644 --- a/Core/Security/VOMSService.py +++ b/Core/Security/VOMSService.py @@ -59,7 +59,7 @@ def attGetUserNickname(self, dn, _ca=None): :param str dn: user DN :param str _ca: CA, kept for backward compatibility - :return: S_OK with Value: nickname + :return: S_OK with Value: nickname """ if self.userDict is None: @@ -75,16 +75,18 @@ def attGetUserNickname(self, dn, _ca=None): return S_ERROR(DErrno.EVOMS, "No nickname defined") return S_OK(nickname) - def getUsers(self): + def getUsers(self, proxyPath=None): """ Get all the users of the VOMS VO with their detailed information + :param str proxyPath: proxy path + :return: user dictionary keyed by the user DN """ if not self.urls: return S_ERROR(DErrno.ENOAUTH, "No VOMS server defined") - userProxy = getProxyLocation() + userProxy = proxyPath or getProxyLocation() caPath = getCAsLocation() rawUserList = [] result = None diff --git a/Core/Utilities/DErrno.py b/Core/Utilities/DErrno.py index 45d58a14136..920593c467d 100644 --- a/Core/Utilities/DErrno.py +++ b/Core/Utilities/DErrno.py @@ -91,6 +91,7 @@ # DISET: 1X EDISET = 1110 ENOAUTH = 1111 +ECONNECT = 1112 # 3rd party security: 2X E3RDPARTY = 1120 EVOMS = 1121 @@ -167,6 +168,7 @@ # 111X: DISET 1110: 'EDISET', 1111: 'ENOAUTH', + 1112: 'ECONNECT', # 112X: 3rd party security 1120: 'E3RDPARTY', 1121: 'EVOMS', @@ -239,6 +241,7 @@ # 111X: DISET EDISET: "DISET Error", ENOAUTH: "Unauthorized query", + ECONNECT: "Connection error", # 112X: 3rd party security E3RDPARTY: "3rd party security service error", EVOMS: "VOMS Error", diff --git a/Core/Utilities/DictCache.py b/Core/Utilities/DictCache.py index b4bde787ea3..2d5767ae2d4 100644 --- a/Core/Utilities/DictCache.py +++ b/Core/Utilities/DictCache.py @@ -207,6 +207,24 @@ def getKeys(self, validSeconds=0): finally: self.lock.release() + def getDict(self, validSeconds=0): + """ Get dictionary for all contents + + :param int validSeconds: valid time in seconds + + :return: dict + """ + self.lock.acquire() + try: + resDict = {} + limitTime = datetime.datetime.now() + datetime.timedelta(seconds=validSeconds) + for cKey in self.__cache: + if self.__cache[cKey]['expirationTime'] > limitTime: + resDict[cKey] = self.__cache[cKey]['value'] + return resDict + finally: + self.lock.release() + def purgeExpired(self, expiredInSeconds=0): """ Purge all entries that are expired or will be expired in diff --git a/Core/Utilities/Proxy.py b/Core/Utilities/Proxy.py index 655a5a99651..63f8d4b77fa 100644 --- a/Core/Utilities/Proxy.py +++ b/Core/Utilities/Proxy.py @@ -41,10 +41,9 @@ def undecoratedFunction(foo='bar'): import os from DIRAC import gConfig, gLogger, S_ERROR, S_OK +from DIRAC.Core.Utilities.LockRing import LockRing from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOMSAttributeForGroup, getDNForUsername -from DIRAC.Core.Utilities.LockRing import LockRing __RCSID__ = "$Id$" @@ -104,30 +103,32 @@ def wrapped_fcn(*args, **kwargs): return wrapped_fcn -def getProxy(userDNs, userGroup, vomsAttr, proxyFilePath): - """ do the actual download of the proxy, trying the different DNs +def getProxy(user, userGroup, vomsAttr, proxyFilePath): + """ Do the actual download of the proxy, trying the different DNs + + :param str user: user name or DN + :param str userGroup: group name + :param bool vomsAttr: if need VOMSproxy + :param str proxyPathFile: path to proxy file + + :return: S_OK(object)/S_ERROR() -- return proxy as chain """ - for userDN in userDNs: - if vomsAttr: - result = gProxyManager.downloadVOMSProxyToFile(userDN, userGroup, - requiredVOMSAttribute=vomsAttr, - filePath=proxyFilePath, - requiredTimeLeft=3600, - cacheTime=3600) - else: - result = gProxyManager.downloadProxyToFile(userDN, userGroup, - filePath=proxyFilePath, - requiredTimeLeft=3600, - cacheTime=3600) - - if not result['OK']: - gLogger.error("Can't download %sproxy " % ('VOMS' if vomsAttr else ''), - "of '%s', group %s to file: " % (userDN, userGroup) + result['Message']) - else: - return result + if vomsAttr: + result = gProxyManager.downloadVOMSProxyToFile(user, userGroup, + filePath=proxyFilePath, + requiredTimeLeft=3600, + cacheTime=3600) + else: + result = gProxyManager.downloadProxyToFile(user, userGroup, + filePath=proxyFilePath, + requiredTimeLeft=3600, + cacheTime=3600) - # If proxy not found for any DN, return an error - return S_ERROR("Can't download proxy") + if not result['OK']: + gLogger.error("Can't download %sproxy " % ('VOMS' if vomsAttr else ''), + "of '%s', group %s to file: " % (user, userGroup) + result['Message']) + return S_ERROR("Can't download proxy") + return result def executeWithoutServerCertificate(fcn): @@ -151,7 +152,8 @@ def executeWithoutServerCertificate(fcn): """ def wrapped_fcn(*args, **kwargs): - + """ Wraped fuction + """ # Get the lock and acquire it executionLock = LockRing().getLock('_UseUserProxy_', recursive=True) executionLock.acquire() @@ -225,20 +227,7 @@ def _putProxy(userDN=None, userName=None, userGroup=None, vomsFlag=None, proxyFi :returns: Tuple of originalUserProxy, useServerCertificate, executionLock """ # Setup user proxy - if userDN: - userDNs = [userDN] - else: - result = getDNForUsername(userName) - if not result['OK']: - return result - userDNs = result['Value'] # a same user may have more than one DN - - vomsAttr = '' - if vomsFlag: - vomsAttr = getVOMSAttributeForGroup(userGroup) - - result = getProxy(userDNs, userGroup, vomsAttr, proxyFilePath) - + result = getProxy(userDN or userName, userGroup, vomsFlag, proxyFilePath) if not result['OK']: return result diff --git a/Core/Utilities/Shifter.py b/Core/Utilities/Shifter.py index 02928783e4c..5161ee348c9 100644 --- a/Core/Utilities/Shifter.py +++ b/Core/Utilities/Shifter.py @@ -14,7 +14,8 @@ from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath -from DIRAC.ConfigurationSystem.Client.Helpers import Registry +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import findDefaultGroupForUser,\ + getDNForUsernameInGroup, getVOMSAttributeForGroup def getShifterProxy(shifterType, fileName=False): @@ -31,26 +32,28 @@ def getShifterProxy(shifterType, fileName=False): userName = opsHelper.getValue(cfgPath('Shifter', shifterType, 'User'), '') if not userName: return S_ERROR("No shifter User defined for %s" % shifterType) - result = Registry.getDNForUsername(userName) - if not result['OK']: - return result - userDN = result['Value'][0] - result = Registry.findDefaultGroupForDN(userDN) + result = findDefaultGroupForUser(userName) if not result['OK']: return result defaultGroup = result['Value'] userGroup = opsHelper.getValue(cfgPath('Shifter', shifterType, 'Group'), defaultGroup) - vomsAttr = Registry.getVOMSAttributeForGroup(userGroup) + result = getDNForUsernameInGroup(userName, userGroup) + if not result['OK']: + return result + userDN = result['Value'] + if not userDN: + return S_ERROR('No user DN found for shifter %s@%s' % (userName, userGroup)) + vomsAttr = getVOMSAttributeForGroup(userGroup) if vomsAttr: gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" % (vomsAttr, userName, userGroup, userDN)) - result = gProxyManager.downloadVOMSProxyToFile(userDN, userGroup, + result = gProxyManager.downloadVOMSProxyToFile(userName, userGroup, filePath=fileName, requiredTimeLeft=86400, cacheTime=86400) else: gLogger.info("Getting proxy for shifter %s@%s (%s)" % (userName, userGroup, userDN)) - result = gProxyManager.downloadProxyToFile(userDN, userGroup, + result = gProxyManager.downloadProxyToFile(userName, userGroup, filePath=fileName, requiredTimeLeft=86400, cacheTime=86400) diff --git a/Core/Web/App.py b/Core/Web/App.py new file mode 100644 index 00000000000..182f7d40102 --- /dev/null +++ b/Core/Web/App.py @@ -0,0 +1,230 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import ssl +import imp +import sys +import signal +import tornado.web +import tornado.process +import tornado.httpserver +import tornado.autoreload + +from diraccfg import CFG + +import DIRAC + +from DIRAC import gLogger, gConfig +from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals +from DIRAC.Core.Web.HandlerMgr import HandlerMgr +from DIRAC.Core.Web.TemplateLoader import TemplateLoader +from DIRAC.Core.Web.SessionData import SessionData +from DIRAC.Core.Web import Conf + +__RCSID__ = "$Id$" + + +class App(object): + + def __init__(self, handlersLoc='WebApp.handler'): + self.__handlerMgr = HandlerMgr(handlersLoc, Conf.rootURL()) + self.__servers = {} + self.log = gLogger.getSubLogger("Web") + + def _logRequest(self, handler): + """ Log request """ + status = handler.get_status() + if status < 400: + logm = self.log.notice + elif status < 500: + logm = self.log.warn + else: + logm = self.log.error + request_time = 1000.0 * handler.request.request_time() + logm("%d %s %.2fms" % (status, handler._request_summary(), request_time)) + + def __reloadAppCB(self): + """ Reload """ + gLogger.notice("\n !!!!!! Reloading web app...\n") + + def _loadWebAppCFGFiles(self): + """ + Load WebApp/web.cfg definitions + """ + exts = [] + for ext in CSGlobals.getCSExtensions(): + if ext == "DIRAC": + continue + if ext[-5:] != "DIRAC": + ext = "%sDIRAC" % ext + if ext != "WebAppDIRAC": + exts.append(ext) + exts.append("DIRAC") + exts.append("WebAppDIRAC") + webCFG = CFG() + for modName in reversed(exts): + try: + modPath = imp.find_module(modName)[1] + except ImportError: + continue + gLogger.verbose("Found module %s at %s" % (modName, modPath)) + cfgPath = os.path.join(modPath, "WebApp", "web.cfg") + if not os.path.isfile(cfgPath): + gLogger.verbose("Inexistant %s" % cfgPath) + continue + try: + modCFG = CFG().loadFromFile(cfgPath) + except Exception as excp: + gLogger.error("Could not load %s: %s" % (cfgPath, excp)) + continue + gLogger.verbose("Loaded %s" % cfgPath) + expl = [Conf.BASECS] + while len(expl): + current = expl.pop(0) + if not modCFG.isSection(current): + continue + if modCFG.getOption("%s/AbsoluteDefinition" % current, False): + gLogger.verbose("%s:%s is an absolute definition" % (modName, current)) + try: + webCFG.deleteKey(current) + except BaseException: + pass + modCFG.deleteKey("%s/AbsoluteDefinition" % current) + else: + for sec in modCFG[current].listSections(): + expl.append("%s/%s" % (current, sec)) + # Add the modCFG + webCFG = webCFG.mergeWith(modCFG) + gConfig.loadCFG(webCFG) + + def _loadDefaultWebCFG(self): + """ This method reloads the web.cfg file from etc/web.cfg + + :return: bool + """ + modCFG = None + cfgPath = os.path.join(DIRAC.rootPath, 'etc', 'web.cfg') + isLoaded = True + if not os.path.isfile(cfgPath): + isLoaded = False + else: + try: + modCFG = CFG().loadFromFile(cfgPath) + except Exception as excp: + isLoaded = False + gLogger.error("Could not load %s: %s" % (cfgPath, excp)) + + if modCFG: + if modCFG.isSection("/Website"): + gLogger.warn("%s configuration file is not correct. It is used by the old portal!" % (cfgPath)) + isLoaded = False + else: + gConfig.loadCFG(modCFG) + else: + isLoaded = False + + return isLoaded + + def stopChildProcesses(self, sig, frame): + """ + It is used to properly stop tornado when more than one process is used. + In principle this is doing the job of runsv.... + + :param int sig: the signal sent to the process + :param object frame: execution frame which contains the child processes + """ + # tornado.ioloop.IOLoop.instance().add_timeout(time.time()+5, sys.exit) + for child in frame.f_locals.get('children', []): + gLogger.info("Stopping child processes: %d" % child) + os.kill(child, signal.SIGTERM) + # tornado.ioloop.IOLoop.instance().stop() + # gLogger.info('exit success') + sys.exit(0) + + def bootstrap(self): + """ + Configure and create web app + """ + self.log.always("\n ====== Starting DIRAC web app ====== \n") + + # Load required CFG files + if not self._loadDefaultWebCFG(): + # if we have a web.cfg under etc directory we use it, otherwise + # we use the configuration file defined by the developer + self._loadWebAppCFGFiles() + # Calculating routes + result = self.__handlerMgr.getRoutes() + if not result['OK']: + return result + routes = result['Value'] + # Initialize the session data + SessionData.setHandlers(self.__handlerMgr.getHandlers()['Value']) + # Create the app + tLoader = TemplateLoader(self.__handlerMgr.getPaths("template")) + kw = dict(debug=Conf.devMode(), template_loader=tLoader, cookie_secret=str(Conf.cookieSecret()), + log_function=self._logRequest, autoreload=Conf.numProcesses() < 2) + + # please do no move this lines. The lines must be before the fork_processes + signal.signal(signal.SIGTERM, self.stopChildProcesses) + signal.signal(signal.SIGINT, self.stopChildProcesses) + + # Check processes if we're under a load balancert + if Conf.balancer() and Conf.numProcesses() not in (0, 1): + tornado.process.fork_processes(Conf.numProcesses(), max_restarts=0) + kw['debug'] = False + # Debug mode? + if kw['debug']: + self.log.info("Configuring in developer mode...") + # Configure tornado app + self.__app = tornado.web.Application(routes, **kw) + self.log.notice("Configuring HTTP on port %s" % (Conf.HTTPPort())) + # Create the web servers + srv = tornado.httpserver.HTTPServer(self.__app, xheaders=True) + port = Conf.HTTPPort() + srv.listen(port) + self.__servers[('http', port)] = srv + + Conf.generateRevokedCertsFile() # it is used by nginx.... + + if Conf.HTTPS(): + self.log.notice("Configuring HTTPS on port %s" % Conf.HTTPSPort()) + sslops = dict(certfile=Conf.HTTPSCert(), + keyfile=Conf.HTTPSKey(), + cert_reqs=ssl.CERT_OPTIONAL, + ca_certs=Conf.generateCAFile(), + ssl_version=ssl.PROTOCOL_TLSv1) + + sslprotocol = str(Conf.SSLProrocol()) + aviableProtocols = [i for i in dir(ssl) if i.find('PROTOCOL') == 0] + if sslprotocol and sslprotocol != "": + if (sslprotocol in aviableProtocols): + sslops['ssl_version'] = getattr(ssl, sslprotocol) + else: + message = "%s protocol is not provided." % sslprotocol + message += "The following protocols are provided: %s" % str(aviableProtocols) + gLogger.warn(message) + + self.log.debug(" - %s" % "\n - ".join(["%s = %s" % (k, sslops[k]) for k in sslops])) + srv = tornado.httpserver.HTTPServer(self.__app, ssl_options=sslops, xheaders=True) + port = Conf.HTTPSPort() + srv.listen(port) + self.__servers[('https', port)] = srv + else: + # when NGINX is used then the Conf.HTTPS return False, it means tornado + # does not have to be configured using 443 port + Conf.generateCAFile() # if we use Nginx we have to generate the cas as well... + return result + + def run(self): + """ + Start web servers + """ + bu = Conf.rootURL().strip("/") + urls = [] + for proto, port in self.__servers: + urls.append("%s://0.0.0.0:%s/%s/" % (proto, port, bu)) + self.log.always("Listening on %s" % " and ".join(urls)) + tornado.autoreload.add_reload_hook(self.__reloadAppCB) + tornado.ioloop.IOLoop.instance().start() diff --git a/Core/Web/Conf.py b/Core/Web/Conf.py new file mode 100644 index 00000000000..0e351dfee4a --- /dev/null +++ b/Core/Web/Conf.py @@ -0,0 +1,331 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import uuid +import tempfile +import tornado.process +from DIRAC import gConfig +from DIRAC.Core.Security import Locations, X509Chain, X509CRL + +__RCSID__ = "$Id$" + +BASECS = "/WebApp" + + +def getCSValue(opt, defValue=None): + """ Get option from CS + + :param str opt: option + :param defValue: default value + + :return: defValue type result + """ + return gConfig.getValue("%s/%s" % (BASECS, opt), defValue) + + +def getCSSections(opt): + """ Get sections from CS + + :param str opt: option + + :return: S_OK(list)/S_ERROR() + """ + return gConfig.getSections("%s/%s" % (BASECS, opt)) + + +def getCSOptions(opt): + """ Get options from CS + + :param str opt: option + + :return: S_OK(list)/S_ERROR() + """ + return gConfig.getOptions("%s/%s" % (BASECS, opt)) + + +def getCSOptionsDict(opt): + """ Get options dictionary from CS + + :param str opt: option + + :return: S_OK(dict)/S_ERROR() + """ + return gConfig.getOptionsDict("%s/%s" % (BASECS, opt)) + + +def getTitle(): + """ Get title + + :return: str + """ + defVal = gConfig.getValue("/DIRAC/Configuration/Name", gConfig.getValue("/DIRAC/Setup")) + return "%s - DIRAC" % gConfig.getValue("%s/Title" % BASECS, defVal) + + +def devMode(): + """ Get development mode status + + :result: bool + """ + return getCSValue("DevelopMode", False) + + +def rootURL(): + """ Get root URL + + :return: str + """ + return getCSValue("RootURL", "/DIRAC") + + +def balancer(): + """ Get balancer + + :return: str + """ + b = getCSValue("Balancer", "").lower() + if b in ("", "none"): + return "" + return b + + +def numProcesses(): + """ Get number of processes + + :return: int + """ + return getCSValue("NumProcesses", 1) + + +def HTTPS(): + """ Get flag of enable HTTPS + + :return: bool + """ + if balancer(): + return False + return getCSValue("HTTPS/Enabled", True) + + +def HTTPPort(): + """ Get HTTP port + + :return: int + """ + if balancer(): + default = 8000 + else: + default = 8080 + procAdd = tornado.process.task_id() or 0 + return getCSValue("HTTP/Port", default) + procAdd + + +def HTTPSPort(): + """ Get HTTPS port + + :return: int + """ + return getCSValue("HTTPS/Port", 8443) + + +def HTTPSCert(): + """ Get certificate path for HTTPS + + :return: str + """ + cert = Locations.getHostCertificateAndKeyLocation() + if cert: + cert = cert[0] + else: + cert = "/opt/dirac/etc/grid-security/hostcert.pem" + return getCSValue("HTTPS/Cert", cert) + + +def HTTPSKey(): + """ Get key path for HTTPS + + :return: str + """ + key = Locations.getHostCertificateAndKeyLocation() + if key: + key = key[1] + else: + key = "/opt/dirac/etc/grid-security/hostkey.pem" + return getCSValue("HTTPS/Key", key) + + +def setup(): + """ Get setup path + + :return: str + """ + return gConfig.getValue("/DIRAC/Setup") + + +def cookieSecret(): + """ Get cookie secret + + :return: str + """ + # TODO: Store the secret somewhere + return gConfig.getValue("CookieSecret", uuid.getnode()) + + +def generateCAFile(): + """ Generate a single CA file with all the PEMs + + :return: str or bool + """ + caDir = Locations.getCAsLocation() + for fn in (os.path.join(os.path.dirname(caDir), "cas.pem"), + os.path.join(os.path.dirname(HTTPSCert()), "cas.pem"), + False): + if not fn: + fn = tempfile.mkstemp(prefix="cas.", suffix=".pem")[1] + try: + fd = open(fn, "w") + except IOError: + continue + for caFile in os.listdir(caDir): + caFile = os.path.join(caDir, caFile) + result = X509Chain.X509Chain.instanceFromFile(caFile) + if not result['OK']: + continue + chain = result['Value'] + expired = chain.hasExpired() + if not expired['OK'] or expired['Value']: + continue + fd.write(chain.dumpAllToString()['Value']) + fd.close() + return fn + return False + + +def generateRevokedCertsFile(): + """ Generate a single CA file with all the PEMs + + :return: str or bool + """ + caDir = Locations.getCAsLocation() + for fn in (os.path.join(os.path.dirname(caDir), "allRevokedCerts.pem"), + os.path.join(os.path.dirname(HTTPSCert()), "allRevokedCerts.pem"), + False): + if not fn: + fn = tempfile.mkstemp(prefix="allRevokedCerts", suffix=".pem")[1] + try: + fd = open(fn, "w") + except IOError: + continue + for caFile in os.listdir(caDir): + caFile = os.path.join(caDir, caFile) + result = X509CRL.X509CRL.instanceFromFile(caFile) + if not result['OK']: + continue + chain = result['Value'] + fd.write(chain.dumpAllToString()['Value']) + fd.close() + return fn + return False + + +def getAuthSectionForHandler(route): + """ Get auth section for handler + + :return: str + """ + return "%s/Access/%s" % (BASECS, route) + + +def getTheme(): + """ Get theme + + :return: str + """ + return getCSValue("Theme", "tabs") + + +def getIcon(): + """ Get icon path + + :return: str + """ + return getCSValue("Icon", "/static/core/img/icons/system/favicon.ico") + + +def SSLProrocol(): + """ Get ssl protocol + + :return: str + """ + return getCSValue("SSLProtcol", "") + + +def getDefaultStaticDirs(): + """ Get default static directories + + :return: list + """ + defDirs = getCSValue("DefaultStaticDirs", ['defaults', 'demo']) + if defDirs == ['None']: + return [] + return defDirs + + +def getStaticDirs(): + """ Get static directories + + :return: str + """ + return list(set(getCSValue("StaticDirs", []) + getDefaultStaticDirs())) + + +def getLogo(): + """ Get logo path + + :return: str + """ + return getCSValue("Logo", "/static/core/img/icons/system/_logo_waiting.gif") + + +def getBackgroud(): + """ Get background path + + :return: str + """ + return getCSValue("BackgroundImage", "/static/core/img/wallpapers/dirac_background_6.png") + + +def getWelcome(): + """ Get welcome + + :return: str + """ + return getCSValue("WelcomeHTML", "") + + +def bugReportURL(): + """ Get bug report URL + + :return: str + """ + return getCSValue("bugReportURL", "") + + +def getAuthNames(): + """ Get enabled id providers + + :return: S_OK(list)/S_ERROR() + """ + return getCSSections("TypeAuths") + + +def getAppSettings(app): + """ Get applications options + + :param str app: application name + + :return: S_OK(dict)/S_ERROR + """ + return gConfig.getOptionsDictRecursively("%s/%s" % (BASECS, app)) diff --git a/Core/Web/CoreHandler.py b/Core/Web/CoreHandler.py new file mode 100644 index 00000000000..20a0679d4d9 --- /dev/null +++ b/Core/Web/CoreHandler.py @@ -0,0 +1,35 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import urlparse +import tornado.web + +from DIRAC.Core.Web import Conf + +__RCSID__ = "$Id$" + + +class CoreHandler(tornado.web.RequestHandler): + + def initialize(self, action): + self.__action = action + + def get(self, setup, group, route): + if self.__action == "addSlash": + o = urlparse.urlparse(self.request.uri) + proto = self.request.protocol + if 'X-Scheme' in self.request.headers: + proto = self.request.headers['X-Scheme'] + nurl = "%s://%s%s/" % (proto, self.request.host, o.path) + if o.query: + nurl = "%s?%s" % (nurl, o.query) + self.redirect(nurl, permanent=True) + elif self.__action == "sendToRoot": + dest = "/" + rootURL = Conf.rootURL() + if rootURL: + dest += "%s/" % rootURL.strip("/") + if setup and group: + dest += "s:%s/g:%s/" % (setup, group) + self.redirect(dest) diff --git a/Core/Web/HandlerMgr.py b/Core/Web/HandlerMgr.py new file mode 100644 index 00000000000..870d75dfd77 --- /dev/null +++ b/Core/Web/HandlerMgr.py @@ -0,0 +1,175 @@ +""" Basic modules for loading handlers +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import re +import imp +import inspect +import collections + +from DIRAC import S_OK, S_ERROR, rootPath, gLogger +from DIRAC.Core.Web import Conf +from DIRAC.Core.Web.WebHandler import WebHandler, WebSocketHandler +from DIRAC.Core.Web.CoreHandler import CoreHandler +from DIRAC.Core.Web.StaticHandler import StaticHandler +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader +from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton +from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals + +__RCSID__ = "$Id$" + + +class HandlerMgr(object): + __metaclass__ = DIRACSingleton + + def __init__(self, sysService=[], baseURL="/"): + """ Constructor + + :param list sysService: DIRAC system services + :param basestring baseURL: base URL + """ + self.__baseURL = baseURL.strip("/") + if sysService and not isinstance(sysService, list): + sysService = sysService.replace(' ', '').split(',') + self.__sysServices = sysService + self.__routes = [] + self.__handlers = [] + self.__setupGroupRE = r"(?:/s:([\w-]*)/g:([\w.-]*))?" + self.__shySetupGroupRE = r"(?:/s:(?:[\w-]*)/g:(?:[\w.-]*))?" + self.log = gLogger.getSubLogger("Routing") + + def getPaths(self, dirName): + """ Get lists of paths for all installed and enabled extensions + + :param basestring dirName: path to handlers + + :return: list + """ + pathList = [] + extNames = CSGlobals.getCSExtensions() + if 'WebAppDIRAC' in extNames: + extNames.append(extNames.pop(extNames.index('WebAppDIRAC'))) + for extName in extNames: + if not extName.endswith('DIRAC'): + extName = "%sDIRAC" % extName + try: + modFile, modPath, desc = imp.find_module(extName) + # to match in the real root path to enabling module web extensions (static, templates...) + realModPath = os.path.realpath(modPath) + except ImportError: + continue + staticPath = os.path.join(realModPath, "WebApp", dirName) + if os.path.isdir(staticPath): + pathList.append(staticPath) + return pathList + + def __calculateRoutes(self): + """ Load all handlers and generate the routes + + :return: S_OK()/S_ERROR() + """ + ol = ObjectLoader() + hendlerList = [] + self.log.debug("Added services: %s" % ','.join(self.__sysServices)) + for origin in self.__sysServices: + result = ol.getObjects(origin, parentClass=WebHandler, recurse=True) + if not result['OK']: + return result + hendlerList += list(result['Value'].items()) + self.__handlers = collections.OrderedDict(hendlerList) + + staticPaths = self.getPaths("static") + self.log.verbose("Static paths found:\n - %s" % "\n - ".join(staticPaths)) + self.__routes = [] + + # Add some standard paths for static files + statDirectories = Conf.getStaticDirs() + self.log.info("The following static directories are used:%s" % str(statDirectories)) + for stdir in statDirectories: + pattern = '/%s/(.*)' % stdir + self.__routes.append((pattern, StaticHandler, dict(pathList=['%s/webRoot/www/%s' % (rootPath, stdir)]))) + self.log.debug(" - Static route: %s" % pattern) + + for pattern in ((r"/static/(.*)", r"/(favicon\.ico)", r"/(robots\.txt)")): + pattern = r"%s%s" % (self.__shySetupGroupRE, pattern) + if self.__baseURL: + pattern = "/%s%s" % (self.__baseURL, pattern) + self.__routes.append((pattern, StaticHandler, dict(pathList=staticPaths))) + self.log.debug(" - Static route: %s" % pattern) + for hn in self.__handlers: + self.log.info("Found handler %s" % hn) + handler = self.__handlers[hn] + # CHeck it has AUTH_PROPS + if isinstance(handler.AUTH_PROPS, type(None)): + return S_ERROR("Handler %s does not have AUTH_PROPS defined. Fix it!" % hn) + # Get the root for the handler + if handler.LOCATION: + handlerRoute = handler.LOCATION.strip("/") and "/%s" % handler.LOCATION.strip("/") or '' + else: + handlerRoute = hn[len(re.sub(r".[A-z]+$", "", hn)):].replace(".", "/").replace("Handler", "") + # Add the setup group RE before + baseRoute = self.__setupGroupRE + # IF theres a base url like /DIRAC add it + baseRoute = "/%s%s" % (self.__baseURL or '', baseRoute) + # Set properly the LOCATION after calculating where it is with helpers to add group and setup later + handler.LOCATION = handlerRoute + handler.PATH_RE = re.compile("%s(%s/[A-z]+|.)" % (baseRoute, handlerRoute)) + if handler.OVERPATH: + handler.PATH_RE = re.compile(handler.PATH_RE.pattern + '(/[A-z0-9=-_/|]+)?') + handler.URLSCHEMA = "/%s%%(setup)s%%(group)s%%(location)s/%%(action)s" % (self.__baseURL) + if issubclass(handler, WebSocketHandler): + handler.PATH_RE = re.compile("%s(%s)" % (baseRoute, handlerRoute)) + route = "%s(%s)" % (baseRoute, handlerRoute) + self.__routes.append((route, handler)) + self.log.verbose(" - WebSocket %s -> %s" % (handlerRoute, hn)) + self.log.debug(" * %s" % route) + continue + # Look for methods that are exported + for mName, mObj in inspect.getmembers(handler): + if inspect.ismethod(mObj) and mName.find("web_") == 0: + if mName == "web_index": + # Index methods have the bare url + self.log.verbose(" - Route %s -> %s.web_index" % (handlerRoute, hn)) + route = "%s(%s/)" % (baseRoute, handlerRoute) + self.__routes.append((route, handler)) + self.__routes.append(("%s(%s)" % (baseRoute, handlerRoute), CoreHandler, dict(action='addSlash'))) + else: + # Normal methods get the method appended without web_ + self.log.verbose(" - Route %s/%s -> %s.%s" % (handlerRoute, mName[4:], hn, mName)) + route = "%s(%s/%s)" % (baseRoute, handlerRoute, mName[4:]) + # Use request path as options/values, for ex. ../method/