From fc5100219885bb68e2adf266903790b1573f653e Mon Sep 17 00:00:00 2001 From: TaykYoku Date: Tue, 30 Jun 2020 16:45:45 +0200 Subject: [PATCH 001/105] CS: use PM and AM cli caches in Registry --- .../Client/Helpers/Registry.py | 388 ++++++++++++------ 1 file changed, 265 insertions(+), 123 deletions(-) diff --git a/ConfigurationSystem/Client/Helpers/Registry.py b/ConfigurationSystem/Client/Helpers/Registry.py index 92098ac0776..f473d247e36 100644 --- a/ConfigurationSystem/Client/Helpers/Registry.py +++ b/ConfigurationSystem/Client/Helpers/Registry.py @@ -10,6 +10,8 @@ from DIRAC.Core.Utilities import DErrno from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO +from DIRAC.FrameworkSystem.Client.ProxyManagerData import gProxyManagerData +from DIRAC.FrameworkSystem.Client.AuthManagerData import gAuthManagerData __RCSID__ = "$Id$" @@ -18,6 +20,16 @@ gBaseRegistrySection = "/Registry" +def getVOMSInfo(vo=None, dn=None): + """ Get cached information from VOMS API + + :param list dn: requested DN + + :return: S_OK(dict)/S_ERROR() + """ + return gProxyManagerData.getActualVOMSesDNs(voList=[vo] if vo else vo, dnList=[dn] if dn else dn) + + def getUsernameForDN(dn, usersList=None): """ Find DIRAC user for DN @@ -35,18 +47,26 @@ def getUsernameForDN(dn, usersList=None): for username in usersList: if dn in gConfig.getValue("%s/Users/%s/DN" % (gBaseRegistrySection, username), []): return S_OK(username) + + # Get users profiles from session manager cache + result = gAuthManagerData.getIDsForDN(dn) + if result['OK']: + for uid in result['Value']: + result = getUsernameForID(uid) + if result['OK']: + return result + return S_ERROR("No username found for dn %s" % dn) -def getDNForUsername(username): - """ Get user DN for user +def getDNsForUsernameFromSC(username): + """ Find DNs for DIRAC user from CS - :param str username: user name + :param str username: DIRAC user - :return: S_OK(str)/S_ERROR() + :return: list -- contain DNs """ - dnList = gConfig.getValue("%s/Users/%s/DN" % (gBaseRegistrySection, username), []) - return S_OK(dnList) if dnList else S_ERROR("No DN found for user %s" % username) + return gConfig.getValue("%s/Users/%s/DN" % (gBaseRegistrySection, username), []) def getDNForHost(host): @@ -60,18 +80,53 @@ def getDNForHost(host): return S_OK(dnList) if dnList else S_ERROR("No DN found for host %s" % host) -def getGroupsForDN(dn): +def getGroupsForDN(dn, groupsList=None): """ Get all possible groups for DN :param str dn: user DN + :param list groupsList: group list where need to search :return: S_OK(list)/S_ERROR() -- contain list of groups """ dn = dn.strip() + groups = [] + if not groupsList: + result = gConfig.getSections("%s/Groups" % gBaseRegistrySection) + if not result['OK']: + return result + groupsList = result['Value'] + result = getUsernameForDN(dn) if not result['OK']: return result - return getGroupsForUser(result['Value']) + user = result['Value'] + + # Get VOMS information cache + result = getVOMSInfo(dn=dn) + if not result['OK']: + return result + vomsData = result['Value'] + + result = getVOsWithVOMS() + if not result['OK']: + return result + vomsVOs = result['Value'] + + for group in groupsList: + if user in getGroupOption(group, 'Users', []): + vo = getGroupOption(group, 'VO') + # Is VOMS VO? + if vo in vomsVOs and vomsData.get(vo) and vomsData[vo]['OK'] and vomsData[vo]['Value']: + voData = vomsData[vo]['Value'] + role = getGroupOption(group, 'VOMSRole') + if not role or role in voData[dn]['VOMSRoles']: + groups.append(group) + else: + # If it's not VOMS VO or cannot get information from VOMS + groups.append(group) + + groups.sort() + return S_OK(list(set(groups))) if groups else S_ERROR('No groups found for %s' % dn) def __getGroupsWithAttr(attrName, value): @@ -94,14 +149,27 @@ def __getGroupsWithAttr(attrName, value): return S_OK(groups) if groups else S_ERROR("No groups found for %s=%s" % (attrName, value)) -def getGroupsForUser(username): - """ Find groups for user +def getGroupsForUser(username, groupsList=None): + """ Find groups for user or if set reseachedGroup check it for user :param str username: user name + :param list groupsList: groups - :return: S_OK(list)/S_ERROR() -- contain list of groups + :return: S_OK(list or bool)/S_ERROR() -- contain list of groups or status group for user """ - return __getGroupsWithAttr('Users', username) + if not groupsList: + retVal = gConfig.getSections("%s/Groups" % gBaseRegistrySection) + if not retVal['OK']: + return retVal + groupsList = retVal['Value'] + + groups = [] + for group in groupsList: + if username in getGroupOption(group, 'Users', []): + groups.append(group) + + groups.sort() + return S_OK(list(set(groups))) if groups else S_ERROR('No groups found for %s user' % username) def getGroupsForVO(vo): @@ -203,7 +271,7 @@ def getAllGroups(): return result['Value'] if result['OK'] else [] -def getUsersInGroup(groupName, defaultValue=None): +def getUsersInGroup(group, defaultValue=None): """ Find all users for group :param str group: group name @@ -211,8 +279,9 @@ def getUsersInGroup(groupName, defaultValue=None): :return: list """ - option = "%s/Groups/%s/Users" % (gBaseRegistrySection, groupName) - return gConfig.getValue(option, [] if defaultValue is None else defaultValue) + users = getGroupOption(group, 'Users', []) + users.sort() + return list(set(users)) or [] if defaultValue is None else defaultValue def getUsersInVO(vo, defaultValue=None): @@ -223,45 +292,57 @@ def getUsersInVO(vo, defaultValue=None): :return: list """ + users = [] result = getGroupsForVO(vo) - if not result['OK'] or not result['Value']: - return [] if defaultValue is None else defaultValue - groups = result['Value'] + if result['OK'] and result['Value']: + for group in result['Value']: + users += getUsersInGroup(group) - userList = [] - for group in groups: - userList += getUsersInGroup(group) - return userList + users.sort() + return list(set(users)) or [] if defaultValue is None else defaultValue -def getDNsInVO(vo): - """ Get all DNs that have a VO users +def getDNsInGroup(group, checkStatus=False): + """ Find user DNs for DIRAC group - :param str vo: VO name + :param str group: group name + :param bool checkStatus: don't add suspended DNs :return: list """ - DNs = [] - for user in getUsersInVO(vo): - result = getDNForUsername(user) - if result['OK']: - DNs.extend(result['Value']) - return DNs - - -def getDNsInGroup(groupName): - """ Find all DNs for DIRAC group + vomsData = {} + vo = getGroupOption(group, 'VO') - :param str groupName: group name + # Get VOMS information for VO, if it's VOMS VO + result = getVOsWithVOMS(vo) + if not result['OK']: + return result + if result['Value']: + result = getVOMSInfo(vo=vo) + if not result['OK']: + return result + vomsData = result['Value'] - :return: list - """ DNs = [] - for user in getUsersInGroup(groupName): - result = getDNForUsername(user) - if result['OK']: - DNs.extend(result['Value']) - return DNs + for username in getGroupOption(group, 'Users', []): + if checkStatus and vo in getUserOption(username, 'Suspended', []): + continue + result = getDNsForUsername(username) + if not result['OK']: + return result + userDNs = result['Value'] + if vomsData.get(vo) and vomsData[vo]['OK']: + voData = vomsData[vo]['Value'] + role = getGroupOption(group, 'VOMSRole') + for dn in userDNs: + if dn in voData: + if not checkStatus or not voData[dn]['Suspended']: + if not role or role in voData[dn]['ActuelRoles' if checkStatus else 'VOMSRoles']: + DNs.append(dn) + else: + DNs += userDNs + + return list(set(DNs)) def getPropertiesForGroup(groupName, defaultValue=None): @@ -298,8 +379,6 @@ def getPropertiesForEntity(group, name="", dn="", defaultValue=None): :return: defaultValue or list """ - if defaultValue is None: - defaultValue = [] if group == 'hosts': if not name: result = getHostnameForDN(dn) @@ -466,15 +545,16 @@ def getVOMSVOForGroup(group): return vomsVO -def getGroupsWithVOMSAttribute(vomsAttr): +def getGroupsWithVOMSAttribute(vomsAttr, groupsList=None): """ Search groups with VOMS attribute :param str vomsAttr: VOMS attribute + :param list groupsList: groups where need to search :return: list """ groups = [] - for group in gConfig.getSections("%s/Groups" % (gBaseRegistrySection)).get('Value', []): + for group in groupsList or getAllGroups(): if vomsAttr == gConfig.getValue("%s/Groups/%s/VOMSRole" % (gBaseRegistrySection, group), ""): groups.append(group) return groups @@ -569,10 +649,10 @@ def getVOMSRoleGroupMapping(vo=''): def getUsernameForID(ID, usersList=None): """ Get DIRAC user name by ID - :param basestring ID: user ID + :param str ID: user ID :param list usersList: list of DIRAC user names - :return: S_OK(basestring)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ if not usersList: result = gConfig.getSections("%s/Users" % gBaseRegistrySection) @@ -585,113 +665,175 @@ def getUsernameForID(ID, usersList=None): return S_ERROR("No username found for ID %s" % ID) -def getCAForUsername(username): - """ Get CA option by user name +def getDNProperty(dn, prop, defaultValue=None, username=None): + """ Get user DN property - :param str username: user name + :param str dn: user DN + :param str prop: property name + :param defaultValue: default value + :param str username: username - :return: S_OK(str)/S_ERROR() + :return: S_OK()/S_ERROR() """ - dnList = gConfig.getValue("%s/Users/%s/CA" % (gBaseRegistrySection, username), []) - return S_OK(dnList) if dnList else S_ERROR("No CA found for user %s" % username) + if not username: + result = getUsernameForDN(dn) + if not result['OK']: + return result + username = result['Value'] + + root = "%s/Users/%s/DNProperties" % (gBaseRegistrySection, username) + result = gConfig.getSections(root) + if not result['OK']: + return result + for section in result['Value']: + if dn == gConfig.getValue("%s/%s/DN" % (root, section)): + return S_OK(gConfig.getValue("%s/%s/%s" % (root, section, prop), defaultValue)) + return S_OK(defaultValue) + +def isDownloadableGroup(groupName): + """ Get permission to download proxy with group in a argument -def getDNProperty(userDN, value, defaultValue=None): - """ Get property from DNProperties section by user DN + :params str groupName: DIRAC group - :param str userDN: user DN - :param str value: option that need to get - :param defaultValue: default value + :return: boolean + """ + if getGroupOption(groupName, 'DownloadableProxy') in [False, 'False', 'false', 'no']: + return False + return True + + +def getEmailsForGroup(groupName): + """ Get email list of users in group - :return: S_OK()/S_ERROR() -- str or list that contain option value + :param str groupName: DIRAC group name + + :return: list(list) -- inner list contains emails for a user """ - result = getUsernameForDN(userDN) - if not result['OK']: - return result - pathDNProperties = "%s/Users/%s/DNProperties" % (gBaseRegistrySection, result['Value']) - result = gConfig.getSections(pathDNProperties) - if result['OK']: - for section in result['Value']: - if userDN == gConfig.getValue("%s/%s/DN" % (pathDNProperties, section)): - return S_OK(gConfig.getValue("%s/%s/%s" % (pathDNProperties, section, value), defaultValue)) - return S_OK(defaultValue) + emails = [] + for username in getUsersInGroup(groupName): + email = getUserOption(username, 'Email', []) + emails.append(email) + return emails + + +def getIDsForUsername(username): + """ Return IDs for DIRAC user + + :param str username: DIRAC user + + :return: list -- contain IDs + """ + return gConfig.getValue("%s/Users/%s/ID" % (gBaseRegistrySection, username), []) -def getProxyProvidersForDN(userDN): - """ Get proxy providers by user DN +def getVOsWithVOMS(voList=None): + """ Get all the configured VOMS VOs - :param str userDN: user DN + :param list voList: VOs where to look :return: S_OK(list)/S_ERROR() """ - return getDNProperty(userDN, 'ProxyProviders', []) + vos = [] + if not voList: + result = getVOs() + if result['OK']: + # Hack to run integration tests where not exist VO section. + #return result + + voList = result['Value'] + for vo in voList or []: + if getVOOption(vo, 'VOMSName'): + vos.append(vo) + return S_OK(vos) -def getDNFromProxyProviderForUserID(proxyProvider, userID): - """ Get groups by user DN in DNProperties +def getDNsForUsername(username): + """ Find all DNs for DIRAC user - :param str proxyProvider: proxy provider name - :param str userID: user identificator + :param str username: DIRAC user - :return: S_OK(str)/S_ERROR() + :return: S_OK(list)/S_ERROR() -- contain DNs """ - # Get user name - result = getUsernameForID(userID) - if not result['OK']: - return result - # Get DNs from user - result = getDNForUsername(result['Value']) - if not result['OK']: - return result - for DN in result['Value']: - result = getProxyProvidersForDN(DN) - if not result['OK']: - return result - if proxyProvider in result['Value']: - return S_OK(DN) - return S_ERROR(errno.ENODATA, - "No DN found for %s proxy provider for user ID %s" % (proxyProvider, userID)) + userDNs = getDNsForUsernameFromSC(username) + for uid in getIDsForUsername(username): + result = gAuthManagerData.getDNsForID(uid) + if result['OK']: + userDNs += result['Value'] + return S_OK(list(set(userDNs))) -def isDownloadableGroup(groupName): - """ Get permission to download proxy with group in a argument +def getDNForUsernameInGroup(username, group, checkStatus=False): + """ Get user DN for user in group - :params str groupName: DIRAC group + :param str username: user name + :param str group: group name + :param bool checkStatus: don't add suspended DNs - :return: boolean + :return: S_OK(str)/S_ERROR() """ - if getGroupOption(groupName, 'DownloadableProxy') in [False, 'False', 'false', 'no']: - return False - return True + result = getDNsForUsernameInGroup(username, group, checkStatus) + return S_OK(result['Value'][0]) if result['OK'] else result -def getUserDict(username): - """ Get full information from user section +def getDNsForUsernameInGroup(username, group, checkStatus=False): + """ Get user DN for user in group - :param str username: DIRAC user name + :param str username: user name + :param str group: group name + :param bool checkStatus: don't add suspended DNs - :return: S_OK()/S_ERROR() + :return: S_OK(str)/S_ERROR() """ - resDict = {} - relPath = '%s/Users/%s/' % (gBaseRegistrySection, username) - result = gConfig.getConfigurationTree(relPath) + if username not in getGroupOption(group, 'Users', []): + return S_ERROR('%s group not have %s user.' % (group, username)) + + DNs = [] + result = getDNsForUsername(username) if not result['OK']: return result - for key, value in result['Value'].items(): - if value: - resDict[key.replace(relPath, '')] = value - return S_OK(resDict) + userDNs = result['Value'] + vo = getGroupOption(group, 'VO') + if checkStatus and vo in getUserOption(username, 'Suspended', []): + return S_ERROR('%s marked as suspended for %s VO.' % (username, vo)) -def getEmailsForGroup(groupName): - """ Get email list of users in group + result = getVOsWithVOMS(vo) + if not result['OK']: + return result + if result['Value']: + result = getVOMSInfo(vo=vo) + if not result['OK']: + return result + vomsData = result['Value'] + if vomsData.get(vo) and vomsData[vo]['OK']: + voData = vomsData[vo]['Value'] + role = getGroupOption(group, 'VOMSRole') + for dn in userDNs: + if dn in voData: + if not checkStatus or not voData[dn]['Suspended']: + if not role or role in voData[dn]['ActuelRoles' if checkStatus else 'VOMSRoles']: + DNs.append(dn) + else: + DNs += userDNs + else: + DNs += userDNs + + if DNs: + return S_OK(list(set(DNs))) + return S_ERROR('For %s@%s not found DN%s.' % (username, group, ' or it suspended' if checkStatus else '')) + + +def findSomeDNToUseForGroupsThatNotNeedDN(username): + """ This method is HACK for groups that not need DN from user, like as dirac_user, dirac_admin + In this cause we will search first DN in CS or any DN that we can to find - :param str groupName: DIRAC group name + :param str username: user name - :return: list(list) -- inner list contains emails for a user + :return: S_OK(str)/S_ERROR() """ - emails = [] - for username in getUsersInGroup(groupName): - email = getUserOption(username, 'Email', []) - emails.append(email) - return emails + defDNs = getDNsForUsernameFromSC(username) + if not defDNs: + result = getDNsForUsername(username) + return S_OK(result['Value'][0]) if result['OK'] else result + return S_OK(defDNs[0]) From ab8ed4d897f1a7fedec42defb7736a0d0b07d36c Mon Sep 17 00:00:00 2001 From: TaykYoku Date: Tue, 30 Jun 2020 16:47:09 +0200 Subject: [PATCH 002/105] CS: modify methods in Resources and Utilities --- .../Client/Helpers/Resources.py | 92 ++++++++++++------- ConfigurationSystem/Client/Utilities.py | 6 +- 2 files changed, 62 insertions(+), 36 deletions(-) diff --git a/ConfigurationSystem/Client/Helpers/Resources.py b/ConfigurationSystem/Client/Helpers/Resources.py index 28d88a719de..788d93e42ad 100644 --- a/ConfigurationSystem/Client/Helpers/Resources.py +++ b/ConfigurationSystem/Client/Helpers/Resources.py @@ -465,42 +465,68 @@ def getFilterConfig(filterID): return gConfig.getOptionsDict('Resources/LogFilters/%s' % filterID) -def getInfoAboutProviders(of=None, providerName=None, option='', section=''): - """ Get the information about providers - - :param basestring of: provider of what(Id, Proxy or etc.) need to look, - None, "all" to get list of instance of what this providers - :param basestring providerName: provider name, - None, "all" to get list of providers names - :param basestring option: option name that need to get, - None, "all" to get all options in a section - :param basestring section: section path in root section of provider, - "all" to get options in all sections - - :return: S_OK()/S_ERROR() +def getProvidersForInstance(instance, providerType=None): + """ Get providers for instance + + :param str instance: instance of what this providers + :param str providerType: provider type + + :return: S_OK(list)/S_ERROR() + """ + result = gConfig.getSections('%s/%sProviders' % (gBaseResourcesSection, instance)) + if not result['OK'] or not result['Value'] or (result['OK'] and not providerType): + return result + data = [] + for prov in result['Value']: + if providerType == gConfig.getValue('%s/%sProviders/%s/ProviderType' % + (gBaseResourcesSection, instance, prov)): + data.append(prov) + return S_OK(data) + + +def getProviderByAlias(alias, instance=None): + """ Find provider name by alias + + :param str alias: other registered provider name + :param str instance: provider of what + + :return: S_OK(str)/S_ERROR() """ - if not of or of == "all": + instances = [instance] or [] + if not instances: result = gConfig.getSections(gBaseResourcesSection) if not result['OK']: return result - return S_OK([i.replace('Providers', '') for i in result['Value']]) - if not providerName or providerName == "all": - return gConfig.getSections('%s/%sProviders' % (gBaseResourcesSection, of)) - if not option or option == 'all': - if not section: - return gConfig.getOptionsDict("%s/%sProviders/%s" % (gBaseResourcesSection, of, providerName)) - elif section == "all": - resDict = {} - relPath = "%s/%sProviders/%s/" % (gBaseResourcesSection, of, providerName) - result = gConfig.getConfigurationTree(relPath) + for section in result['Value']: + if section[-9:] == 'Providers': + instances.append(section[:-9]) + for instance in instances: + result = getProvidersForInstance(instance) + if not result['OK']: + return result + for provider in result['Value']: + if alias in gConfig.getValue("%s/%sProviders/%s/Aliases" % (gBaseResourcesSection, + instance, provider), []): + return S_OK(provider) + return S_ERROR('No found any provider for %s' % alias) + + +def getProviderInfo(provider): + """ Get provider info + + :param str provider: provider + + :return: S_OK(dict)/S_ERROR() + """ + result = gConfig.getSections(gBaseResourcesSection) + if not result['OK']: + return result + for section in result['Value']: + if section[-9:] == 'Providers': + result = getProvidersForInstance(section[:-9]) if not result['OK']: return result - for key, value in result['Value'].items(): # can be an iterator - if value: - resDict[key.replace(relPath, '')] = value - return S_OK(resDict) - else: - return gConfig.getSections('%s/%sProviders/%s/%s/' % (gBaseResourcesSection, of, providerName, section)) - else: - return S_OK(gConfig.getValue('%s/%sProviders/%s/%s/%s' % (gBaseResourcesSection, of, providerName, - section, option))) + if provider in result['Value']: + return gConfig.getOptionsDictRecursively("%s/%s/%s/" % (gBaseResourcesSection, + section, provider)) + return S_ERROR('%s provider not found.' % provider) diff --git a/ConfigurationSystem/Client/Utilities.py b/ConfigurationSystem/Client/Utilities.py index a8f788cea1a..2f7f8b98c64 100644 --- a/ConfigurationSystem/Client/Utilities.py +++ b/ConfigurationSystem/Client/Utilities.py @@ -701,14 +701,14 @@ def getElasticDBParameters(fullname): return S_OK(parameters) -def getOAuthAPI(instance='Production'): - """ Get OAuth API url +def getAuthAPI(instance='Production'): + """ Get Auth REST API url :param str instance: instance :return: str """ - return gConfig.getValue("/Systems/Framework/%s/URLs/OAuthAPI" % instance) + return gConfig.getValue("/Systems/Framework/%s/URLs/AuthAPI" % instance) def getDIRACGOCDictionary(): From 3e37bdf5f968fd2cd6b9cea4253eb37948db3450 Mon Sep 17 00:00:00 2001 From: TaykYoku Date: Tue, 30 Jun 2020 16:47:29 +0200 Subject: [PATCH 003/105] CS: fix test --- ConfigurationSystem/test/Test_agentOptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ConfigurationSystem/test/Test_agentOptions.py b/ConfigurationSystem/test/Test_agentOptions.py index 4b3e8c2ebf3..5706348df8a 100644 --- a/ConfigurationSystem/test/Test_agentOptions.py +++ b/ConfigurationSystem/test/Test_agentOptions.py @@ -75,7 +75,7 @@ ('DIRAC.WorkloadManagementSystem.Agent.StatesAccountingAgent', {}), ('DIRAC.WorkloadManagementSystem.Agent.StatesMonitoringAgent', {}), ('DIRAC.WorkloadManagementSystem.Agent.SiteDirector', - {'SpecialMocks': {'findGenericPilotCredentials': S_OK(('a', 'b'))}}), + {'SpecialMocks': {'findGenericPilotCredentials': S_OK(('a', 'b', 'c'))}}), # ('DIRAC.WorkloadManagementSystem.Agent.MultiProcessorSiteDirector', {}), # not inheriting from AgentModule ] From 16a75e292b1d3ba8696f09054959e65c638da41e Mon Sep 17 00:00:00 2001 From: TaykYoku Date: Tue, 30 Jun 2020 16:47:49 +0200 Subject: [PATCH 004/105] CS: add rest api --- .../Utilities/ConfigurationHandler.py | 92 +++++++++++++++++++ ConfigurationSystem/Utilities/__init__.py | 5 + 2 files changed, 97 insertions(+) create mode 100644 ConfigurationSystem/Utilities/ConfigurationHandler.py create mode 100755 ConfigurationSystem/Utilities/__init__.py diff --git a/ConfigurationSystem/Utilities/ConfigurationHandler.py b/ConfigurationSystem/Utilities/ConfigurationHandler.py new file mode 100644 index 00000000000..2b8a91f3dbc --- /dev/null +++ b/ConfigurationSystem/Utilities/ConfigurationHandler.py @@ -0,0 +1,92 @@ +""" HTTP API of the DIRAC configuration data, rewrite from RESTDIRAC project +""" +import re +import json + +from tornado import web, gen +from tornado.template import Template + +from DIRAC import S_OK, S_ERROR, gConfig, gLogger +from DIRAC.ConfigurationSystem.Client.Helpers import Resources, Registry +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager + +from DIRAC.Core.Web.WebHandler import WebHandler, asyncGen, WErr + +__RCSID__ = "$Id$" + + +class ConfigurationHandler(WebHandler): + OVERPATH = True + AUTH_PROPS = "all" + LOCATION = "/" + + def initialize(self): + super(ConfigurationHandler, self).initialize() + self.args = {} + for arg in self.request.arguments: + if len(self.request.arguments[arg]) > 1: + self.args[arg] = self.request.arguments[arg] + else: + self.args[arg] = self.request.arguments[arg][0] or '' + return S_OK() + + @asyncGen + def web_conf(self): + """ Configuration endpoint, used to: + GET /conf/get? -- get configuration information, with arguments + * options: + * fullCFG - to get dump of configuration + * option - option path to get option value + * options - section path to get list of options + * section - section path to get dict of all options/values there + * sections - section path to get list of sections there + * version - version of configuration that request information(optional) + + GET /conf/? -- get some information by using helpers methods + * helper method - helper method of configuration service + * arguments - arguments specifecly for every helper method + + :return: json with requested data + """ + self.log.notice('Request configuration information') + optns = self.overpath.strip('/').split('/') + if not optns or len(optns) > 1: + raise WErr(404, "Wrone way") + + result = S_ERROR('%s request unsuported' % optns[0]) + if optns[0] == 'get': + if 'version' in self.args and (self.args.get('version') or '0') >= gConfigurationData.getVersion(): + self.finish() + + result = {} + if 'fullCFG' in self.args: + remoteCFG = yield self.threadTask(gConfigurationData.getRemoteCFG) + result['Value'] = str(remoteCFG) + elif 'option' in self.args: + result = yield self.threadTask(gConfig.getOption, self.args['option']) + elif 'section' in self.args: + result = yield self.threadTask(gConfig.getOptionsDict, self.args['section']) + elif 'options' in self.args: + result = yield self.threadTask(gConfig.getOptions, self.args['options']) + elif 'sections' in self.args: + result = yield self.threadTask(gConfig.getSections, self.args['sections']) + else: + raise WErr(500, 'Invalid argument') + elif optns[0] == 'getGroupsStatusByUsername': + result = yield self.threadTask(gProxyManager.getGroupsStatusByUsername, **self.args) + elif any([optns[0] == m and re.match('^[a-z][A-z]+', m) for m in dir(Registry)]) and self.isRegisteredUser(): + result = yield self.threadTask(getattr(Registry, optns[0]), **self.args) + else: + raise WErr(500, '%s request unsuported' % optns[0]) + # result = yield self.threadTask(getattr(Registry, optns[0]), **self.args) + + if not result['OK']: + raise WErr(404, result['Message']) + self.finishJEncode(result['Value']) + + @asyncGen + def post(self): + """ Post method + """ + pass diff --git a/ConfigurationSystem/Utilities/__init__.py b/ConfigurationSystem/Utilities/__init__.py new file mode 100755 index 00000000000..2492dd6ec85 --- /dev/null +++ b/ConfigurationSystem/Utilities/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +# $HeadURL$ +__RCSID__ = "$Id$" From f3a5fa1215d7bf95a10c1196954d2030fc7f2ef6 Mon Sep 17 00:00:00 2001 From: TaykYoku Date: Tue, 30 Jun 2020 16:54:56 +0200 Subject: [PATCH 005/105] Core: align with new Registry --- Core/Base/API.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Core/Base/API.py b/Core/Base/API.py index 7d86ebe5d28..37adb98602d 100644 --- a/Core/Base/API.py +++ b/Core/Base/API.py @@ -9,7 +9,7 @@ from DIRAC import gLogger, gConfig, S_OK, S_ERROR from DIRAC.Core.Security.ProxyInfo import getProxyInfo, formatProxyInfoAsString -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNsForUsername from DIRAC.Core.Utilities.Version import getCurrentVersion __RCSID__ = '$Id$' @@ -141,9 +141,11 @@ def _getCurrentUser(self): gLogger.debug(formatProxyInfoAsString(proxyInfo)) if 'group' not in proxyInfo: return self._errorReport('Proxy information does not contain the group', res['Message']) - res = getDNForUsername(proxyInfo['username']) - if not res['OK']: - return self._errorReport('Failed to get proxies for user', res['Message']) + result = getDNsForUsername(proxyInfo['username']) + if not result['OK']: + return self._errorReport('Failed to get proxies for user', result['Message']) + if not result['Value']: + return self._errorReport('Failed to get proxies for user', "No DNs found for %s" % proxyInfo['username']) return S_OK(proxyInfo['username']) ############################################################################# From 1b652821f50131f34253d106156fad0c125a97e2 Mon Sep 17 00:00:00 2001 From: TaykYoku Date: Tue, 30 Jun 2020 16:57:04 +0200 Subject: [PATCH 006/105] Core: add DB version --- Core/Base/DB.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/Core/Base/DB.py b/Core/Base/DB.py index 363bd493826..b196b5e04aa 100755 --- a/Core/Base/DB.py +++ b/Core/Base/DB.py @@ -5,7 +5,7 @@ from __future__ import division from __future__ import print_function -from DIRAC import gLogger, gConfig +from DIRAC import gLogger, gConfig, S_OK, S_ERROR from DIRAC.Core.Utilities.MySQL import MySQL from DIRAC.ConfigurationSystem.Client.Utilities import getDBParameters from DIRAC.ConfigurationSystem.Client.PathFinder import getDatabaseSection @@ -18,9 +18,16 @@ class DB(MySQL): """ def __init__(self, dbname, fullname, debug=False): + """ C'or + :param str dbname: database name + :param str fullname: full name + :param bool debug: debug mode + """ + self.versionDB = 0 self.fullname = fullname database_name = dbname + self.versionTable = '%s_Version' % database_name self.log = gLogger.getSubLogger(database_name) result = getDBParameters(fullname) @@ -44,6 +51,23 @@ def __init__(self, dbname, fullname, debug=False): if not self._connected: raise RuntimeError("Can not connect to DB '%s', exiting..." % self.dbName) + # Initialize version + result = self._query("show tables") + if result['OK']: + if self.versionTable not in [t[0] for t in result['Value']]: + result = self._createTables({self.versionTable: {'Fields': {'Version': 'INTEGER NOT NULL'}, + 'PrimaryKey': 'Version'}}) + if not result['OK']: + raise RuntimeError("Can not initialize %s version: %s" % (self.dbName, result['Message'])) + result = self._query("SELECT Version FROM `%s`" % self.versionTable) + if result['OK']: + if len(result['Value']) > 0: + self.versionDB = result['Value'][0][0] + else: + result = self._update("INSERT INTO `%s` (Version) VALUES (%s)" % (self.versionTable, self.versionDB)) + if not result['OK']: + raise RuntimeError("Can not initialize %s version: %s" % (self.dbName, result['Message'])) + self.log.info("===================== MySQL ======================") self.log.info("User: " + self.dbUser) self.log.info("Host: " + self.dbHost) @@ -54,5 +78,27 @@ def __init__(self, dbname, fullname, debug=False): ############################################################################# def getCSOption(self, optionName, defaultValue=None): + """ Get option from CS + + :param str optionName: option name + :param defaultValue: default value + + :return: value that inherits the defaultValue type + """ cs_path = getDatabaseSection(self.fullname) return gConfig.getValue("/%s/%s" % (cs_path, optionName), defaultValue) + + def updateDBVersion(self, version): + """ Update DB version + + :param int version: version number + + :return: S_OK()/S_ERROR() + """ + result = self._query('DELETE FROM `%s_Version`' % self.dbName) + if result['OK']: + result = self._update("INSERT INTO `%s_Version` (Version) VALUES (%s)" % (self.dbName, version)) + if not result['OK']: + return S_ERROR("Can not initialize %s version: %s" % (self.dbName, result['Message'])) + self.versionDB = version + return S_OK() From 08c69390f87bf9ddc0a10dcb7560b8b3f6d922b6 Mon Sep 17 00:00:00 2001 From: TaykYoku Date: Tue, 30 Jun 2020 17:01:26 +0200 Subject: [PATCH 007/105] Core: allow to delegate through delegateID in BaseClient --- Core/DISET/private/BaseClient.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Core/DISET/private/BaseClient.py b/Core/DISET/private/BaseClient.py index 9eeae56ddfc..e095c54e667 100755 --- a/Core/DISET/private/BaseClient.py +++ b/Core/DISET/private/BaseClient.py @@ -17,7 +17,6 @@ from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL, getServiceFailoverURL -from DIRAC.ConfigurationSystem.Client.Helpers import Registry from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck from DIRAC.Core.DISET.private.TransportPool import getGlobalTransportPool from DIRAC.Core.DISET.ThreadConfig import ThreadConfig @@ -35,6 +34,7 @@ class BaseClient(object): KW_TIMEOUT = "timeout" KW_SETUP = "setup" KW_VO = "VO" + KW_DELEGATED_ID = "delegatedID" KW_DELEGATED_DN = "delegatedDN" KW_DELEGATED_GROUP = "delegatedGroup" KW_IGNORE_GATEWAYS = "ignoreGateways" @@ -256,17 +256,19 @@ def __discoverExtraCredentials(self): self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS] # Are we delegating something? + delegatedID = self.kwargs.get(self.KW_DELEGATED_ID) or self.__threadConfig.getID() delegatedDN = self.kwargs.get(self.KW_DELEGATED_DN) or self.__threadConfig.getDN() delegatedGroup = self.kwargs.get(self.KW_DELEGATED_GROUP) or self.__threadConfig.getGroup() + + if delegatedID: + self.kwargs[self.KW_DELEGATED_ID] = delegatedID if delegatedDN: self.kwargs[self.KW_DELEGATED_DN] = delegatedDN - if not delegatedGroup: - result = Registry.findDefaultGroupForDN(delegatedDN) - if not result['OK']: - return result - delegatedGroup = result['Value'] + if delegatedGroup: self.kwargs[self.KW_DELEGATED_GROUP] = delegatedGroup - self.__extraCredentials = (delegatedDN, delegatedGroup) + + if delegatedID or delegatedDN: + self.__extraCredentials = (delegatedID or delegatedDN, delegatedGroup) return S_OK() def __findServiceURL(self): From 5f75ca576719136fc9c99fa9a0152d8227140a69 Mon Sep 17 00:00:00 2001 From: TaykYoku Date: Tue, 30 Jun 2020 17:04:07 +0200 Subject: [PATCH 008/105] Core: modify AuthManager to use IDs, align with new Registry, fix test --- Core/DISET/AuthManager.py | 468 ++++++++++++++-------------- Core/DISET/test/Test_AuthManager.py | 47 ++- 2 files changed, 282 insertions(+), 233 deletions(-) diff --git a/Core/DISET/AuthManager.py b/Core/DISET/AuthManager.py index 847043cc987..fb1b2e268df 100755 --- a/Core/DISET/AuthManager.py +++ b/Core/DISET/AuthManager.py @@ -5,180 +5,261 @@ from __future__ import print_function import six + +from DIRAC.Core.Utilities import List +from DIRAC.Core.Security import Properties +from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.ConfigurationSystem.Client.Helpers import Registry -from DIRAC.FrameworkSystem.Client.Logger import gLogger -from DIRAC.Core.Security import Properties -from DIRAC.Core.Utilities import List __RCSID__ = "$Id$" +KW_ID = 'ID' +KW_DN = 'DN' +KW_GROUP = 'group' +KW_USERNAME = 'username' +KW_HOSTS_GROUP = 'hosts' +KW_PROPERTIES = 'properties' +KW_EXTRA_CREDENTIALS = 'extraCredentials' + + +def forwardingCredentials(credDict, logObj=gLogger): + """ Check whether the credentials are being forwarded by a valid source and extract it + + :param dict credDict: Credentials to ckeck + :param object logObj: logger + + :return: bool + """ + if isinstance(credDict.get(KW_EXTRA_CREDENTIALS), (tuple, list)): + retVal = Registry.getHostnameForDN(credDict.get(KW_DN)) + if not retVal['OK']: + logObj.debug("The credentials forwarded not by a host:", credDict.get(KW_DN)) + return False + hostname = retVal['Value'] + if Properties.TRUSTED_HOST not in Registry.getPropertiesForHost(hostname, []): + logObj.debug("The credentials forwarded by a %s host, but it is not a trusted one" % hostname) + return False + if credDict[KW_EXTRA_CREDENTIALS][0][0] == '/': + credDict[KW_DN] = credDict[KW_EXTRA_CREDENTIALS][0] + credDict[KW_ID] = None + else: + credDict[KW_ID] = credDict[KW_EXTRA_CREDENTIALS][0] + credDict[KW_DN] = None + credDict[KW_GROUP] = credDict[KW_EXTRA_CREDENTIALS][1] + del credDict[KW_EXTRA_CREDENTIALS] + return True + return False + + +def initializationOfSession(credDict, logObj=gLogger): + """ Discover the username associated to the authentication session. It will check if the selected group is valid. + The username will be included in the credentials dictionary. And will discover DN for group if last not set. + + :param dict credDict: Credentials to check + :param object logObj: logger + + :return: bool -- specifying whether the username was found + """ + # Find user + result = Registry.getUsernameForID(credDict[KW_ID]) + if not result['OK']: + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + credDict[KW_USERNAME] = result['Value'] + return True + + +def initializationOfCertificate(credDict, logObj=gLogger): + """ Discover the username associated to the certificate DN. It will check if the selected group is valid. + The username will be included in the credentials dictionary. + + :param dict credDict: Credentials to check + :param object logObj: logger + + :return: bool -- specifying whether the username was found + """ + logObj.debug("initializationOfCertificate") + # Search host + result = Registry.getHostnameForDN(credDict[KW_DN]) + if result['OK'] and result['Value']: + credDict[KW_USERNAME] = result['Value'] + credDict[KW_GROUP] = KW_HOSTS_GROUP + return True + elif credDict.get(KW_GROUP) == KW_HOSTS_GROUP: + logObj.warn("Cannot find hostname for DN %s: %s" % (credDict[KW_DN], result['Message'])) + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + + # Search user + result = Registry.getUsernameForDN(credDict[KW_DN]) + if not result['OK']: + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + credDict[KW_USERNAME] = result['Value'] + return True + + +def initializationOfGroup(credDict, logObj=gLogger): + """ Check/get default group + + :param dict credDict: Credentials to check + :param object logObj: logger + + :return: bool -- specifying whether the username was found + """ + logObj.debug("initializationOfGroup", credDict) + # Find/check group + credDict[KW_PROPERTIES] = [] + if not credDict.get(KW_GROUP) or credDict[KW_GROUP] == 'visitor': + result = Registry.findDefaultGroupForUser(credDict[KW_USERNAME]) + if not result['OK']: + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + credDict[KW_GROUP] = result['Value'] + logObj.debug("initializationOfGroup 1") + if credDict[KW_GROUP] == KW_HOSTS_GROUP: + logObj.debug("initializationOfGroup 12") + credDict[KW_PROPERTIES] = Registry.getPropertiesForHost(credDict[KW_USERNAME], []) + return True + logObj.debug("initializationOfGroup 2") + if not Registry.getGroupsForUser(credDict[KW_USERNAME], groupsList=[credDict[KW_GROUP]]).get('Value'): + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + + # Get DN for user/group + result = Registry.getDNsForUsernameInGroup(credDict[KW_USERNAME], credDict[KW_GROUP], checkStatus=True) + if not result['OK']: + logObj.error(result['Message']) + credDict[KW_GROUP] = "visitor" + return False + + # Set DN if authorization not througth certificate + if not credDict.get(KW_DN): + credDict[KW_DN] = result['Value'][0] + + # Check if DN match for group + if credDict[KW_DN] not in result["Value"]: + logObj.error('%s DN is not match for %s group.' % (credDict[KW_DN], credDict[KW_GROUP])) + credDict[KW_GROUP] = "visitor" + return False + + # Fill group properties + credDict[KW_PROPERTIES] = Registry.getPropertiesForGroup(credDict[KW_GROUP], []) + return True + class AuthManager(object): """ Handle Service Authorization """ - __authLogger = gLogger.getSubLogger("Authorization") - KW_HOSTS_GROUP = 'hosts' - KW_DN = 'DN' - KW_GROUP = 'group' - KW_EXTRA_CREDENTIALS = 'extraCredentials' - KW_PROPERTIES = 'properties' - KW_USERNAME = 'username' def __init__(self, authSection): - """ - Constructor + """ Constructor - :type authSection: string - :param authSection: Section containing the authorization rules + :param str authSection: Section containing the authorization rules """ self.authSection = authSection def authQuery(self, methodQuery, credDict, defaultProperties=False): - """ - Check if the query is authorized for a credentials dictionary - - :type methodQuery: string - :param methodQuery: Method to test - :type credDict: dictionary - :param credDict: dictionary containing credentials for test. The dictionary can contain the DN - and selected group. - :return: Boolean result of test + """ Check if the query is authorized for a credentials dictionary + + :param str methodQuery: Method to test + :param dict credDict: dictionary containing credentials for test. The dictionary can contain the DN + and selected group. + :param defaultProperties: default properties + :type defaultProperties: list or tuple + + :return: bool -- result of test """ userString = "" - if self.KW_DN in credDict: - userString += "DN=%s" % credDict[self.KW_DN] - if self.KW_GROUP in credDict: - userString += " group=%s" % credDict[self.KW_GROUP] - if self.KW_EXTRA_CREDENTIALS in credDict: - userString += " extraCredentials=%s" % str(credDict[self.KW_EXTRA_CREDENTIALS]) + if KW_ID in credDict: + userString += " ID=%s" % credDict[KW_ID] + if KW_DN in credDict: + userString += " DN=%s" % credDict[KW_DN] + if credDict.get(KW_GROUP): + userString += " group=%s" % credDict[KW_GROUP] + if KW_EXTRA_CREDENTIALS in credDict: + userString += " extraCredentials=%s" % str(credDict[KW_EXTRA_CREDENTIALS]) self.__authLogger.debug("Trying to authenticate %s" % userString) + # Get properties requiredProperties = self.getValidPropertiesForMethod(methodQuery, defaultProperties) + lowerCaseProperties = [prop.lower() for prop in requiredProperties] or ['any'] + allowAll = "any" in lowerCaseProperties or "all" in lowerCaseProperties + # Extract valid groups validGroups = self.getValidGroups(requiredProperties) - lowerCaseProperties = [prop.lower() for prop in requiredProperties] - if not lowerCaseProperties: - lowerCaseProperties = ['any'] + self.__authLogger.debug("validGroups: ", validGroups) - allowAll = "any" in lowerCaseProperties or "all" in lowerCaseProperties - # Set no properties by default - credDict[self.KW_PROPERTIES] = [] - # Check non secure backends - if self.KW_DN not in credDict or not credDict[self.KW_DN]: - if allowAll and not validGroups: - self.__authLogger.debug("Accepted request from unsecure transport") - return True + # Read extra credentials + if KW_EXTRA_CREDENTIALS in credDict: + # Is it a host? and HACK TO MAINTAIN COMPATIBILITY + if credDict.get(KW_EXTRA_CREDENTIALS) == KW_HOSTS_GROUP: + credDict[KW_GROUP] = credDict[KW_EXTRA_CREDENTIALS] + del credDict[KW_EXTRA_CREDENTIALS] + # Check if query comes though a gateway/web server + elif forwardingCredentials(credDict, logObj=self.__authLogger): + self.__authLogger.debug("Query comes from a gateway") + return self.authQuery(methodQuery, credDict, requiredProperties) else: - self.__authLogger.debug( - "Explicit property required and query seems to be coming through an unsecure transport") - return False - # Check if query comes though a gateway/web server - if self.forwardedCredentials(credDict): - self.__authLogger.debug("Query comes from a gateway") - self.unpackForwardedCredentials(credDict) - return self.authQuery(methodQuery, credDict, requiredProperties) - # Get the properties - # Check for invalid forwarding - if self.KW_EXTRA_CREDENTIALS in credDict: - # Invalid forwarding? - if not isinstance(credDict[self.KW_EXTRA_CREDENTIALS], six.string_types): - self.__authLogger.debug("The credentials seem to be forwarded by a host, but it is not a trusted one") return False - # Is it a host? - if self.KW_EXTRA_CREDENTIALS in credDict and credDict[self.KW_EXTRA_CREDENTIALS] == self.KW_HOSTS_GROUP: - # Get the nickname of the host - credDict[self.KW_GROUP] = credDict[self.KW_EXTRA_CREDENTIALS] - # HACK TO MAINTAIN COMPATIBILITY - else: - if self.KW_EXTRA_CREDENTIALS in credDict and self.KW_GROUP not in credDict: - credDict[self.KW_GROUP] = credDict[self.KW_EXTRA_CREDENTIALS] - # END OF HACK - # Get the username - if self.KW_DN in credDict and credDict[self.KW_DN]: - if self.KW_GROUP not in credDict: - result = Registry.findDefaultGroupForDN(credDict[self.KW_DN]) - if not result['OK']: - credDict[self.KW_USERNAME] = "anonymous" - credDict[self.KW_GROUP] = "visitor" - else: - credDict[self.KW_GROUP] = result['Value'] - if credDict[self.KW_GROUP] == self.KW_HOSTS_GROUP: - # For host - if not self.getHostNickName(credDict): - self.__authLogger.warn("Host is invalid") - if not allowAll: - return False - # If all, then set anon credentials - credDict[self.KW_USERNAME] = "anonymous" - credDict[self.KW_GROUP] = "visitor" - else: - # For users - username = self.getUsername(credDict) - suspended = self.isUserSuspended(credDict) - if not username: - self.__authLogger.warn("User is invalid or does not belong to the group it's saying") - if suspended: - self.__authLogger.warn("User is Suspended") - - if not username or suspended: - if not allowAll: - return False - # If all, then set anon credentials - credDict[self.KW_USERNAME] = "anonymous" - credDict[self.KW_GROUP] = "visitor" - else: - if not allowAll: - return False - credDict[self.KW_USERNAME] = "anonymous" - credDict[self.KW_GROUP] = "visitor" - # If any or all in the props, allow - allowGroup = not validGroups or credDict[self.KW_GROUP] in validGroups - if allowAll and allowGroup: - return True - # Check authorized groups - if "authenticated" in lowerCaseProperties and allowGroup: - return True - if not self.matchProperties(credDict, requiredProperties): - self.__authLogger.warn("Client is not authorized\nValid properties: %s\nClient: %s" % - (requiredProperties, credDict)) - return False - elif not allowGroup: - self.__authLogger.warn("Client is not authorized\nValid groups: %s\nClient: %s" % - (validGroups, credDict)) - return False - return True + # Check authorization + authorized = True + # Search user name + if not credDict.get(KW_USERNAME): + if credDict.get(KW_DN): + # With certificate + authorized = initializationOfCertificate(credDict, logObj=self.__authLogger) + elif credDict.get(KW_ID): + # With IdP session + authorized = initializationOfSession(credDict, logObj=self.__authLogger) + else: + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + authorized = False - def getHostNickName(self, credDict): - """ - Discover the host nickname associated to the DN. - The nickname will be included in the credentials dictionary. + # Search group + if authorized: + authorized = initializationOfGroup(credDict, logObj=self.__authLogger) - :type credDict: dictionary - :param credDict: Credentials to ckeck - :return: Boolean specifying whether the nickname was found - """ - if self.KW_DN not in credDict: - return True - if self.KW_GROUP not in credDict: - return False - retVal = Registry.getHostnameForDN(credDict[self.KW_DN]) - if not retVal['OK']: - gLogger.warn("Cannot find hostname for DN %s: %s" % (credDict[self.KW_DN], retVal['Message'])) - return False - credDict[self.KW_USERNAME] = retVal['Value'] - credDict[self.KW_PROPERTIES] = Registry.getPropertiesForHost(credDict[self.KW_USERNAME], []) - return True + # Authorize check + if allowAll or authorized: + # Properties check + if not self.matchProperties(credDict, list(set(requiredProperties) - set(['Any', 'any', + 'All', 'all', + 'authenticated', + 'Authenticated']))): + self.__authLogger.warn("Client is not authorized\nValid properties: %s\nClient: %s" % + (requiredProperties, credDict)) + return False + # Groups check + elif validGroups and credDict[KW_GROUP] not in validGroups: + self.__authLogger.warn("Client is not authorized\nValid groups: %s\nClient: %s" % + (validGroups, credDict)) + return False + else: + if not authorized: + self.__authLogger.debug("Accepted request from unsecure transport") + return True + else: + self.__authLogger.debug("User is invalid or does not belong to the group it's saying") + return False def getValidPropertiesForMethod(self, method, defaultProperties=False): - """ - Get all authorized groups for calling a method + """ Get all authorized groups for calling a method + + :param str method: Method to test + :param defaultProperties: default properties + :type defaultProperties: list or tuple - :type method: string - :param method: Method to test - :return: List containing the allowed groups + :return: list -- List containing the allowed groups """ authProps = gConfig.getValue("%s/%s" % (self.authSection, method), []) if authProps: @@ -197,11 +278,11 @@ def getValidPropertiesForMethod(self, method, defaultProperties=False): return [] def getValidGroups(self, rawProperties): - """ Get valid groups as specified in the method authorization rules + """ Get valid groups as specified in the method authorization rules + + :param list rawProperties: all method properties - :param rawProperties: all method properties - :type rawProperties: python:list - :return: list of allowed groups or [] + :return: list -- list of allowed groups """ validGroups = [] for prop in list(rawProperties): @@ -219,103 +300,26 @@ def getValidGroups(self, rawProperties): validGroups = list(set(validGroups)) return validGroups - def forwardedCredentials(self, credDict): - """ - Check whether the credentials are being forwarded by a valid source - - :type credDict: dictionary - :param credDict: Credentials to ckeck - :return: Boolean with the result - """ - if self.KW_EXTRA_CREDENTIALS in credDict and isinstance(credDict[self.KW_EXTRA_CREDENTIALS], (tuple, list)): - if self.KW_DN in credDict: - retVal = Registry.getHostnameForDN(credDict[self.KW_DN]) - if retVal['OK']: - hostname = retVal['Value'] - if Properties.TRUSTED_HOST in Registry.getPropertiesForHost(hostname, []): - return True - return False - - def unpackForwardedCredentials(self, credDict): - """ - Extract the forwarded credentials - - :type credDict: dictionary - :param credDict: Credentials to unpack - """ - credDict[self.KW_DN] = credDict[self.KW_EXTRA_CREDENTIALS][0] - credDict[self.KW_GROUP] = credDict[self.KW_EXTRA_CREDENTIALS][1] - del(credDict[self.KW_EXTRA_CREDENTIALS]) - - def getUsername(self, credDict): - """ - Discover the username associated to the DN. It will check if the selected group is valid. - The username will be included in the credentials dictionary. - - :type credDict: dictionary - :param credDict: Credentials to check - :return: Boolean specifying whether the username was found - """ - if self.KW_DN not in credDict: - return True - if self.KW_GROUP not in credDict: - result = Registry.findDefaultGroupForDN(credDict[self.KW_DN]) - if not result['OK']: - return False - credDict[self.KW_GROUP] = result['Value'] - credDict[self.KW_PROPERTIES] = Registry.getPropertiesForGroup(credDict[self.KW_GROUP], []) - usersInGroup = Registry.getUsersInGroup(credDict[self.KW_GROUP], []) - if not usersInGroup: - return False - retVal = Registry.getUsernameForDN(credDict[self.KW_DN], usersInGroup) - if retVal['OK']: - credDict[self.KW_USERNAME] = retVal['Value'] - return True - return False + def matchProperties(self, credDict, validProps, caseSensitive=False): + """ Return True if one or more properties are in the valid list of properties - def isUserSuspended(self, credDict): - """ Discover if the user is in Suspended status + :param dict credDict: credentials to match + :param list validProps: List of valid properties + :param bool caseSensitive: Map lower case properties to properties to make the check in + lowercase but return the proper case - :param dict credDict: Credentials to check - :return: Boolean True if user is Suspended + :return: bool -- specifying whether any property has matched the valid ones """ - # Update credDict if the username is not there - if self.KW_USERNAME not in credDict: - self.getUsername(credDict) - # If username or group is not known we can not judge if the user is suspended - # These cases are treated elsewhere anyway - if self.KW_USERNAME not in credDict or self.KW_GROUP not in credDict: - return False - suspendedVOList = Registry.getUserOption(credDict[self.KW_USERNAME], 'Suspended', []) - if not suspendedVOList: - return False - vo = Registry.getVOForGroup(credDict[self.KW_GROUP]) - if vo in suspendedVOList: + if not validProps: return True - return False - - def matchProperties(self, credDict, validProps, caseSensitive=False): - """ - Return True if one or more properties are in the valid list of properties - - :type props: list - :param props: List of properties to match - :type validProps: list - :param validProps: List of valid properties - :return: Boolean specifying whether any property has matched the valid ones - """ - - # HACK: Map lower case properties to properties to make the check in lowercase but return the proper case if not caseSensitive: validProps = dict((prop.lower(), prop) for prop in validProps) else: validProps = dict((prop, prop) for prop in validProps) - groupProperties = credDict[self.KW_PROPERTIES] foundProps = [] - for prop in groupProperties: + for prop in credDict[KW_PROPERTIES]: if not caseSensitive: prop = prop.lower() if prop in validProps: foundProps.append(validProps[prop]) - credDict[self.KW_PROPERTIES] = foundProps - return foundProps + return bool(foundProps) diff --git a/Core/DISET/test/Test_AuthManager.py b/Core/DISET/test/Test_AuthManager.py index 56645994f00..8e9c2129983 100644 --- a/Core/DISET/test/Test_AuthManager.py +++ b/Core/DISET/test/Test_AuthManager.py @@ -4,14 +4,43 @@ from __future__ import division from __future__ import print_function +import os +import pickle import unittest from diraccfg import CFG -from DIRAC import gConfig +from DIRAC import gConfig, rootPath from DIRAC.Core.DISET.AuthManager import AuthManager __RCSID__ = "$Id$" +workDir = os.path.join(gConfig.getValue('/LocalSite/InstancePath', rootPath), 'work/ProxyManager') + +voDict = { + 'testVO': { + '/User/test/DN/CN=userS': { + 'Suspended': True, + 'VOMSRoles': [u'/testVO'], + 'ActuelRoles': [], + 'SuspendedRoles': [] + }, + '/User/test/DN/CN=userA': { + 'Suspended': False, + 'VOMSRoles': [u'/testVO'], + 'ActuelRoles': [], + 'SuspendedRoles': [] + } + }, + 'testVOOther': { + '/User/test/DN/CN=userS': { + 'Suspended': False, + 'VOMSRoles': [u'/testVOOther'], + 'ActuelRoles': [], + 'SuspendedRoles': [] + } + } +} + testSystemsCFG = """ Systems { @@ -46,6 +75,7 @@ testVO { VOAdmin = userA + VOMSName = testVO } testVOBad { @@ -54,6 +84,7 @@ testVOOther { VOAdmin = userA + VOMSName = testVOOther } } Users @@ -91,6 +122,7 @@ { Users = userA, userS VO = testVO + VOMSRole = /testVO Properties = NormalUser } group_test_other @@ -113,6 +145,19 @@ class AuthManagerTest(unittest.TestCase): """ Base class for the Modules test cases """ + @classmethod + def setUpClass(cls): + if not os.path.exists(workDir): + os.makedirs(workDir) + for vo, infoDict in voDict.items(): + with open(os.path.join(workDir, vo + '.pkl'), 'wb+') as f: + pickle.dump(infoDict, f, pickle.HIGHEST_PROTOCOL) + + @classmethod + def tearDownClass(cls): + for vo in voDict.keys(): + if os.path.exists(os.path.join(workDir, vo + '.pkl')): + os.remove(os.path.join(workDir, vo + '.pkl')) def setUp(self): self.authMgr = AuthManager('/Systems/Service/Authorization') From 8b0143a9576db9202f172b4097ec535ed3e05a32 Mon Sep 17 00:00:00 2001 From: TaykYoku Date: Tue, 30 Jun 2020 17:06:21 +0200 Subject: [PATCH 009/105] Core: research connectingCredentials in getRemoteCredentials --- Core/DISET/RequestHandler.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Core/DISET/RequestHandler.py b/Core/DISET/RequestHandler.py index 0d423b92ac1..a13046c4688 100755 --- a/Core/DISET/RequestHandler.py +++ b/Core/DISET/RequestHandler.py @@ -18,6 +18,8 @@ from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.Core.Security.Properties import CS_ADMINISTRATOR +from DIRAC.Core.DISET.AuthManager import initializationOfCertificate, initializationOfGroup,\ + forwardingCredentials, initializationOfSession def getServiceOption(serviceInfo, optionName, defaultValue): @@ -98,7 +100,14 @@ def getRemoteCredentials(self): :return: Credentials dictionary of remote peer. """ - return self.__trPool.get(self.__trid).getConnectingCredentials() + credDict = self.__trPool.get(self.__trid).getConnectingCredentials() + forwardingCredentials(credDict, logObj=self.log) + if credDict.get('ID'): + initializationOfSession(credDict, logObj=self.log) + else: + initializationOfCertificate(credDict, logObj=self.log) + initializationOfGroup(credDict, logObj=self.log) + return credDict @classmethod def getCSOption(cls, optionName, defaultValue=False): From cfc90888183506f3876a5f226de27411dccd845f Mon Sep 17 00:00:00 2001 From: TaykYoku Date: Tue, 30 Jun 2020 17:07:34 +0200 Subject: [PATCH 010/105] Core: use ID as IdP ID in TreadConfig --- Core/DISET/ThreadConfig.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/Core/DISET/ThreadConfig.py b/Core/DISET/ThreadConfig.py index bc86d58936a..a70de604f38 100644 --- a/Core/DISET/ThreadConfig.py +++ b/Core/DISET/ThreadConfig.py @@ -28,6 +28,7 @@ def reset(self): """ Reset extra information """ self.__DN = False + self.__ID = False self.__group = False self.__deco = False self.__setup = False @@ -74,21 +75,19 @@ def getGroup(self): """ return self.__group - def setID(self, DN, group): + def setID(self, ID): """ Set user ID - :param str DN: user DN - :param str group: user group + :param str ID: user ID """ - self.__DN = DN - self.__group = group + self.__ID = ID def getID(self): """ Return user ID - :return: tuple + :return: str """ - return (self.__DN, self.__group) + return self.__ID def setSetup(self, setup): """ Set setup name @@ -109,19 +108,17 @@ def dump(self): :return: tuple """ - return (self.__DN, self.__group, self.__setup) + return (self.__ID, self.__DN, self.__group, self.__setup) def load(self, tp): """ Save extra information :param tuple tp: contain DN, group name, setup name """ - if tp[0]: - self.__DN = tp[0] - if tp[1]: - self.__group = tp[1] - if tp[2]: - self.__setup = tp[2] + self.__ID = tp[0] or self.__ID + self.__DN = tp[1] or self.__DN + self.__group = tp[2] or self.__group + self.__setup = tp[3] or self.__setup def threadDeco(method): From 1eb839683ba0d10e08a1ae1929af2f69d9e04678 Mon Sep 17 00:00:00 2001 From: TaykYoku Date: Tue, 30 Jun 2020 17:17:10 +0200 Subject: [PATCH 011/105] Core: move WebApp cores to DIRAC --- Core/Web/App.py | 224 ++++++++++++++++ Core/Web/Conf.py | 331 +++++++++++++++++++++++ Core/Web/CoreHandler.py | 35 +++ Core/Web/HandlerMgr.py | 176 ++++++++++++ Core/Web/SessionData.py | 178 +++++++++++++ Core/Web/StaticHandler.py | 29 ++ Core/Web/TemplateLoader.py | 32 +++ Core/Web/WebHandler.py | 442 +++++++++++++++++++++++++++++++ Core/Web/__init__.py | 5 + Core/scripts/dirac-webapp-run.py | 54 ++++ 10 files changed, 1506 insertions(+) create mode 100644 Core/Web/App.py create mode 100644 Core/Web/Conf.py create mode 100644 Core/Web/CoreHandler.py create mode 100644 Core/Web/HandlerMgr.py create mode 100644 Core/Web/SessionData.py create mode 100644 Core/Web/StaticHandler.py create mode 100644 Core/Web/TemplateLoader.py create mode 100644 Core/Web/WebHandler.py create mode 100755 Core/Web/__init__.py create mode 100644 Core/scripts/dirac-webapp-run.py diff --git a/Core/Web/App.py b/Core/Web/App.py new file mode 100644 index 00000000000..4ddcf69f7cb --- /dev/null +++ b/Core/Web/App.py @@ -0,0 +1,224 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import ssl +import imp +import sys +import signal +import tornado.web +import tornado.process +import tornado.httpserver +import tornado.autoreload + +from diraccfg import CFG + +import DIRAC + +from DIRAC import gLogger, gConfig +from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals +from DIRAC.Core.Web.HandlerMgr import HandlerMgr +from DIRAC.Core.Web.TemplateLoader import TemplateLoader +from DIRAC.Core.Web.SessionData import SessionData +from DIRAC.Core.Web import Conf + +__RCSID__ = "$Id$" + + +class App(object): + + def __init__(self, handlersLoc='WebApp.handler'): + self.__handlerMgr = HandlerMgr(handlersLoc, Conf.rootURL()) + self.__servers = {} + self.log = gLogger.getSubLogger("Web") + + def _logRequest(self, handler): + status = handler.get_status() + if status < 400: + logm = self.log.notice + elif status < 500: + logm = self.log.warn + else: + logm = self.log.error + request_time = 1000.0 * handler.request.request_time() + logm("%d %s %.2fms" % (status, handler._request_summary(), request_time)) + + def __reloadAppCB(self): + gLogger.notice("\n !!!!!! Reloading web app...\n") + + def _loadWebAppCFGFiles(self): + """ + Load WebApp/web.cfg definitions + """ + exts = [] + for ext in CSGlobals.getCSExtensions(): + if ext == "DIRAC": + continue + if ext[-5:] != "DIRAC": + ext = "%sDIRAC" % ext + if ext != "WebAppDIRAC": + exts.append(ext) + exts.append("DIRAC") + exts.append("WebAppDIRAC") + webCFG = CFG() + for modName in reversed(exts): + try: + modPath = imp.find_module(modName)[1] + except ImportError: + continue + gLogger.verbose("Found module %s at %s" % (modName, modPath)) + cfgPath = os.path.join(modPath, "WebApp", "web.cfg") + if not os.path.isfile(cfgPath): + gLogger.verbose("Inexistant %s" % cfgPath) + continue + try: + modCFG = CFG().loadFromFile(cfgPath) + except Exception as excp: + gLogger.error("Could not load %s: %s" % (cfgPath, excp)) + continue + gLogger.verbose("Loaded %s" % cfgPath) + expl = [Conf.BASECS] + while len(expl): + current = expl.pop(0) + if not modCFG.isSection(current): + continue + if modCFG.getOption("%s/AbsoluteDefinition" % current, False): + gLogger.verbose("%s:%s is an absolute definition" % (modName, current)) + try: + webCFG.deleteKey(current) + except BaseException: + pass + modCFG.deleteKey("%s/AbsoluteDefinition" % current) + else: + for sec in modCFG[current].listSections(): + expl.append("%s/%s" % (current, sec)) + # Add the modCFG + webCFG = webCFG.mergeWith(modCFG) + gConfig.loadCFG(webCFG) + + def _loadDefaultWebCFG(self): + """ This method reloads the web.cfg file from etc/web.cfg """ + modCFG = None + cfgPath = os.path.join(DIRAC.rootPath, 'etc', 'web.cfg') + isLoaded = True + if not os.path.isfile(cfgPath): + isLoaded = False + else: + try: + modCFG = CFG().loadFromFile(cfgPath) + except Exception as excp: + isLoaded = False + gLogger.error("Could not load %s: %s" % (cfgPath, excp)) + + if modCFG: + if modCFG.isSection("/Website"): + gLogger.warn("%s configuration file is not correct. It is used by the old portal!" % (cfgPath)) + isLoaded = False + else: + gConfig.loadCFG(modCFG) + else: + isLoaded = False + + return isLoaded + + def stopChildProcesses(self, sig, frame): + """ + It is used to properly stop tornado when more than one process is used. + In principle this is doing the job of runsv.... + :param int sig: the signal sent to the process + :param object frame: execution frame which contains the child processes + """ + # tornado.ioloop.IOLoop.instance().add_timeout(time.time()+5, sys.exit) + for child in frame.f_locals.get('children', []): + gLogger.info("Stopping child processes: %d" % child) + os.kill(child, signal.SIGTERM) + # tornado.ioloop.IOLoop.instance().stop() + # gLogger.info('exit success') + sys.exit(0) + + def bootstrap(self): + """ + Configure and create web app + """ + self.log.always("\n ====== Starting DIRAC web app ====== \n") + + # Load required CFG files + if not self._loadDefaultWebCFG(): + # if we have a web.cfg under etc directory we use it, otherwise + # we use the configuration file defined by the developer + self._loadWebAppCFGFiles() + # Calculating routes + result = self.__handlerMgr.getRoutes() + if not result['OK']: + return result + routes = result['Value'] + # Initialize the session data + SessionData.setHandlers(self.__handlerMgr.getHandlers()['Value']) + # Create the app + tLoader = TemplateLoader(self.__handlerMgr.getPaths("template")) + kw = dict(debug=Conf.devMode(), template_loader=tLoader, cookie_secret=str(Conf.cookieSecret()), + log_function=self._logRequest, autoreload=Conf.numProcesses() < 2) + + # please do no move this lines. The lines must be before the fork_processes + signal.signal(signal.SIGTERM, self.stopChildProcesses) + signal.signal(signal.SIGINT, self.stopChildProcesses) + + # Check processes if we're under a load balancert + if Conf.balancer() and Conf.numProcesses() not in (0, 1): + tornado.process.fork_processes(Conf.numProcesses(), max_restarts=0) + kw['debug'] = False + # Debug mode? + if kw['debug']: + self.log.info("Configuring in developer mode...") + # Configure tornado app + self.__app = tornado.web.Application(routes, **kw) + self.log.notice("Configuring HTTP on port %s" % (Conf.HTTPPort())) + # Create the web servers + srv = tornado.httpserver.HTTPServer(self.__app, xheaders=True) + port = Conf.HTTPPort() + srv.listen(port) + self.__servers[('http', port)] = srv + + Conf.generateRevokedCertsFile() # it is used by nginx.... + + if Conf.HTTPS(): + self.log.notice("Configuring HTTPS on port %s" % Conf.HTTPSPort()) + sslops = dict(certfile=Conf.HTTPSCert(), + keyfile=Conf.HTTPSKey(), + cert_reqs=ssl.CERT_OPTIONAL, + ca_certs=Conf.generateCAFile(), + ssl_version=ssl.PROTOCOL_TLSv1) + + sslprotocol = str(Conf.SSLProrocol()) + aviableProtocols = [i for i in dir(ssl) if i.find('PROTOCOL') == 0] + if sslprotocol and sslprotocol != "": + if (sslprotocol in aviableProtocols): + sslops['ssl_version'] = getattr(ssl, sslprotocol) + else: + message = "%s protocol is not provided." % sslprotocol + message += "The following protocols are provided: %s" % str(aviableProtocols) + gLogger.warn(message) + + self.log.debug(" - %s" % "\n - ".join(["%s = %s" % (k, sslops[k]) for k in sslops])) + srv = tornado.httpserver.HTTPServer(self.__app, ssl_options=sslops, xheaders=True) + port = Conf.HTTPSPort() + srv.listen(port) + self.__servers[('https', port)] = srv + else: + # when NGINX is used then the Conf.HTTPS return False, it means tornado + # does not have to be configured using 443 port + Conf.generateCAFile() # if we use Nginx we have to generate the cas as well... + return result + + def run(self): + """ + Start web servers + """ + bu = Conf.rootURL().strip("/") + urls = [] + for proto, port in self.__servers: + urls.append("%s://0.0.0.0:%s/%s/" % (proto, port, bu)) + self.log.always("Listening on %s" % " and ".join(urls)) + tornado.autoreload.add_reload_hook(self.__reloadAppCB) + tornado.ioloop.IOLoop.instance().start() diff --git a/Core/Web/Conf.py b/Core/Web/Conf.py new file mode 100644 index 00000000000..0e351dfee4a --- /dev/null +++ b/Core/Web/Conf.py @@ -0,0 +1,331 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import uuid +import tempfile +import tornado.process +from DIRAC import gConfig +from DIRAC.Core.Security import Locations, X509Chain, X509CRL + +__RCSID__ = "$Id$" + +BASECS = "/WebApp" + + +def getCSValue(opt, defValue=None): + """ Get option from CS + + :param str opt: option + :param defValue: default value + + :return: defValue type result + """ + return gConfig.getValue("%s/%s" % (BASECS, opt), defValue) + + +def getCSSections(opt): + """ Get sections from CS + + :param str opt: option + + :return: S_OK(list)/S_ERROR() + """ + return gConfig.getSections("%s/%s" % (BASECS, opt)) + + +def getCSOptions(opt): + """ Get options from CS + + :param str opt: option + + :return: S_OK(list)/S_ERROR() + """ + return gConfig.getOptions("%s/%s" % (BASECS, opt)) + + +def getCSOptionsDict(opt): + """ Get options dictionary from CS + + :param str opt: option + + :return: S_OK(dict)/S_ERROR() + """ + return gConfig.getOptionsDict("%s/%s" % (BASECS, opt)) + + +def getTitle(): + """ Get title + + :return: str + """ + defVal = gConfig.getValue("/DIRAC/Configuration/Name", gConfig.getValue("/DIRAC/Setup")) + return "%s - DIRAC" % gConfig.getValue("%s/Title" % BASECS, defVal) + + +def devMode(): + """ Get development mode status + + :result: bool + """ + return getCSValue("DevelopMode", False) + + +def rootURL(): + """ Get root URL + + :return: str + """ + return getCSValue("RootURL", "/DIRAC") + + +def balancer(): + """ Get balancer + + :return: str + """ + b = getCSValue("Balancer", "").lower() + if b in ("", "none"): + return "" + return b + + +def numProcesses(): + """ Get number of processes + + :return: int + """ + return getCSValue("NumProcesses", 1) + + +def HTTPS(): + """ Get flag of enable HTTPS + + :return: bool + """ + if balancer(): + return False + return getCSValue("HTTPS/Enabled", True) + + +def HTTPPort(): + """ Get HTTP port + + :return: int + """ + if balancer(): + default = 8000 + else: + default = 8080 + procAdd = tornado.process.task_id() or 0 + return getCSValue("HTTP/Port", default) + procAdd + + +def HTTPSPort(): + """ Get HTTPS port + + :return: int + """ + return getCSValue("HTTPS/Port", 8443) + + +def HTTPSCert(): + """ Get certificate path for HTTPS + + :return: str + """ + cert = Locations.getHostCertificateAndKeyLocation() + if cert: + cert = cert[0] + else: + cert = "/opt/dirac/etc/grid-security/hostcert.pem" + return getCSValue("HTTPS/Cert", cert) + + +def HTTPSKey(): + """ Get key path for HTTPS + + :return: str + """ + key = Locations.getHostCertificateAndKeyLocation() + if key: + key = key[1] + else: + key = "/opt/dirac/etc/grid-security/hostkey.pem" + return getCSValue("HTTPS/Key", key) + + +def setup(): + """ Get setup path + + :return: str + """ + return gConfig.getValue("/DIRAC/Setup") + + +def cookieSecret(): + """ Get cookie secret + + :return: str + """ + # TODO: Store the secret somewhere + return gConfig.getValue("CookieSecret", uuid.getnode()) + + +def generateCAFile(): + """ Generate a single CA file with all the PEMs + + :return: str or bool + """ + caDir = Locations.getCAsLocation() + for fn in (os.path.join(os.path.dirname(caDir), "cas.pem"), + os.path.join(os.path.dirname(HTTPSCert()), "cas.pem"), + False): + if not fn: + fn = tempfile.mkstemp(prefix="cas.", suffix=".pem")[1] + try: + fd = open(fn, "w") + except IOError: + continue + for caFile in os.listdir(caDir): + caFile = os.path.join(caDir, caFile) + result = X509Chain.X509Chain.instanceFromFile(caFile) + if not result['OK']: + continue + chain = result['Value'] + expired = chain.hasExpired() + if not expired['OK'] or expired['Value']: + continue + fd.write(chain.dumpAllToString()['Value']) + fd.close() + return fn + return False + + +def generateRevokedCertsFile(): + """ Generate a single CA file with all the PEMs + + :return: str or bool + """ + caDir = Locations.getCAsLocation() + for fn in (os.path.join(os.path.dirname(caDir), "allRevokedCerts.pem"), + os.path.join(os.path.dirname(HTTPSCert()), "allRevokedCerts.pem"), + False): + if not fn: + fn = tempfile.mkstemp(prefix="allRevokedCerts", suffix=".pem")[1] + try: + fd = open(fn, "w") + except IOError: + continue + for caFile in os.listdir(caDir): + caFile = os.path.join(caDir, caFile) + result = X509CRL.X509CRL.instanceFromFile(caFile) + if not result['OK']: + continue + chain = result['Value'] + fd.write(chain.dumpAllToString()['Value']) + fd.close() + return fn + return False + + +def getAuthSectionForHandler(route): + """ Get auth section for handler + + :return: str + """ + return "%s/Access/%s" % (BASECS, route) + + +def getTheme(): + """ Get theme + + :return: str + """ + return getCSValue("Theme", "tabs") + + +def getIcon(): + """ Get icon path + + :return: str + """ + return getCSValue("Icon", "/static/core/img/icons/system/favicon.ico") + + +def SSLProrocol(): + """ Get ssl protocol + + :return: str + """ + return getCSValue("SSLProtcol", "") + + +def getDefaultStaticDirs(): + """ Get default static directories + + :return: list + """ + defDirs = getCSValue("DefaultStaticDirs", ['defaults', 'demo']) + if defDirs == ['None']: + return [] + return defDirs + + +def getStaticDirs(): + """ Get static directories + + :return: str + """ + return list(set(getCSValue("StaticDirs", []) + getDefaultStaticDirs())) + + +def getLogo(): + """ Get logo path + + :return: str + """ + return getCSValue("Logo", "/static/core/img/icons/system/_logo_waiting.gif") + + +def getBackgroud(): + """ Get background path + + :return: str + """ + return getCSValue("BackgroundImage", "/static/core/img/wallpapers/dirac_background_6.png") + + +def getWelcome(): + """ Get welcome + + :return: str + """ + return getCSValue("WelcomeHTML", "") + + +def bugReportURL(): + """ Get bug report URL + + :return: str + """ + return getCSValue("bugReportURL", "") + + +def getAuthNames(): + """ Get enabled id providers + + :return: S_OK(list)/S_ERROR() + """ + return getCSSections("TypeAuths") + + +def getAppSettings(app): + """ Get applications options + + :param str app: application name + + :return: S_OK(dict)/S_ERROR + """ + return gConfig.getOptionsDictRecursively("%s/%s" % (BASECS, app)) diff --git a/Core/Web/CoreHandler.py b/Core/Web/CoreHandler.py new file mode 100644 index 00000000000..20a0679d4d9 --- /dev/null +++ b/Core/Web/CoreHandler.py @@ -0,0 +1,35 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import urlparse +import tornado.web + +from DIRAC.Core.Web import Conf + +__RCSID__ = "$Id$" + + +class CoreHandler(tornado.web.RequestHandler): + + def initialize(self, action): + self.__action = action + + def get(self, setup, group, route): + if self.__action == "addSlash": + o = urlparse.urlparse(self.request.uri) + proto = self.request.protocol + if 'X-Scheme' in self.request.headers: + proto = self.request.headers['X-Scheme'] + nurl = "%s://%s%s/" % (proto, self.request.host, o.path) + if o.query: + nurl = "%s?%s" % (nurl, o.query) + self.redirect(nurl, permanent=True) + elif self.__action == "sendToRoot": + dest = "/" + rootURL = Conf.rootURL() + if rootURL: + dest += "%s/" % rootURL.strip("/") + if setup and group: + dest += "s:%s/g:%s/" % (setup, group) + self.redirect(dest) diff --git a/Core/Web/HandlerMgr.py b/Core/Web/HandlerMgr.py new file mode 100644 index 00000000000..6f8c2a123b7 --- /dev/null +++ b/Core/Web/HandlerMgr.py @@ -0,0 +1,176 @@ +""" Basic modules for loading handlers +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import re +import imp +import inspect +import collections + +from DIRAC import S_OK, S_ERROR, rootPath, gLogger +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader +from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton +from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals + +from DIRAC.Core.Web import Conf +from DIRAC.Core.Web.WebHandler import WebHandler, WebSocketHandler +from DIRAC.Core.Web.CoreHandler import CoreHandler +from DIRAC.Core.Web.StaticHandler import StaticHandler + +__RCSID__ = "$Id$" + + +class HandlerMgr(object): + __metaclass__ = DIRACSingleton + + def __init__(self, sysService=[], baseURL="/"): + """ Constructor + + :param list sysService: DIRAC system services + :param basestring baseURL: base URL + """ + self.__baseURL = baseURL.strip("/") + if sysService and not isinstance(sysService, list): + sysService = sysService.replace(' ', '').split(',') + self.__sysServices = sysService + self.__routes = [] + self.__handlers = [] + self.__setupGroupRE = r"(?:/s:([\w-]*)/g:([\w.-]*))?" + self.__shySetupGroupRE = r"(?:/s:(?:[\w-]*)/g:(?:[\w.-]*))?" + self.log = gLogger.getSubLogger("Routing") + + def getPaths(self, dirName): + """ Get lists of paths for all installed and enabled extensions + + :param basestring dirName: path to handlers + + :return: list + """ + pathList = [] + extNames = CSGlobals.getCSExtensions() + if 'WebAppDIRAC' in extNames: + extNames.append(extNames.pop(extNames.index('WebAppDIRAC'))) + for extName in extNames: + if not extName.endswith('DIRAC'): + extName = "%sDIRAC" % extName + try: + modFile, modPath, desc = imp.find_module(extName) + # to match in the real root path to enabling module web extensions (static, templates...) + realModPath = os.path.realpath(modPath) + except ImportError: + continue + staticPath = os.path.join(realModPath, "WebApp", dirName) + if os.path.isdir(staticPath): + pathList.append(staticPath) + return pathList + + def __calculateRoutes(self): + """ Load all handlers and generate the routes + + :return: S_OK()/S_ERROR() + """ + ol = ObjectLoader() + hendlerList = [] + self.log.debug("Added services: %s" % ','.join(self.__sysServices)) + for origin in self.__sysServices: + result = ol.getObjects(origin, parentClass=WebHandler, recurse=True) + if not result['OK']: + return result + hendlerList += list(result['Value'].items()) + self.__handlers = collections.OrderedDict(hendlerList) + + staticPaths = self.getPaths("static") + self.log.verbose("Static paths found:\n - %s" % "\n - ".join(staticPaths)) + self.__routes = [] + + # Add some standard paths for static files + statDirectories = Conf.getStaticDirs() + self.log.info("The following static directories are used:%s" % str(statDirectories)) + for stdir in statDirectories: + pattern = '/%s/(.*)' % stdir + self.__routes.append((pattern, StaticHandler, dict(pathList=['%s/webRoot/www/%s' % (rootPath, stdir)]))) + self.log.debug(" - Static route: %s" % pattern) + + for pattern in ((r"/static/(.*)", r"/(favicon\.ico)", r"/(robots\.txt)")): + pattern = r"%s%s" % (self.__shySetupGroupRE, pattern) + if self.__baseURL: + pattern = "/%s%s" % (self.__baseURL, pattern) + self.__routes.append((pattern, StaticHandler, dict(pathList=staticPaths))) + self.log.debug(" - Static route: %s" % pattern) + for hn in self.__handlers: + self.log.info("Found handler %s" % hn) + handler = self.__handlers[hn] + # CHeck it has AUTH_PROPS + if isinstance(handler.AUTH_PROPS, type(None)): + return S_ERROR("Handler %s does not have AUTH_PROPS defined. Fix it!" % hn) + # Get the root for the handler + if handler.LOCATION: + handlerRoute = handler.LOCATION.strip("/") and "/%s" % handler.LOCATION.strip("/") or '' + else: + handlerRoute = hn[len(re.sub(r".[A-z]+$", "", hn)):].replace(".", "/").replace("Handler", "") + # Add the setup group RE before + baseRoute = self.__setupGroupRE + # IF theres a base url like /DIRAC add it + baseRoute = "/%s%s" % (self.__baseURL or '', baseRoute) + # Set properly the LOCATION after calculating where it is with helpers to add group and setup later + handler.LOCATION = handlerRoute + handler.PATH_RE = re.compile("%s(%s/[A-z]+|.)" % (baseRoute, handlerRoute)) + if handler.OVERPATH: + handler.PATH_RE = re.compile(handler.PATH_RE.pattern + '(/[A-z0-9=-_/|]+)?') + handler.URLSCHEMA = "/%s%%(setup)s%%(group)s%%(location)s/%%(action)s" % (self.__baseURL) + if issubclass(handler, WebSocketHandler): + handler.PATH_RE = re.compile("%s(%s)" % (baseRoute, handlerRoute)) + route = "%s(%s)" % (baseRoute, handlerRoute) + self.__routes.append((route, handler)) + self.log.verbose(" - WebSocket %s -> %s" % (handlerRoute, hn)) + self.log.debug(" * %s" % route) + continue + # Look for methods that are exported + for mName, mObj in inspect.getmembers(handler): + if inspect.ismethod(mObj) and mName.find("web_") == 0: + if mName == "web_index": + # Index methods have the bare url + self.log.verbose(" - Route %s -> %s.web_index" % (handlerRoute, hn)) + route = "%s(%s/)" % (baseRoute, handlerRoute) + self.__routes.append((route, handler)) + self.__routes.append(("%s(%s)" % (baseRoute, handlerRoute), CoreHandler, dict(action='addSlash'))) + else: + # Normal methods get the method appended without web_ + self.log.verbose(" - Route %s/%s -> %s.%s" % (handlerRoute, mName[4:], hn, mName)) + route = "%s(%s/%s)" % (baseRoute, handlerRoute, mName[4:]) + # Use request path as options/values, for ex. ../method/