diff --git a/FrameworkSystem/Client/UserProfileClient.py b/FrameworkSystem/Client/UserProfileClient.py index ee63384b3de..93914f65cce 100644 --- a/FrameworkSystem/Client/UserProfileClient.py +++ b/FrameworkSystem/Client/UserProfileClient.py @@ -130,3 +130,6 @@ def getUserProfileNames(self, permission=dict()): it returns the available profile names by not taking account the permission: ReadAccess and PublishAccess """ return self.__getRPCClient().getUserProfileNames(permission) + + def listStatesForWeb(self, permission={}): + return self.__getRPCClient().listStatesForWeb(permission) diff --git a/FrameworkSystem/DB/UserProfileDB.py b/FrameworkSystem/DB/UserProfileDB.py index bd104c522c0..d6d593dd0f0 100755 --- a/FrameworkSystem/DB/UserProfileDB.py +++ b/FrameworkSystem/DB/UserProfileDB.py @@ -9,6 +9,8 @@ import sys import hashlib +import cachetools + from DIRAC import S_OK, S_ERROR, gLogger, gConfig from DIRAC.Core.Utilities import Time from DIRAC.ConfigurationSystem.Client.Helpers import Registry @@ -76,6 +78,7 @@ def __init__(self): """ self.__permValues = ['USER', 'GROUP', 'VO', 'ALL'] self.__permAttrs = ['ReadAccess', 'PublishAccess'] + self.__cache = cachetools.TTLCache(1024, 15) DB.__init__(self, 'UserProfileDB', 'Framework/UserProfileDB') retVal = self.__initializeDB() if not retVal['OK']: @@ -123,14 +126,32 @@ def __getGroupId(self, groupName, insertIfMissing=True): def __getVOId(self, voName, insertIfMissing=True): return self.__getObjId(voName, 'VO', 'up_VOs', insertIfMissing) + def __getFieldsCached(self, tableName, outFields, condDict): + """Call getFields with a TTL cache + + The UserProfileDB is written in such a way that repeatedly makes the same + DB queries thousands of times. To workaround this, use a simple short-lived + TTL cache to dramatically improve performance. + """ + key = (tableName, tuple(outFields), tuple(sorted(condDict.items()))) + if key not in self.__cache: + result = self.getFields(tableName, outFields, condDict) + if not result['OK']: + return result + data = result['Value'] + if len(data) > 0: + objId = data[0][0] + self.updateFields(tableName, ['LastAccess'], ['UTC_TIMESTAMP()'], {'Id': objId}) + self.__cache[key] = result + return self.__cache[key] + def __getObjId(self, objValue, varName, tableName, insertIfMissing=True): - result = self.getFields(tableName, ['Id'], {varName: objValue}) + result = self.__getFieldsCached(tableName, ['Id'], {varName: objValue}) if not result['OK']: return result data = result['Value'] if len(data) > 0: objId = data[0][0] - self.updateFields(tableName, ['LastAccess'], ['UTC_TIMESTAMP()'], {'Id': objId}) return S_OK(objId) if not insertIfMissing: return S_ERROR("No entry %s for %s defined in the DB" % (objValue, varName)) diff --git a/FrameworkSystem/Service/UserProfileManagerHandler.py b/FrameworkSystem/Service/UserProfileManagerHandler.py index e81d85b2808..2dd9e6b16cd 100644 --- a/FrameworkSystem/Service/UserProfileManagerHandler.py +++ b/FrameworkSystem/Service/UserProfileManagerHandler.py @@ -79,6 +79,32 @@ def export_deleteProfileVar(self, profileName, varName): userGroup = credDict['group'] return gUPDB.deleteVar(userName, userGroup, profileName, varName) + types_listStatesForWeb = [dict] + + def export_listStatesForWeb(self, permission): + retVal = self.export_getUserProfileNames(permission) + if not retVal["OK"]: + return retVal + data = retVal['Value'] + + records = [] + for i in data: + application = i.replace('Web/application/', '') + retVal = self.export_listAvailableProfileVars(i) + if not retVal['OK']: + return retVal + states = retVal['Value'] + for state in states: + record = dict(zip(['user', 'group', 'vo', 'name'], state)) + record['app'] = application + retVal = self.export_getProfileVarPermissions(i, record['name']) + if not retVal['OK']: + return retVal + record['permissions'] = retVal['Value'] + records += [record] + + return S_OK(records) + types_listAvailableProfileVars = [types.StringTypes] def export_listAvailableProfileVars(self, profileName, filterDict={}): diff --git a/docs/requirements.txt b/docs/requirements.txt index 397440010ff..a3bd7009f29 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,6 +5,7 @@ M2Crypto>=0.36 Sphinx>=1.8.0 docutils>=0.15 boto3 +cachetools<4 elasticsearch_dsl future futures diff --git a/docs/requirements_py3.txt b/docs/requirements_py3.txt index a2c9075a5b3..9c5b823a89d 100644 --- a/docs/requirements_py3.txt +++ b/docs/requirements_py3.txt @@ -5,6 +5,7 @@ M2Crypto==0.32 Sphinx>=1.8.0 boto3 elasticsearch_dsl +cachetools future futures matplotlib diff --git a/environment.yml b/environment.yml index d5df47ad7df..2fa389c3611 100644 --- a/environment.yml +++ b/environment.yml @@ -12,6 +12,7 @@ dependencies: - boto3 - certifi - cmreshandler >1.0.0b4 + - cachetools <4 - docutils - elasticsearch-dsl ~=6.3.1 - fts-rest diff --git a/requirements.txt b/requirements.txt index f134560bafb..0b550c7d4d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ boto3 #asn1 M2Crypto>=0.36 autopep8==1.3.3 +cachetools<4 certifi coverage docutils