From 1cf791954287c87c8093a170b4351cdfbc4c5c10 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 26 May 2021 16:11:38 +0200 Subject: [PATCH 1/3] Add listStatesForWeb endpoint to make Public State Manager considerably faster --- FrameworkSystem/Client/UserProfileClient.py | 4 +++ FrameworkSystem/DB/UserProfileDB.py | 25 ++++++++++++++++-- .../Service/UserProfileManagerHandler.py | 26 +++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/FrameworkSystem/Client/UserProfileClient.py b/FrameworkSystem/Client/UserProfileClient.py index ee63384b3de..d726bd2028b 100644 --- a/FrameworkSystem/Client/UserProfileClient.py +++ b/FrameworkSystem/Client/UserProfileClient.py @@ -130,3 +130,7 @@ def getUserProfileNames(self, permission=dict()): it returns the available profile names by not taking account the permission: ReadAccess and PublishAccess """ return self.__getRPCClient().getUserProfileNames(permission) + + def listStatesForWeb(self, permission={}): + rpcClient = self.__getRPCClient() + return rpcClient.listStatesForWeb(permission) diff --git a/FrameworkSystem/DB/UserProfileDB.py b/FrameworkSystem/DB/UserProfileDB.py index bd104c522c0..d6d593dd0f0 100755 --- a/FrameworkSystem/DB/UserProfileDB.py +++ b/FrameworkSystem/DB/UserProfileDB.py @@ -9,6 +9,8 @@ import sys import hashlib +import cachetools + from DIRAC import S_OK, S_ERROR, gLogger, gConfig from DIRAC.Core.Utilities import Time from DIRAC.ConfigurationSystem.Client.Helpers import Registry @@ -76,6 +78,7 @@ def __init__(self): """ self.__permValues = ['USER', 'GROUP', 'VO', 'ALL'] self.__permAttrs = ['ReadAccess', 'PublishAccess'] + self.__cache = cachetools.TTLCache(1024, 15) DB.__init__(self, 'UserProfileDB', 'Framework/UserProfileDB') retVal = self.__initializeDB() if not retVal['OK']: @@ -123,14 +126,32 @@ def __getGroupId(self, groupName, insertIfMissing=True): def __getVOId(self, voName, insertIfMissing=True): return self.__getObjId(voName, 'VO', 'up_VOs', insertIfMissing) + def __getFieldsCached(self, tableName, outFields, condDict): + """Call getFields with a TTL cache + + The UserProfileDB is written in such a way that repeatedly makes the same + DB queries thousands of times. To workaround this, use a simple short-lived + TTL cache to dramatically improve performance. + """ + key = (tableName, tuple(outFields), tuple(sorted(condDict.items()))) + if key not in self.__cache: + result = self.getFields(tableName, outFields, condDict) + if not result['OK']: + return result + data = result['Value'] + if len(data) > 0: + objId = data[0][0] + self.updateFields(tableName, ['LastAccess'], ['UTC_TIMESTAMP()'], {'Id': objId}) + self.__cache[key] = result + return self.__cache[key] + def __getObjId(self, objValue, varName, tableName, insertIfMissing=True): - result = self.getFields(tableName, ['Id'], {varName: objValue}) + result = self.__getFieldsCached(tableName, ['Id'], {varName: objValue}) if not result['OK']: return result data = result['Value'] if len(data) > 0: objId = data[0][0] - self.updateFields(tableName, ['LastAccess'], ['UTC_TIMESTAMP()'], {'Id': objId}) return S_OK(objId) if not insertIfMissing: return S_ERROR("No entry %s for %s defined in the DB" % (objValue, varName)) diff --git a/FrameworkSystem/Service/UserProfileManagerHandler.py b/FrameworkSystem/Service/UserProfileManagerHandler.py index e81d85b2808..2dd9e6b16cd 100644 --- a/FrameworkSystem/Service/UserProfileManagerHandler.py +++ b/FrameworkSystem/Service/UserProfileManagerHandler.py @@ -79,6 +79,32 @@ def export_deleteProfileVar(self, profileName, varName): userGroup = credDict['group'] return gUPDB.deleteVar(userName, userGroup, profileName, varName) + types_listStatesForWeb = [dict] + + def export_listStatesForWeb(self, permission): + retVal = self.export_getUserProfileNames(permission) + if not retVal["OK"]: + return retVal + data = retVal['Value'] + + records = [] + for i in data: + application = i.replace('Web/application/', '') + retVal = self.export_listAvailableProfileVars(i) + if not retVal['OK']: + return retVal + states = retVal['Value'] + for state in states: + record = dict(zip(['user', 'group', 'vo', 'name'], state)) + record['app'] = application + retVal = self.export_getProfileVarPermissions(i, record['name']) + if not retVal['OK']: + return retVal + record['permissions'] = retVal['Value'] + records += [record] + + return S_OK(records) + types_listAvailableProfileVars = [types.StringTypes] def export_listAvailableProfileVars(self, profileName, filterDict={}): From ff60eb180a9db58b146da0f3e81826d4d8860a3c Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 26 May 2021 18:06:26 +0200 Subject: [PATCH 2/3] Add cachetools to dependency files --- docs/requirements.txt | 1 + docs/requirements_py3.txt | 1 + environment.yml | 1 + requirements.txt | 1 + 4 files changed, 4 insertions(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 397440010ff..a3bd7009f29 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,6 +5,7 @@ M2Crypto>=0.36 Sphinx>=1.8.0 docutils>=0.15 boto3 +cachetools<4 elasticsearch_dsl future futures diff --git a/docs/requirements_py3.txt b/docs/requirements_py3.txt index a2c9075a5b3..9c5b823a89d 100644 --- a/docs/requirements_py3.txt +++ b/docs/requirements_py3.txt @@ -5,6 +5,7 @@ M2Crypto==0.32 Sphinx>=1.8.0 boto3 elasticsearch_dsl +cachetools future futures matplotlib diff --git a/environment.yml b/environment.yml index d5df47ad7df..2fa389c3611 100644 --- a/environment.yml +++ b/environment.yml @@ -12,6 +12,7 @@ dependencies: - boto3 - certifi - cmreshandler >1.0.0b4 + - cachetools <4 - docutils - elasticsearch-dsl ~=6.3.1 - fts-rest diff --git a/requirements.txt b/requirements.txt index f134560bafb..0b550c7d4d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ boto3 #asn1 M2Crypto>=0.36 autopep8==1.3.3 +cachetools<4 certifi coverage docutils From a96106d6a42386d81c0a80b150d81379755b12c7 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 27 May 2021 08:55:29 +0100 Subject: [PATCH 3/3] Update FrameworkSystem/Client/UserProfileClient.py Co-authored-by: fstagni --- FrameworkSystem/Client/UserProfileClient.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/FrameworkSystem/Client/UserProfileClient.py b/FrameworkSystem/Client/UserProfileClient.py index d726bd2028b..93914f65cce 100644 --- a/FrameworkSystem/Client/UserProfileClient.py +++ b/FrameworkSystem/Client/UserProfileClient.py @@ -132,5 +132,4 @@ def getUserProfileNames(self, permission=dict()): return self.__getRPCClient().getUserProfileNames(permission) def listStatesForWeb(self, permission={}): - rpcClient = self.__getRPCClient() - return rpcClient.listStatesForWeb(permission) + return self.__getRPCClient().listStatesForWeb(permission)