[v7r1] Add listStatesForWeb endpoint to make Public State Manager faster

FrameworkSystem/Client/UserProfileClient.py

@@ -130,3 +130,6 @@ def getUserProfileNames(self, permission=dict()):
     it returns the available profile names by not taking account the permission: ReadAccess and PublishAccess
     """
     return self.__getRPCClient().getUserProfileNames(permission)
+
+  def listStatesForWeb(self, permission={}):
+    return self.__getRPCClient().listStatesForWeb(permission)

FrameworkSystem/DB/UserProfileDB.py

@@ -9,6 +9,8 @@
 import sys
 import hashlib
 
+import cachetools
+
 from DIRAC import S_OK, S_ERROR, gLogger, gConfig
 from DIRAC.Core.Utilities import Time
 from DIRAC.ConfigurationSystem.Client.Helpers import Registry
@@ -76,6 +78,7 @@ def __init__(self):
     """
     self.__permValues = ['USER', 'GROUP', 'VO', 'ALL']
     self.__permAttrs = ['ReadAccess', 'PublishAccess']
+    self.__cache = cachetools.TTLCache(1024, 15)
     DB.__init__(self, 'UserProfileDB', 'Framework/UserProfileDB')
     retVal = self.__initializeDB()
     if not retVal['OK']:
@@ -123,14 +126,32 @@ def __getGroupId(self, groupName, insertIfMissing=True):
   def __getVOId(self, voName, insertIfMissing=True):
     return self.__getObjId(voName, 'VO', 'up_VOs', insertIfMissing)
 
+  def __getFieldsCached(self, tableName, outFields, condDict):
+    """Call getFields with a TTL cache
+
+    The UserProfileDB is written in such a way that repeatedly makes the same
+    DB queries thousands of times. To workaround this, use a simple short-lived
+    TTL cache to dramatically improve performance.
+    """
+    key = (tableName, tuple(outFields), tuple(sorted(condDict.items())))
+    if key not in self.__cache:
+      result = self.getFields(tableName, outFields, condDict)
+      if not result['OK']:
+        return result
+      data = result['Value']
+      if len(data) > 0:
+        objId = data[0][0]
+        self.updateFields(tableName, ['LastAccess'], ['UTC_TIMESTAMP()'], {'Id': objId})
+      self.__cache[key] = result
+    return self.__cache[key]
+
   def __getObjId(self, objValue, varName, tableName, insertIfMissing=True):
-    result = self.getFields(tableName, ['Id'], {varName: objValue})
+    result = self.__getFieldsCached(tableName, ['Id'], {varName: objValue})
     if not result['OK']:
       return result
     data = result['Value']
     if len(data) > 0:
       objId = data[0][0]
-      self.updateFields(tableName, ['LastAccess'], ['UTC_TIMESTAMP()'], {'Id': objId})
       return S_OK(objId)
     if not insertIfMissing:
       return S_ERROR("No entry %s for %s defined in the DB" % (objValue, varName))

FrameworkSystem/Service/UserProfileManagerHandler.py

@@ -79,6 +79,32 @@ def export_deleteProfileVar(self, profileName, varName):
     userGroup = credDict['group']
     return gUPDB.deleteVar(userName, userGroup, profileName, varName)
 
+  types_listStatesForWeb = [dict]
+
+  def export_listStatesForWeb(self, permission):
+    retVal = self.export_getUserProfileNames(permission)
+    if not retVal["OK"]:
+      return retVal
+    data = retVal['Value']
+
+    records = []
+    for i in data:
+      application = i.replace('Web/application/', '')
+      retVal = self.export_listAvailableProfileVars(i)
+      if not retVal['OK']:
+        return retVal
+      states = retVal['Value']
+      for state in states:
+        record = dict(zip(['user', 'group', 'vo', 'name'], state))
+        record['app'] = application
+        retVal = self.export_getProfileVarPermissions(i, record['name'])
+        if not retVal['OK']:
+          return retVal
+        record['permissions'] = retVal['Value']
+        records += [record]
+
+    return S_OK(records)
+
   types_listAvailableProfileVars = [types.StringTypes]
 
   def export_listAvailableProfileVars(self, profileName, filterDict={}):

docs/requirements.txt

@@ -5,6 +5,7 @@ M2Crypto>=0.36
 Sphinx>=1.8.0
 docutils>=0.15
 boto3
+cachetools<4
 elasticsearch_dsl
 future
 futures

docs/requirements_py3.txt

@@ -5,6 +5,7 @@ M2Crypto==0.32
 Sphinx>=1.8.0
 boto3
 elasticsearch_dsl
+cachetools
 future
 futures
 matplotlib

environment.yml

@@ -12,6 +12,7 @@ dependencies:
   - boto3
   - certifi
   - cmreshandler >1.0.0b4
+  - cachetools <4
   - docutils
   - elasticsearch-dsl ~=6.3.1
   - fts-rest

requirements.txt

@@ -7,6 +7,7 @@ boto3
 #asn1
 M2Crypto>=0.36
 autopep8==1.3.3
+cachetools<4
 certifi
 coverage
 docutils