[v7r0] Configuration options docs for RMS

ConfigurationSystem/test/Test_agentOptions.py

@@ -20,16 +20,8 @@
           ('DIRAC.FrameworkSystem.Agent.ErrorMessageMonitor', {}),
           ('DIRAC.FrameworkSystem.Agent.SystemLoggingDBCleaner', {'IgnoreOptions': ['RemoveDate']}),
           ('DIRAC.FrameworkSystem.Agent.TopErrorMessagesReporter', {}),
-          ('DIRAC.RequestManagementSystem.Agent.CleanReqDBAgent', {'IgnoreOptions': ['KickLimit', 'KickGraceHours',
-                                                                                     'DeleteGraceDays']}),
-          ('DIRAC.RequestManagementSystem.Agent.RequestExecutingAgent', {'IgnoreOptions': ['MaxProcess',
-                                                                                           'ProcessTaskTimeout',
-                                                                                           'RequestsPerCycle',
-                                                                                           'OperationHandlers',
-                                                                                           'MinProcess', 'MaxAttempts',
-                                                                                           'ProcessPoolQueueSize',
-                                                                                           'ProcessPoolSleep',
-                                                                                           'FTSMode',
+          ('DIRAC.RequestManagementSystem.Agent.CleanReqDBAgent', {}),
+          ('DIRAC.RequestManagementSystem.Agent.RequestExecutingAgent', {'IgnoreOptions': ['FTSMode',
                                                                                            'OperationHandlers'],
                                                                          'SpecialMocks': {'gConfig': S_OK([])}}),
           ('DIRAC.ResourceStatusSystem.Agent.CacheFeederAgent', {}),

RequestManagementSystem/Agent/CleanReqDBAgent.py

@@ -4,19 +4,18 @@
 # Author: Krzysztof.Ciba@NOSPAMgmail.com
 # Date: 2013/05/17 08:31:26
 ########################################################################
-""" :mod: CleanReqDBAgent
+"""Cleaning the RequestDB from obsolete records and kicking assigned requests
 
-    =====================
+.. literalinclude:: ../ConfigTemplate.cfg
+  :start-after: ##BEGIN CleanReqDBAgent
+  :end-before: ##END
+  :dedent: 2
+  :caption: CleanReqDBAgent options
 
-    .. module: CleanReqDBAgent
+.. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com
 
-    :synopsis: cleaning RequestDB from obsolete records and kicking assigned requests
-
-    .. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com
-
-    cleaning ReqDB from obsolete records and kicking assigned requests
 """
-__RCSID__ = "$Id: $"
+__RCSID__ = "$Id$"
 # #
 # @file CleanReqDBAgent.py
 # @author Krzysztof.Ciba@NOSPAMgmail.com

RequestManagementSystem/Agent/RequestExecutingAgent.py

@@ -4,17 +4,21 @@
 # Date: 2013/03/12 15:36:47
 ########################################################################
 
-""" :mod: RequestExecutingAgent
+"""Agent processing the requests
 
-    ===========================
+ .. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com
 
-    .. module: RequestExecutingAgent
+See also the information about the :ref:`requestManagementSystem`.
 
-    :synopsis: request executing agent
+The following options can be set for the RequestExecutingAgent. The configuration also includes the
+``OperationHandlers`` available in DIRAC.
 
-    .. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com
+.. literalinclude:: ../ConfigTemplate.cfg
+  :start-after: ##BEGIN RequestExecutingAgent
+  :end-before: ##END
+  :dedent: 2
+  :caption: RequestExecutingAgent options
 
-    request processing agent
 """
 
 __RCSID__ = '$Id$'
@@ -75,17 +79,15 @@ class RequestExecutingAgent(AgentModule):
   # # requests/cycle
   __requestsPerCycle = 100
   # # minimal nb of subprocess running
-  __minProcess = 2
+  __minProcess = 20
   # # maximal nb of subprocess executed same time
-  __maxProcess = 4
+  __maxProcess = 20
   # # ProcessPool queue size
   __queueSize = 20
   # # file timeout
   __fileTimeout = 300
   # # operation timeout
   __operationTimeout = 300
-  # # ProcessTask default timeout in seconds
-  __taskTimeout = 900
   # # ProcessPool finalization timeout
   __poolTimeout = 900
   # # ProcessPool sleep time
@@ -104,15 +106,15 @@ def __init__(self, *args, **kwargs):
     self.log.info("Requests/cycle = %d" % self.__requestsPerCycle)
     self.__minProcess = self.am_getOption("MinProcess", self.__minProcess)
     self.log.info("ProcessPool min process = %d" % self.__minProcess)
-    self.__maxProcess = self.am_getOption("MaxProcess", 4)
+    self.__maxProcess = self.am_getOption("MaxProcess", self.__maxProcess)
     self.log.info("ProcessPool max process = %d" % self.__maxProcess)
     self.__queueSize = self.am_getOption("ProcessPoolQueueSize", self.__queueSize)
     self.log.info("ProcessPool queue size = %d" % self.__queueSize)
     self.__poolTimeout = int(self.am_getOption("ProcessPoolTimeout", self.__poolTimeout))
     self.log.info("ProcessPool timeout = %d seconds" % self.__poolTimeout)
     self.__poolSleep = int(self.am_getOption("ProcessPoolSleep", self.__poolSleep))
     self.log.info("ProcessPool sleep time = %d seconds" % self.__poolSleep)
-    self.__bulkRequest = self.am_getOption("BulkRequest", 0)
+    self.__bulkRequest = self.am_getOption("BulkRequest", self.__bulkRequest)
     self.log.info("Bulk request size = %d" % self.__bulkRequest)
 
     # # keep config path and agent name

RequestManagementSystem/ConfigTemplate.cfg

@@ -1,13 +1,17 @@
 Services
 {
+  ##BEGIN ReqManager
   ReqManager
   {
     Port = 9140
+    # If > 0, delay retry for this many minutes
+    ConstantRequestDelay = 0
     Authorization
     {
       Default = authenticated
     }
   }
+  ##END
   ReqProxy
   {
     Port = 9161
@@ -20,19 +24,23 @@ Services
 }
 Agents
 {
+  ##BEGIN RequestExecutingAgent
   RequestExecutingAgent
   {
     PollingTime = 60
-    RequestsPerCycle = 50
-    MinProcess = 1
-    MaxProcess = 8
-    ProcessPoolQueueSize = 25
+    # number of Requests to execute per cycle
+    RequestsPerCycle = 100
+    # minimum number of workers process in the ProcessPool
+    MinProcess = 20
+    # maximum number of workers process in the ProcessPool; recommended to set it to the same value as MinProcess
+    MaxProcess = 20
+    # queue depth of the ProcessPool
+    ProcessPoolQueueSize = 20
+    # timeout for the ProcessPool finalization
     ProcessPoolTimeout = 900
-    ProcessTaskTimeout = 900
-    ProcessPoolSleep = 4
-    #TimeOut = 300
-    #TimeOutPerFile = 300
-    MaxAttempts = 256
+    # sleep time before retrying to get a free slot in the ProcessPool
+    ProcessPoolSleep = 5
+    # If a positive integer n is given, we fetch n requests at once from the DB. Otherwise, one by one
     BulkRequest = 0
     OperationHandlers
     {
@@ -97,14 +105,22 @@ Agents
       }
     }
   }
+  ##END
+  ##BEGIN CleanReqDBAgent
   CleanReqDBAgent
   {
      PollingTime = 60
      ControlDirectory = control/RequestManagement/CleanReqDBAgent
-     DeleteGraceDays = 30
+     # How many days, until finished requests are deleted
+     DeleteGraceDays = 60
+     # How many requests are deleted per cycle
      DeleteLimit = 100
+     # If failed requests are deleted
      DeleteFailed = False
-     KickGraceHours = 2
-     KickLimit = 100
+     # How many hours a request can stay assigned
+     KickGraceHours = 1
+     # How many requests are kicked per cycle
+     KickLimit = 10000
   }
+  ##END
 }

RequestManagementSystem/Service/ReqManagerHandler.py

@@ -1,11 +1,15 @@
 #####################################################################
 # File: ReqManagerHandler.py
 ########################################################################
-"""
-:mod: ReqManagerHandler
+"""Implementation of the RequestDB service in the DISET framework
+
+
+.. literalinclude:: ../ConfigTemplate.cfg
+  :start-after: ##BEGIN ReqManager
+  :end-before: ##END
+  :dedent: 2
+  :caption: ReqManager options
 
-.. module: ReqManagerHandler
-  :synopsis: Implementation of the RequestDB service in the DISET framework
 
 """
 __RCSID__ = "$Id$"

docs/source/AdministratorGuide/Configuration/ConfReference/Systems/index.rst

@@ -5,14 +5,18 @@ Systems configuration
 
 Each DIRAC system has its corresponding section in the Configuration namespace.
 
+.. note:: The configuration options for services and agents are being moved to the :ref:`Code
+          Documentation <code_documentation>`. You can find the options for each service and agent
+          on the individual documentation page of the respective agent or service.
+
 .. toctree::
    :maxdepth: 1
 
    Accounting/index
    Configuration/index
    DataManagement/index
    WorkloadManagement/index
-   RequestManagement/index
+   /CodeDocumentation/RequestManagementSystem/RequestManagementSystem_Module
    Framework/index
    StorageManagement/index
    Transformation/index

docs/source/AdministratorGuide/Systems/RequestManagement/rmsComponents.rst

@@ -21,7 +21,7 @@ ReqManager
 
 This is the service in front of the DB. It has the following special configuration options:
 
-* `constantRequestDelay`: (default 0 minut) if not 0, this is the constant retry delay we add when putting a Request back to the DB
+* ``ConstantRequestDelay``: (default 0 minutes) if not 0, this is the constant retry delay we add when putting a Request back to the DB
 
 .. _RequestExecutinAgent:
 
@@ -37,29 +37,15 @@ At the end of the execution, if the Request comes from a job, we set the job to
 
 The RequestExecutingAgent is one of the few that can be duplicated. There are protections to make sure that a Request is only processed by one REA at the time.
 
-=====================
-Configuration options
-=====================
-
-On top of the standard agent options, the REA accepts the following configuration
-
-
-* `BulkRequest` (default 0): If a positive integer `n` is given, we fetch `n` requests at once from the DB. Otherwise, one by one
-* `MinProcess` (default 2): minimum number of workers process in the `ProcessPool`
-* `MaxProcess` (default 4): maximum number of workers process in the `ProcessPool`
-* `OperationHandlers`: There should be in this section one section per OperationHandler (see :ref:`rmsOpType`)
-* `ProcessPoolQueueSize` (default 20): queue depth of the `ProcessPool`
-* `ProcessPoolTimeout` (default 900 seconds): timeout for the `ProcessPool` finalization
-* `ProcessPoolSleep` (default 5 seconds): sleep time before retrying to get a free slot in the `ProcessPool`
-* `RequestsPerCycle` (default 100): number of Requests to execute per cycle
+Configuration options are described :mod:`here <DIRAC.RequestManagementSystem.Agent.RequestExecutingAgent>`.
 
 ==============
 Retry strategy
 ==============
 
 Operations are normally retried several times in case they fail. There is a delay between each execution, depending on the case:
 
-* If the option `constantRequestDelay` is set in the :ref:`ReqManager`, then we apply that one
+* If the option ``ConstantRequestDelay`` is set in the :ref:`ReqManager`, then we apply that one
 * If one of the StorageElement (source or target) is banned, then we wait 1 hour (except if the SE is always banned, then we fail the Operation)
 * Otherwise the delay increases following a logarithmic scale with the number of attempts
 
@@ -69,13 +55,9 @@ Operations are normally retried several times in case they fail. There is a dela
 CleanReqDBAgent
 ---------------
 
-This agent cleans the DB from old Requests in final state. Special configuration options are
+This agent cleans the DB from old Requests in final state.
 
-* `DeleteGraceDays`: (default 60)  Delay after which Requests are removed
-* `DeleteLimit`: (default 100)  Maximum number of Requests to remove per cycle
-* `DeleteFailed`: (default False)  Whether to delete also Failed request
-* `KickGraceHours`: (default 1)  After how long we should kick the Requests in `Assigned`
-* `KickLimit`: (default 10000)  Maximum number of requests kicked by cycle
+Configuration options are described :mod:`here <DIRAC.RequestManagementSystem.Agent.CleanReqDBAgent>`.
 
 .. _reqProxy: