diff --git a/ConfigurationSystem/Agent/Bdii2CSAgent.py b/ConfigurationSystem/Agent/Bdii2CSAgent.py index d9c880c1dd3..78c6c942b84 100644 --- a/ConfigurationSystem/Agent/Bdii2CSAgent.py +++ b/ConfigurationSystem/Agent/Bdii2CSAgent.py @@ -1,8 +1,8 @@ """ The Bdii2CSAgent performs checking BDII for availability of CE and SE - resources for a given or any configured VO. It detects resources not yet - present in the CS and notifies the administrators. - For the CEs and SEs already present in the CS, the agent is updating - if necessary settings which were changed in the BDII recently +resources for a given or any configured VO. It detects resources not yet +present in the CS and notifies the administrators. +For the CEs and SEs already present in the CS, the agent is updating +if necessary settings which were changed in the BDII recently The following options can be set for the Bdii2CSAgent. diff --git a/ConfigurationSystem/Agent/VOMS2CSAgent.py b/ConfigurationSystem/Agent/VOMS2CSAgent.py index 66a8c61cf13..ac13f4ec262 100644 --- a/ConfigurationSystem/Agent/VOMS2CSAgent.py +++ b/ConfigurationSystem/Agent/VOMS2CSAgent.py @@ -1,8 +1,17 @@ """ -VOMS2CSAgent performs the following operations: +The VOMS2CSAgent extracts VO user information from VOMS servers. -- Adds new users for the given VO taking into account the VO VOMS information -- Updates the data in the CS for existing users including DIRAC group membership +It performs the following operations: + +- Extracts user info from the VOMS server using its REST interface +- Finds user DN's not yet registered in the DIRAC Registry +- For each new DN it constructs a DIRAC login name by a best guess or using the nickname VOMS attribute +- Registers new users to the DIRAC Registry including group membership +- Updates information for already registered users, including group membership +- Sends report for performed operation to the VO administrator + +The agent is performing its operations with credentials of the VO administrator as defined +in the ``/Registry/VO/`` configuration section. See also :ref:`registry_vo`. The following options can be set for the VOMS2CSAgent. @@ -12,6 +21,9 @@ :dedent: 2 :caption: VOMS2CSAgent options +.. note:: The options *AutoAddUsers*, *AutoModifyUsers*, *AutoDeleteUsers* can be overridden by the + corresponding options defined in the ``/Registry/VO/`` configuration section. + """ from __future__ import absolute_import diff --git a/ConfigurationSystem/ConfigTemplate.cfg b/ConfigurationSystem/ConfigTemplate.cfg index 356b39f790d..ab87fa92cd0 100644 --- a/ConfigurationSystem/ConfigTemplate.cfg +++ b/ConfigurationSystem/ConfigTemplate.cfg @@ -1,18 +1,25 @@ Services { + ##BEGIN Server Server { HandlerPath = DIRAC/ConfigurationSystem/Service/ConfigurationHandler.py Port = 9135 + # Subsection to configure authorization over the service Authorization { + # Default authorization Default = authenticated + #Define who can commit new configuration commitNewData = CSAdministrator + # Define who can roll back the configuration to a previous version rollbackToVersion = CSAdministrator + # Define who can get version contents getVersionContents = ServiceAdministrator getVersionContents += CSAdministrator } } + ##END } Agents { @@ -23,8 +30,11 @@ Agents PollingTime = 14400 BannedCEs = BannedSEs = + # Only treat these sites SelectedSites = + # Process Computing Elements ProcessCEs = yes + # Process Storage Elements ProcessSEs = no # Mail Notifications options MailTo = @@ -32,6 +42,7 @@ Agents VirtualOrganization = # Flag to turn to False if you want this agent to write in the CS DryRun = True + # Host to query, must include port Host = lcg-bdii.cern.ch:2170 # URLs for Glue2, if filled and GLUE2Only is False, the agent will look under theses URLs for Glue2 information GLUE2URLs = @@ -45,12 +56,19 @@ Agents #Time between cycles in seconds PollingTime = 14400 MailFrom = noreply@dirac.system + # If users will be added automatically AutoAddUsers = True + # If users will be modified automatically AutoModifyUsers = True + # Users no longer registered in VOMS are automatically deleted from DIRAC AutoDeleteUsers = True + # If suspended status is lifted, if lifted in VOMS AutoLiftSuspendedStatus = True + # Detailed report on users per group send to the VO administrator DetailedReport = True + # Automatically create user home directory in the File Catalog MakeHomeDirectory = False + # List of VO names VO = Any # Flag to turn to False if you want this agent to write in the CS (more granularity within other options) DryRun = True @@ -63,6 +81,8 @@ Agents PollingTime = 14400 # Flag to turn to False if you want this agent to write in the CS DryRun = True + # if False, disable the updating of perfSONAR endpoints from GOCDB + UpdatePerfSONARS = True } ##END } diff --git a/ConfigurationSystem/Service/ConfigurationHandler.py b/ConfigurationSystem/Service/ConfigurationHandler.py index 4e31258ec55..b061bd10e91 100755 --- a/ConfigurationSystem/Service/ConfigurationHandler.py +++ b/ConfigurationSystem/Service/ConfigurationHandler.py @@ -1,4 +1,14 @@ """ The CS! (Configuration Service) + + +The following options can be set for the Configuration Service. + +.. literalinclude:: ../ConfigTemplate.cfg + :start-after: ##BEGIN Server + :end-before: ##END + :dedent: 2 + :caption: Service options + """ __RCSID__ = "$Id$" diff --git a/ConfigurationSystem/test/Test_agentOptions.py b/ConfigurationSystem/test/Test_agentOptions.py index ccb42113e6d..197a123da67 100644 --- a/ConfigurationSystem/test/Test_agentOptions.py +++ b/ConfigurationSystem/test/Test_agentOptions.py @@ -10,7 +10,8 @@ 'BufferTimeout']}), ('DIRAC.ConfigurationSystem.Agent.Bdii2CSAgent', {'IgnoreOptions': ['BannedCEs', 'BannedSEs', 'DryRun', 'AlternativeBDIIs', 'VO']}), - ('DIRAC.ConfigurationSystem.Agent.GOCDB2CSAgent', {'IgnoreOptions': ['Cycles', 'DryRun']}), + ('DIRAC.ConfigurationSystem.Agent.GOCDB2CSAgent', {'IgnoreOptions': ['Cycles', 'DryRun', + 'UpdatePerfSONARS']}), ('DIRAC.ConfigurationSystem.Agent.VOMS2CSAgent', {'IgnoreOptions': ['VO']}), ('DIRAC.DataManagementSystem.Agent.FTS3Agent', {}), ('DIRAC.FrameworkSystem.Agent.CAUpdateAgent', {}), diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Registry/VO/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Registry/VO/index.rst index 64439fe6521..0899b44bbcc 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Registry/VO/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Registry/VO/index.rst @@ -1,3 +1,5 @@ +.. _registry_vo: + Registry / VO - Subsections ============================== diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Agents/Bdii2CSAgent/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Agents/Bdii2CSAgent/index.rst deleted file mode 100644 index 814ab7a1496..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Agents/Bdii2CSAgent/index.rst +++ /dev/null @@ -1,35 +0,0 @@ -Systems / Configuration / / Agents /Bdii2CSAgent - Sub-subsection -============================================================================ - -Bdii2CSAgent is the agent in charge of updating sites parameters configuration for a specific VO: - - - Queries BDII for Computing Elements (CEs) information and update the CS. - - Queries BDII for Storage Elements (SEs) information and update the CS. - -The attributes of this agent are shown in the table below: - -+-----------------------+-------------------------------------------------+-------------------------------------------------+ -| **Name** | **Description** | **Example** | -+-----------------------+-------------------------------------------------+-------------------------------------------------+ -| *AlternativeBDIIs* | List of alternatives BDIIs | AlternativeBDIIs = bdii01.in2p3.fr | -+-----------------------+-------------------------------------------------+-------------------------------------------------+ -| *GLUE2URLs* | URLs to use for GLUE2 in addition | top-bdii.cern.ch:2170 | -+-----------------------+-------------------------------------------------+-------------------------------------------------+ -| *GLUE2Only* | Only search GLUE2, not GLUE1. If true only the | False | -| | URL under *Host* is queried, not those under | | -| | *GLUE2URLs* | | -+-----------------------+-------------------------------------------------+-------------------------------------------------+ -| *Host* | Host to query, must include port | lcg-bdii.cern.ch:2170 | -+-----------------------+-------------------------------------------------+-------------------------------------------------+ -| *MailTo* | E-mail of the person in charge of | MailTo = hamar@cppm.in2p3.fr | -| | update the Sites configuration | | -+-----------------------+-------------------------------------------------+-------------------------------------------------+ -| *MailFrom* | E-mail address used to send the | MailFrom = dirac@mardirac.in2p3.fr | -| | information to be updated | | -+-----------------------+-------------------------------------------------+-------------------------------------------------+ -| *ProcessCEs* | Process Computing Elements | ProcessCEs = True | -+-----------------------+-------------------------------------------------+-------------------------------------------------+ -| *ProcessSEs* | Process Storage Elements | ProcessSEs = True | -+-----------------------+-------------------------------------------------+-------------------------------------------------+ -| *VirtualOrganization* | Name of the VO | VirtualOrganization = vo.formation.idgrilles.fr | -+-----------------------+-------------------------------------------------+-------------------------------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Agents/GOCDB2CSAgent/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Agents/GOCDB2CSAgent/index.rst deleted file mode 100644 index 3d7c2fe524c..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Agents/GOCDB2CSAgent/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -Systems / Configuration / / Agents /GOCDB2CSAgent - Sub-subsection -================================================================================ - -Synchronizes information between GOCDB and DIRAC configuration System (CS) - -The attributes of this agent are showed in the table below: - -+-----------------------+--------------------------------------+----------------------------------------------+ -| **Name** | **Description** | **Example** | -+-----------------------+--------------------------------------+----------------------------------------------+ -| *UpdatePerfSONARS* | Sync perfSONAR end points to CS | UpdatePerfSONARS = True | -+-----------------------+--------------------------------------+----------------------------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Agents/VOMS2CSAgent/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Agents/VOMS2CSAgent/index.rst deleted file mode 100644 index 27345401a59..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Agents/VOMS2CSAgent/index.rst +++ /dev/null @@ -1,46 +0,0 @@ -Systems / Configuration / / Agents /VOMS2CSAgent - Sub-subsection -================================================================================ - -VOMS2CSAgent queries VOMS servers and updates the users and groups as defined in the Configuration Registry -for the given VO and groups in this VO. It performs the following operations: - -* Extracts user info from the VOMS server using its REST interface -* Finds user DN's not yet registered in the DIRAC Registry -* For each new DN it constructs a DIRAC login name by a best guess or using the nickname VOMS attribute -* Registers new users to the DIRAC Registry including group membership -* Updates information for already registered users -* Sends report for performed operation to the VO administrator - -The agent is performing its operations with credentials of the VO administrator as defined -in the /Registry/VO/ configuration section. - -The configuration options of this agent are shown in the table below: - -+-----------------------+--------------------------------------+----------------------------------------------+ -| **Name** | **Description** | **Example** | -+-----------------------+--------------------------------------+----------------------------------------------+ -| *VO* | List of VO names | VO = biomed, eiscat.se, compchem | -+-----------------------+--------------------------------------+----------------------------------------------+ -| *MailTo* | E-mail of the person in charge of | MailTo = hamar@cppm.in2p3.fr | -| | update the Sites configuration | | -+-----------------------+--------------------------------------+----------------------------------------------+ -| *MailFrom* | E-mail address used to send the | MailFrom = dirac@mardirac.in2p3.fr | -| | information to be updated | | -+-----------------------+--------------------------------------+----------------------------------------------+ -| *AutoAddUsers* | If users will be added automatically | AutoAddUsers = True | -+-----------------------+--------------------------------------+----------------------------------------------+ -| *AutoModifyUsers* | If users will be modified | | -| | automatically | AutoModifyUsers = True | -+-----------------------+--------------------------------------+----------------------------------------------+ -| *AutoDeleteUsers* | Users no more registered in VOMS are | | -| | automatically deleted from DIRAC | AutoDeleteUsers = False | -+-----------------------+--------------------------------------+----------------------------------------------+ -| *DetailedReport* | Detailed report on users per group | | -| | sent to the VO administrator | DetailedReport = True | -+-----------------------+--------------------------------------+----------------------------------------------+ -| *MakeHomeDirectory* | Automatically create user home | | -| | directory in the File Catalog | MakeHomeDirectory = False | -+-----------------------+--------------------------------------+----------------------------------------------+ - -Remark: options *AutoAddUsers*, *AutoModifyUsers*, *AutoDeleteUsers* can be overridden by the corresponding -options defined in the /Registry/VO/ configuration section. \ No newline at end of file diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Agents/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Agents/index.rst deleted file mode 100644 index 989564caae2..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Agents/index.rst +++ /dev/null @@ -1,51 +0,0 @@ -Systems / Configuration / / Agents - Sub-subsection -================================================================ - - In this subsection each agent is described. - -+----------+----------------------------------+-------------+ -| **Name** | **Description** | **Example** | -+----------+----------------------------------+-------------+ -| *Agent* | Subsection named as the agent is | CE2CSAgent | -| | called. | | -+----------+----------------------------------+-------------+ - -Common options for all the agents: - -+---------------------+---------------------------------------+------------------------------+ -| **Name** | **Description** | **Example** | -+---------------------+---------------------------------------+------------------------------+ -| *LogLevel* | Log Level associated to the agent | LogLevel = DEBUG | -+---------------------+---------------------------------------+------------------------------+ -| *LogBackends* | | LogBackends = stdout, server | -+---------------------+---------------------------------------+------------------------------+ -| *MaxCycles* | Maximum number of cycles made for | MaxCycles = 500 | -| | Agent | | -+---------------------+---------------------------------------+------------------------------+ -| *MonitoringEnabled* | Indicates if the monitoring of agent | MonitoringEnabled = True | -| | is enabled. Boolean values | | -+---------------------+---------------------------------------+------------------------------+ -| *PollingTime* | Each many time a new cycle must start | PollingTime = 2600 | -| | expresed in seconds | | -+---------------------+---------------------------------------+------------------------------+ -| *Status* | Agent Status, possible values Active | Status = Active | -| | or Inactive | | -+---------------------+---------------------------------------+------------------------------+ -| *DryRun* | If True, the agent won't change | DryRun = False | -| | the CS | | -+---------------------+---------------------------------------+------------------------------+ -| *WatchdogTime* | If > 0 will kill the agent if the | | WatchdogTime = 3600 | -| | cycle exceeds WatchdogTime in seconds | | (default is 0) | -| | to force a restart of the agent | | -+---------------------+---------------------------------------+------------------------------+ - - - -Agents associated with Configuration System: - -.. toctree:: - :maxdepth: 2 - - Bdii2CSAgent/index - VOMS2CSAgent/index - GOCDB2CSAgent/index diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Services/Server/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Services/Server/index.rst deleted file mode 100644 index a9e34ee9ed8..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Services/Server/index.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _ConfigurationServer: - -Systems / Configuration / / Service / Server - Sub-subsection -======================================================================== - -In this subsection the Server service is configured. The attributes are showed in the following table: - -+------------------------------------+--------------------------------------------+-------------------------------------------------------------------------+ -| **Name** | **Description** | **Example** | -+------------------------------------+--------------------------------------------+-------------------------------------------------------------------------+ -| *HandlerPath* | Relative path directory where the | HandlerPath = DIRAC/ConfigurationSystem/Service/ConfigurationHandler.py | -| | service is located | | -+------------------------------------+--------------------------------------------+-------------------------------------------------------------------------+ -| *Port* | Port where the service is responding | Port = 9135 | -+------------------------------------+--------------------------------------------+-------------------------------------------------------------------------+ -| *Authorization* | Subsection to configure authorization over | Authorization | -| | the service | | -+------------------------------------+--------------------------------------------+-------------------------------------------------------------------------+ -| *Authorization/Default* | Default authorization | Default = all | -+------------------------------------+--------------------------------------------+-------------------------------------------------------------------------+ -| *Authorization/commitNewData* | Define who can commit new configuration | commitNewData = CSAdministrator | -+------------------------------------+--------------------------------------------+-------------------------------------------------------------------------+ -| *Authorization/getVersionContents* | Define who can get version contents | getVersionContents = CSAdministrator | -+------------------------------------+--------------------------------------------+-------------------------------------------------------------------------+ -| *Authorization/rollBackToVersion* | Define who can roll back the configuration | rollBackToVersion = ServiceAdministrator | -| | to a previous version | rollBackToVersion += CSAdministrator | -+------------------------------------+--------------------------------------------+-------------------------------------------------------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Services/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Services/index.rst deleted file mode 100644 index 81df0e504e0..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/Services/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -Systems / Configuration / / Service - Sub-subsection -================================================================= - -In this subsection all the services of Configuration system are described. - -.. toctree:: - :maxdepth: 2 - - Server/index \ No newline at end of file diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/index.rst index 6501f258c53..c5dacadbd50 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/index.rst @@ -6,6 +6,4 @@ In this subsection are described the databases, services and URLs related with A .. toctree:: :maxdepth: 2 - Services/index - Agents/index - URLs/index \ No newline at end of file + URLs/index