From 845972f479ce59bdfc7a6d5b0362fdf1bf3b31a7 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Wed, 22 Apr 2020 15:58:48 +0200 Subject: [PATCH 1/5] Acc.NetworkAgent: add config doc --- AccountingSystem/Agent/NetworkAgent.py | 12 ++++++++++-- AccountingSystem/ConfigTemplate.cfg | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/AccountingSystem/Agent/NetworkAgent.py b/AccountingSystem/Agent/NetworkAgent.py index 5a77a459162..23d94aabbc1 100644 --- a/AccountingSystem/Agent/NetworkAgent.py +++ b/AccountingSystem/Agent/NetworkAgent.py @@ -1,5 +1,12 @@ """ Accounting agent to consume perfSONAR network metrics received via a message queue. + +.. literalinclude:: ../ConfigTemplate.cfg + :start-after: ##BEGIN NetworkAgent + :end-before: ##END + :dedent: 2 + :caption: NetworkAgent options + """ from datetime import datetime @@ -20,8 +27,9 @@ class NetworkAgent (AgentModule): Results are stored in the accounting database. """ - BUFFER_TIMEOUT = 3600 # default number of seconds after which network accounting - # objects are removed form the the temporary buffer + #: default number of seconds after which network accounting + #: objects are removed from the temporary buffer + BUFFER_TIMEOUT = 3600 def initialize(self): diff --git a/AccountingSystem/ConfigTemplate.cfg b/AccountingSystem/ConfigTemplate.cfg index 30e5cf12509..f3f8aabeee8 100644 --- a/AccountingSystem/ConfigTemplate.cfg +++ b/AccountingSystem/ConfigTemplate.cfg @@ -28,11 +28,16 @@ Services } Agents { + ##BEGIN NetworkAgent NetworkAgent { MaxCycles = 0 PollingTime = 60 + # URI of the MQ of the perfSONAR information MessageQueueURI = + # time how long objects are kept in the buffer if they cannot be written to the DB + BufferTimeout = 3600 } + ##END } From 15e129de2d5d24273324e92f63e32cf4054ac4be Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Wed, 22 Apr 2020 15:59:16 +0200 Subject: [PATCH 2/5] Acc.DataStoreHandler: add config doc --- AccountingSystem/ConfigTemplate.cfg | 4 ++++ AccountingSystem/Service/DataStoreHandler.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/AccountingSystem/ConfigTemplate.cfg b/AccountingSystem/ConfigTemplate.cfg index f3f8aabeee8..19fcd0191e8 100644 --- a/AccountingSystem/ConfigTemplate.cfg +++ b/AccountingSystem/ConfigTemplate.cfg @@ -1,8 +1,11 @@ Services { + ##BEGIN DataStore DataStore { Port = 9133 + # Run compaction, has to be True for Master, False for others + RunBucketing = True Authorization { Default = authenticated @@ -13,6 +16,7 @@ Services regenerateBuckets = ServiceAdministrator } } + ##END ReportGenerator { Port = 9134 diff --git a/AccountingSystem/Service/DataStoreHandler.py b/AccountingSystem/Service/DataStoreHandler.py index f99a4ec4a30..0de511ee9dc 100644 --- a/AccountingSystem/Service/DataStoreHandler.py +++ b/AccountingSystem/Service/DataStoreHandler.py @@ -1,6 +1,13 @@ """ DataStore is the service for inserting accounting reports (rows) in the Accounting DB This service CAN be duplicated iff the first is a "master" and all the others are slaves. + See the information about :ref`datastorehelpers`. + +.. literalinclude:: ../ConfigTemplate.cfg + :start-after: ##BEGIN DataStore + :end-before: ##END + :dedent: 2 + :caption: DataStore options """ From 422d039729cfd9a13662a3d8534eceebaf40c958 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Wed, 22 Apr 2020 15:59:33 +0200 Subject: [PATCH 3/5] Acc.ReportGeneratorHandler: add config doc --- AccountingSystem/ConfigTemplate.cfg | 4 ++++ AccountingSystem/Service/ReportGeneratorHandler.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/AccountingSystem/ConfigTemplate.cfg b/AccountingSystem/ConfigTemplate.cfg index 19fcd0191e8..e2f2f7d97fa 100644 --- a/AccountingSystem/ConfigTemplate.cfg +++ b/AccountingSystem/ConfigTemplate.cfg @@ -17,9 +17,12 @@ Services } } ##END + ##BEGIN ReportGenerator ReportGenerator { Port = 9134 + # folder relative to instance path, where data is stored + DataLocation = data/accountingGraphs Authorization { Default = authenticated @@ -29,6 +32,7 @@ Services } } } + ##END } Agents { diff --git a/AccountingSystem/Service/ReportGeneratorHandler.py b/AccountingSystem/Service/ReportGeneratorHandler.py index 1c9ba223f47..b8aa991d3cc 100644 --- a/AccountingSystem/Service/ReportGeneratorHandler.py +++ b/AccountingSystem/Service/ReportGeneratorHandler.py @@ -1,4 +1,10 @@ """ Module that holds the ReportGeneratorHandler class + +.. literalinclude:: ../ConfigTemplate.cfg + :start-after: ##BEGIN ReportGenerator + :end-before: ##END + :dedent: 2 + :caption: ReportGenerator options """ __RCSID__ = "$Id$" From 5c6647de8e99abe47f7ced1da0f6b8484e81c5f1 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Wed, 22 Apr 2020 16:03:52 +0200 Subject: [PATCH 4/5] AgentOptionTests: adapt NetworkAgent --- ConfigurationSystem/test/Test_agentOptions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ConfigurationSystem/test/Test_agentOptions.py b/ConfigurationSystem/test/Test_agentOptions.py index ccb42113e6d..bb4a26a0989 100644 --- a/ConfigurationSystem/test/Test_agentOptions.py +++ b/ConfigurationSystem/test/Test_agentOptions.py @@ -6,8 +6,7 @@ from DIRAC.tests.Utilities.assertingUtils import AgentOptionsTest from DIRAC import S_OK -AGENTS = [('DIRAC.AccountingSystem.Agent.NetworkAgent', {'IgnoreOptions': ['MaxCycles', 'MessageQueueURI', - 'BufferTimeout']}), +AGENTS = [('DIRAC.AccountingSystem.Agent.NetworkAgent', {'IgnoreOptions': ['MessageQueueURI']}), ('DIRAC.ConfigurationSystem.Agent.Bdii2CSAgent', {'IgnoreOptions': ['BannedCEs', 'BannedSEs', 'DryRun', 'AlternativeBDIIs', 'VO']}), ('DIRAC.ConfigurationSystem.Agent.GOCDB2CSAgent', {'IgnoreOptions': ['Cycles', 'DryRun']}), From 7c1d358bb459755e82f9aab0a9059307ddbdfcb5 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Wed, 22 Apr 2020 18:08:06 +0200 Subject: [PATCH 5/5] Docs.Accounting: remove obsolete pages from ConfReference --- .../Systems/Accounting/Databases/index.rst | 5 +-- .../Accounting/Services/DataStore/index.rst | 6 --- .../Services/ReportGenerator/index.rst | 6 --- .../Systems/Accounting/Services/index.rst | 38 ------------------- .../Systems/Accounting/index.rst | 3 +- 5 files changed, 2 insertions(+), 56 deletions(-) delete mode 100644 docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Services/DataStore/index.rst delete mode 100644 docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Services/ReportGenerator/index.rst delete mode 100644 docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Services/index.rst diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Databases/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Databases/index.rst index 8d23a365d65..d198eb479e0 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Databases/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Databases/index.rst @@ -12,9 +12,6 @@ Databases used by Accounting System. Note that each database is a separate subse +--------------------------------+----------------------------------------------+-----------------------+ | */Host* | Database host server where the DB is located | Host = db01.in2p3.fr | +--------------------------------+----------------------------------------------+-----------------------+ -| */MaxQueueSize* | Maximum number of simultaneous queries to | MaxQueueSize = 10 | -| | the DB per instance of the client | | -+--------------------------------+----------------------------------------------+-----------------------+ The databases associated with Accounting System are: -- AccountingDB \ No newline at end of file +- AccountingDB diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Services/DataStore/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Services/DataStore/index.rst deleted file mode 100644 index c9b90ab4823..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Services/DataStore/index.rst +++ /dev/null @@ -1,6 +0,0 @@ -Systems / AccountingManagement / / Service / DataStore - Sub-subsection -===================================================================================== - -DataStore service is in charge of receiving Accounting data. - -No special options must be configured to use this service. \ No newline at end of file diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Services/ReportGenerator/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Services/ReportGenerator/index.rst deleted file mode 100644 index a0705da51fd..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Services/ReportGenerator/index.rst +++ /dev/null @@ -1,6 +0,0 @@ -Systems / AccountingManagement / / Service / ReportGenerator - Sub-subsection -======================================================================================== - -ReportGenerator service is in charge of producing accounting reports (plots or CSV files). - -No special options must be configured. \ No newline at end of file diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Services/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Services/index.rst deleted file mode 100644 index 67ed0ac8ea2..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/Services/index.rst +++ /dev/null @@ -1,38 +0,0 @@ -Systems / Accounting / / Service - Sub-subsection -=============================================================== - -All the services have common options to be configured for each one. Those options are -presented in the following table: - -+-------------------------+----------------------------------------------+-----------------------------+ -| **Name** | **Description** | **Example** | -+-------------------------+----------------------------------------------+-----------------------------+ -| *LogLevel* | Level of log verbosity | LogLevel = INFO | -+-------------------------+----------------------------------------------+-----------------------------+ -| *LogBackends* | Log backends | LogBackends = stdout | -| | | LogBackends += server | -+-------------------------+----------------------------------------------+-----------------------------+ -| *MaskRequestParameters* | Request to mask the values, possible values: | MaskRequestParameters = yes | -| | yes or no | | -+-------------------------+----------------------------------------------+-----------------------------+ -| *MaxThreads* | Maximum number of threads used in parallel | MaxThreads = 50 | -| | for the server | | -+-------------------------+----------------------------------------------+-----------------------------+ -| *Port* | Port useb by DIRAC service | Port = 9140 | -+-------------------------+----------------------------------------------+-----------------------------+ -| *Protocol* | Protocol used to comunicate with the service | Protocol = dips | -+-------------------------+----------------------------------------------+-----------------------------+ -| *Authorization* | Subsection used to define which kind of | Authorization | -| | Authorization is required to talk with the | | -| | service | | -+-------------------------+----------------------------------------------+-----------------------------+ -| *Authorization/Default* | Define to who is required the authorization | Default = all | -+-------------------------+----------------------------------------------+-----------------------------+ - -Accounting system related services are: - -.. toctree:: - :maxdepth: 2 - - DataStore/index - ReportGenerator/index \ No newline at end of file diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/index.rst index c70736b34d3..27ec7010107 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/index.rst @@ -7,5 +7,4 @@ In this subsection are described the databases, services and URLs related with A :maxdepth: 2 Databases/index - Services/index - URLs/index \ No newline at end of file + URLs/index