From b89ca28624d8168d96aac1eb0b3027e65749839d Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 14 Jan 2021 10:25:50 +0100 Subject: [PATCH 01/27] Modify __init__.py to avoid conflicts with rel-v7r2 --- src/DIRAC/__init__.py | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/src/DIRAC/__init__.py b/src/DIRAC/__init__.py index 134eafb5dd0..c624e2c8365 100755 --- a/src/DIRAC/__init__.py +++ b/src/DIRAC/__init__.py @@ -92,36 +92,18 @@ # Define Version, use an unusual structure to minimise conflicts with rel-v7r2 pythonVersion = pyPlatform.python_version_tuple() if pythonVersion[0] == "3": - version = "7.2" + pass else: majorVersion = 7 minorVersion = 2 patchLevel = 0 preVersion = 28 - # Define version only for python 2, When using python 3 the version definition will change - # as needed by setup tools - version = "v%sr%s" % (majorVersion, minorVersion) - buildVersion = "v%dr%d" % (majorVersion, minorVersion) - if patchLevel: - version = "%sp%s" % (version, patchLevel) - buildVersion = "%s build %s" % (buildVersion, patchLevel) - if preVersion: - version = "%s-pre%s" % (version, preVersion) - buildVersion = "%s pre %s" % (buildVersion, preVersion) - -# Check of python version - -__pythonMajorVersion = ("2", "3") -__pythonMinorVersion = ("7", "8") -if str(pythonVersion[0]) not in __pythonMajorVersion or str(pythonVersion[1]) not in __pythonMinorVersion: - print("Python Version %s not supported by DIRAC" % pyPlatform.python_version()) - print("Supported versions are: ") - for major in __pythonMajorVersion: - for minor in __pythonMinorVersion: - print("%s.%s.x" % (major, minor)) - - sys.exit(1) +version = "v%sr%s" % (majorVersion, minorVersion) +if patchLevel: + version = "%sp%s" % (version, patchLevel) +if preVersion: + version = "%s-pre%s" % (version, preVersion) errorMail = "dirac.alarms@gmail.com" alarmMail = "dirac.alarms@gmail.com" From 0bd32880c2666eacb062d7466d662782b7b5a87f Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 10 Dec 2020 08:43:18 +0100 Subject: [PATCH 02/27] Fix setup.py installation --- pyproject.toml | 5 ++++ setup.cfg | 80 +++++++++++++++++++++++++++++++++++++++++++++++++- setup.py | 43 ++------------------------- 3 files changed, 87 insertions(+), 41 deletions(-) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..1cd983f7a52 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[build-system] +requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] diff --git a/setup.cfg b/setup.cfg index 7b75664a44d..ac6d74ac4fd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,82 @@ -# This file should be used as the pep8/pycodestyle and autopep8 config +############################################################################### +# Setuptools +############################################################################### + +[metadata] +name = DIRAC +description = DIRAC is an interware, meaning a software framework for distributed computing. +long_description = file: README.rst +license = GPL-3.0-only +classifiers = + License :: OSI Approved :: GNU General Public License v3 (GPLv3) + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + +[options] +python_requires = >=3.8 +package_dir= + =src +packages = find: +install_requires = + certifi + diraccfg + future + gfal2-python + M2Crypto + pexpect + psutil + pyasn1 + pyasn1-modules + python-dateutil + requests + six + subprocess32 + +[options.packages.find] +where=src + +[options.extras_require] +server = + # arc + boto3 + botocore + CMRESHandler + elasticsearch + elasticsearch_dsl + fts3-rest + GitPython + ldap3 + matplotlib + mysqlclient + numpy + pillow + pyparsing + python-irodsclient + python-json-logger + pytz + sqlalchemy + stomp.py + suds-jurko + tornado + tornado-m2crypto +testing = + flaky + hypothesis + mock + parameterized + pytest + +[options.entry_points] +# console_scripts = +# dirac-proxy-info = DIRAC.scripts:proxy_info +# dirac-dms-get-file = DIRAC.scripts:dms_get_file +# dirac-show-extensions = DIRAC.scripts:show_extensions + + +############################################################################### +# Linting +############################################################################### # pep8 binary will still look into this section [pep8] diff --git a/setup.py b/setup.py index 577c0c0181a..dcded7c1fbf 100755 --- a/setup.py +++ b/setup.py @@ -1,45 +1,8 @@ -""" Basic setuptools script for DIRAC. - Does not contain any dependency -""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os -import glob -# Actual setuptools -from setuptools import setup, find_packages +from setuptools import setup -# Find the base dir where the setup.py lies -base_dir = os.path.abspath(os.path.dirname(__file__)) - -# Take all the packages but the scripts and tests -allPackages = find_packages(where=base_dir, exclude=["*test*", "*scripts*", "*docs*"]) - -# Because we want to have a 'DIRAC' base module and that the setup.py -# is lying inside it, we need to define a mapping -# < module name : directory > -# e.g. DIRAC.DataManagementSystem is base_dir/DataManagementSystem - -package_dir = dict(("DIRAC.%s" % p, os.path.join(base_dir, p.replace('.', '/'))) for p in allPackages) - -# We also rename the packages so that they contain DIRAC -allPackages = ['DIRAC.%s' % p for p in allPackages] - -# Artificially create the 'DIRAC' package -# at the root -allPackages.insert(0, 'DIRAC') -package_dir['DIRAC'] = base_dir - -# The scripts to be distributed -scripts = glob.glob('%s/*/scripts/*.py' % base_dir) - -setup( - name="DIRAC", - version="7.2.pre28", - url="https://github.com/DIRACGRID/DIRAC", - license="GPLv3", - package_dir=package_dir, - packages=allPackages, - scripts=scripts, -) +# This is required to allow editable pip installs while using the declarative configuration (setup.cfg) +setup() From c161106aad5f18d0a22a6ac49771552316e09c15 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 10 Dec 2020 08:43:32 +0100 Subject: [PATCH 03/27] PY3: Modernise various imports --- .../Client/CmdDirCompletion/DirectoryCompletion.py | 2 +- .../Service/StorageElementProxyHandler.py | 4 ++-- .../DataManagementSystem/private/HttpStorageAccessHandler.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py b/src/DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py index 7ec8396c465..dd1c9f038b9 100644 --- a/src/DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py +++ b/src/DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py @@ -84,7 +84,7 @@ def get_filename(self, path, dirname): if __name__ == "__main__": - from AbstractFileSystem import UnixLikeFileSystem # pylint: disable=import-error + from DIRAC.DataManagementSystem.Client.CmdDirCompletion.AbstractFileSystem import UnixLikeFileSystem ulfs = UnixLikeFileSystem() dc = DirectoryCompletion(ulfs) diff --git a/src/DIRAC/DataManagementSystem/Service/StorageElementProxyHandler.py b/src/DIRAC/DataManagementSystem/Service/StorageElementProxyHandler.py index 0527be85de6..2fbfb556325 100644 --- a/src/DIRAC/DataManagementSystem/Service/StorageElementProxyHandler.py +++ b/src/DIRAC/DataManagementSystem/Service/StorageElementProxyHandler.py @@ -13,7 +13,7 @@ import os import shutil -import SocketServer +from six.moves import socketserver import threading import socket import random @@ -75,7 +75,7 @@ def initializeStorageElementProxyHandler(serviceInfo): return S_OK() -class ThreadedSocketServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): +class ThreadedSocketServer(socketserver.ThreadingMixIn, socketserver.TCPServer): """ bag dummy class to hold ThreadingMixIn and TCPServer """ pass diff --git a/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py b/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py index 850d10607b0..9bcf1a06be4 100644 --- a/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py +++ b/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py @@ -14,10 +14,11 @@ __RCS__ = "$Id$" import os -import BaseHTTPServer import shutil import random +from six.moves import BaseHTTPServer + from DIRAC.Core.Utilities.DictCache import DictCache From 009c7a9b1b8ed9d97d70082e57210e2fb747ddac Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 10 Dec 2020 13:19:02 +0100 Subject: [PATCH 04/27] PY3: Rename some scripts to have _ instead of - to ensure they're valid Python modules --- src/DIRAC/AccountingSystem/scripts/__init__.py | 0 src/DIRAC/ConfigurationSystem/scripts/__init__.py | 0 src/DIRAC/Core/scripts/__init__.py | 0 src/DIRAC/Core/scripts/{dirac-info.py => dirac_info.py} | 0 src/DIRAC/DataManagementSystem/scripts/__init__.py | 0 src/DIRAC/FrameworkSystem/scripts/__init__.py | 0 .../scripts/{dirac-proxy-info.py => dirac_proxy_info.py} | 0 .../scripts/{dirac-proxy-init.py => dirac_proxy_init.py} | 0 src/DIRAC/ProductionSystem/scripts/__init__.py | 0 src/DIRAC/RequestManagementSystem/scripts/__init__.py | 0 src/DIRAC/ResourceStatusSystem/scripts/__init__.py | 0 src/DIRAC/Resources/scripts/__init__.py | 0 src/DIRAC/WorkloadManagementSystem/scripts/__init__.py | 0 13 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/DIRAC/AccountingSystem/scripts/__init__.py create mode 100644 src/DIRAC/ConfigurationSystem/scripts/__init__.py create mode 100644 src/DIRAC/Core/scripts/__init__.py rename src/DIRAC/Core/scripts/{dirac-info.py => dirac_info.py} (100%) create mode 100644 src/DIRAC/DataManagementSystem/scripts/__init__.py create mode 100644 src/DIRAC/FrameworkSystem/scripts/__init__.py rename src/DIRAC/FrameworkSystem/scripts/{dirac-proxy-info.py => dirac_proxy_info.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-proxy-init.py => dirac_proxy_init.py} (100%) create mode 100644 src/DIRAC/ProductionSystem/scripts/__init__.py create mode 100644 src/DIRAC/RequestManagementSystem/scripts/__init__.py create mode 100644 src/DIRAC/ResourceStatusSystem/scripts/__init__.py create mode 100644 src/DIRAC/Resources/scripts/__init__.py create mode 100644 src/DIRAC/WorkloadManagementSystem/scripts/__init__.py diff --git a/src/DIRAC/AccountingSystem/scripts/__init__.py b/src/DIRAC/AccountingSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/ConfigurationSystem/scripts/__init__.py b/src/DIRAC/ConfigurationSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/Core/scripts/__init__.py b/src/DIRAC/Core/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/Core/scripts/dirac-info.py b/src/DIRAC/Core/scripts/dirac_info.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-info.py rename to src/DIRAC/Core/scripts/dirac_info.py diff --git a/src/DIRAC/DataManagementSystem/scripts/__init__.py b/src/DIRAC/DataManagementSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/FrameworkSystem/scripts/__init__.py b/src/DIRAC/FrameworkSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-info.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-proxy-info.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-init.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-proxy-init.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py diff --git a/src/DIRAC/ProductionSystem/scripts/__init__.py b/src/DIRAC/ProductionSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/RequestManagementSystem/scripts/__init__.py b/src/DIRAC/RequestManagementSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/ResourceStatusSystem/scripts/__init__.py b/src/DIRAC/ResourceStatusSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/Resources/scripts/__init__.py b/src/DIRAC/Resources/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/__init__.py b/src/DIRAC/WorkloadManagementSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 6f21685152b6ee4a8988981e1580e7297b599875 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 10 Dec 2020 15:42:27 +0100 Subject: [PATCH 05/27] PY3: Add DIRACScript object and DIRAC's extension_metadata entrypoint --- setup.cfg | 6 +- src/DIRAC/Core/Utilities/DIRACScript.py | 126 ++++++++++++++++++++++++ src/DIRAC/__init__.py | 11 +++ 3 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 src/DIRAC/Core/Utilities/DIRACScript.py diff --git a/setup.cfg b/setup.cfg index ac6d74ac4fd..f0503c70003 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,10 +68,8 @@ testing = pytest [options.entry_points] -# console_scripts = -# dirac-proxy-info = DIRAC.scripts:proxy_info -# dirac-dms-get-file = DIRAC.scripts:dms_get_file -# dirac-show-extensions = DIRAC.scripts:show_extensions +dirac = + metadata = DIRAC:extension_metadata ############################################################################### diff --git a/src/DIRAC/Core/Utilities/DIRACScript.py b/src/DIRAC/Core/Utilities/DIRACScript.py new file mode 100644 index 00000000000..58b9da37d6f --- /dev/null +++ b/src/DIRAC/Core/Utilities/DIRACScript.py @@ -0,0 +1,126 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from collections import defaultdict +import functools + +import six + +from DIRAC import gLogger + + +class DIRACScript(object): + """Decorator for providing command line executables + + All console-scripts entrypoints in DIRAC and downstream extensions should be + wrapped in this decorator to allow extensions to override any entry_point. + """ + def __init__(self): + """ c'tor + """ + pass + + @property + def func(self): + if not hasattr(self, "_func"): + raise NotImplementedError("Something is very wrong") + return self._func + + @func.setter + def func(self, value): + if hasattr(self, "_func"): + raise NotImplementedError("Something is very wrong") + self._func = value + + def __call__(self, func=None): + """Set the wrapped function or call the script + + This function is either called with a decorator or directly to call the + underlying function. When running with Python 2 the raw function will always + be called however in Python 3 the priorities will be applied from the + dirac.extension_metadata entry_point. + """ + # If func is provided then the decorator is being applied to a function + if func is not None: + self.func = func + return functools.wraps(func)(self) + + # Setuptools based installations aren't supported with Python 2 + if six.PY2: + return self.func() # pylint: disable=not-callable + + # This is only available in Python 3.8+ so it has to be here for now + from importlib import metadata # pylint: disable=no-name-in-module + + # Iterate through all known entry_points looking for DIRACScripts + matches = defaultdict(list) + function_name = None + for entrypoint in metadata.entry_points()['console_scripts']: + if not entrypoint.name.startswith("dirac-"): + continue + entrypointFunc = entrypoint.load() + if not isinstance(entrypointFunc, DIRACScript): + raise ImportError( + "Invalid dirac- console_scripts entry_point: " + repr(entrypoint) + "\n" + + "All dirac- console_scripts should be wrapped in the DiracScript " + + "decorator to ensure extension overlays are applied correctly." + ) + matches[entrypoint.name].append(entrypoint) + # If the function is self then we've found the currently called function + if entrypointFunc is self: + function_name = entrypoint.name + + if function_name is None: + # TODO: This should an error once the integration tests modified to use pip install + return self.func() # pylint: disable=not-callable + # raise NotImplementedError("Something is very wrong") + + # Call the entry_point from the extension with the highest priority + rankedExtensions = _extensionsByPriority() + entrypoint = max( + matches[function_name], + key=lambda e: rankedExtensions.index(_entrypointToExtension(e)), + ) + + return entrypoint.load().func() + + +def _entrypointToExtension(entrypoint): + """"Get the extension name from an EntryPoint object""" + # In Python 3.9 this can be "entrypoint.module" + module = entrypoint.pattern.match(entrypoint.value).groupdict()["module"] + extensionName = module.split(".")[0] + return extensionName + + +def _extensionsByPriority(): + """Discover extensions using the setuptools metadata + + TODO: This should move into a function which can also be called to fill the CS + """ + # This is only available in Python 3.8+ so it has to be here for now + from importlib import metadata # pylint: disable=no-name-in-module + + priorties = defaultdict(list) + for entrypoint in metadata.entry_points()['dirac']: + extensionName = _entrypointToExtension(entrypoint) + extension_metadata = entrypoint.load()() + priorties[extension_metadata["priority"]].append(extensionName) + gLogger.verbose( + "Found extension", + "{} with metadata {}".format(extensionName, extension_metadata), + ) + + extensions = [] + for priority, extensionNames in sorted(priorties.items()): + if len(extensionNames) != 1: + gLogger.warn( + "Found multiple extensions with priority", + "{} ({})".format(priority, extensionNames), + ) + # If multiple are passed, sort the extensions so things are deterministic at least + extensions.extend(sorted(extensionNames)) + return extensions diff --git a/src/DIRAC/__init__.py b/src/DIRAC/__init__.py index c624e2c8365..0f36d39606d 100755 --- a/src/DIRAC/__init__.py +++ b/src/DIRAC/__init__.py @@ -201,3 +201,14 @@ def abort(exitCode, *args, **kwargs): except OSError: gLogger.exception('Error while executing DIRAC.abort') os._exit(exitCode) + + +def extension_metadata(): + return { + "priority": 0, + "setups": { + "DIRAC-Certification": { + "ConfigurationServer": "dips://lbcertifdirac70.cern.ch:9135/Configuration/Server", + }, + } + } From b71143596fb54e33bc9cb9f586db4834c73f9abc Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 10 Dec 2020 15:52:47 +0100 Subject: [PATCH 06/27] PY3: Add entrypoints for dirac-info, dirac-proxy-init and dirac-proxy-info --- setup.cfg | 4 + src/DIRAC/Core/scripts/dirac_info.py | 110 +++++---- .../scripts/dirac_proxy_info.py | 221 +++++++++--------- .../scripts/dirac_proxy_init.py | 9 +- 4 files changed, 184 insertions(+), 160 deletions(-) diff --git a/setup.cfg b/setup.cfg index f0503c70003..9e993f784f5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -70,6 +70,10 @@ testing = [options.entry_points] dirac = metadata = DIRAC:extension_metadata +console_scripts = + dirac-info = DIRAC.Core.scripts.dirac_info:main + dirac-proxy-info = DIRAC.FrameworkSystem.scripts.dirac_proxy_info:main + dirac-proxy-init = DIRAC.FrameworkSystem.scripts.dirac_proxy_init:main ############################################################################### diff --git a/src/DIRAC/Core/scripts/dirac_info.py b/src/DIRAC/Core/scripts/dirac_info.py index 94d36a82b74..745ba252afc 100755 --- a/src/DIRAC/Core/scripts/dirac_info.py +++ b/src/DIRAC/Core/scripts/dirac_info.py @@ -10,63 +10,73 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division + __RCSID__ = "$Id$" -import os +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + import os -import DIRAC -from DIRAC import gConfig -from DIRAC.Core.Base import Script -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup -from DIRAC.Core.Utilities.PrettyPrint import printTable + import DIRAC + from DIRAC import gConfig + from DIRAC.Core.Base import Script + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup + from DIRAC.Core.Utilities.PrettyPrint import printTable -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Site' % Script.scriptName, ])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Site' % Script.scriptName, ])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -records = [] + records = [] -records.append(('Setup', gConfig.getValue('/DIRAC/Setup', 'Unknown'))) -records.append(('ConfigurationServer', gConfig.getValue('/DIRAC/Configuration/Servers', []))) -records.append(('Installation path', DIRAC.rootPath)) + records.append(('Setup', gConfig.getValue('/DIRAC/Setup', 'Unknown'))) + records.append(('ConfigurationServer', gConfig.getValue('/DIRAC/Configuration/Servers', []))) + records.append(('Installation path', DIRAC.rootPath)) -if os.path.exists(os.path.join(DIRAC.rootPath, DIRAC.getPlatform(), 'bin', 'mysql')): - records.append(('Installation type', 'server')) -else: - records.append(('Installation type', 'client')) + if os.path.exists(os.path.join(DIRAC.rootPath, DIRAC.getPlatform(), 'bin', 'mysql')): + records.append(('Installation type', 'server')) + else: + records.append(('Installation type', 'client')) + + records.append(('Platform', DIRAC.getPlatform())) -records.append(('Platform', DIRAC.getPlatform())) + ret = getProxyInfo(disableVOMS=True) + if ret['OK']: + if 'group' in ret['Value']: + vo = getVOForGroup(ret['Value']['group']) + else: + vo = getVOForGroup('') + if not vo: + vo = "None" + records.append(('VirtualOrganization', vo)) + if 'identity' in ret['Value']: + records.append(('User DN', ret['Value']['identity'])) + if 'secondsLeft' in ret['Value']: + records.append(('Proxy validity, secs', {'Value': str(ret['Value']['secondsLeft']), 'Just': 'L'})) -ret = getProxyInfo(disableVOMS=True) -if ret['OK']: - if 'group' in ret['Value']: - vo = getVOForGroup(ret['Value']['group']) + if gConfig.getValue('/DIRAC/Security/UseServerCertificate', True): + records.append(('Use Server Certificate', 'Yes')) + else: + records.append(('Use Server Certificate', 'No')) + if gConfig.getValue('/DIRAC/Security/SkipCAChecks', False): + records.append(('Skip CA Checks', 'Yes')) else: - vo = getVOForGroup('') - if not vo: - vo = "None" - records.append(('VirtualOrganization', vo)) - if 'identity' in ret['Value']: - records.append(('User DN', ret['Value']['identity'])) - if 'secondsLeft' in ret['Value']: - records.append(('Proxy validity, secs', {'Value': str(ret['Value']['secondsLeft']), 'Just': 'L'})) - -if gConfig.getValue('/DIRAC/Security/UseServerCertificate', True): - records.append(('Use Server Certificate', 'Yes')) -else: - records.append(('Use Server Certificate', 'No')) -if gConfig.getValue('/DIRAC/Security/SkipCAChecks', False): - records.append(('Skip CA Checks', 'Yes')) -else: - records.append(('Skip CA Checks', 'No')) - -records.append(('DIRAC version', DIRAC.version)) - -fields = ['Option', 'Value'] - -print() -printTable(fields, records, numbering=False) -print() + records.append(('Skip CA Checks', 'No')) + + records.append(('DIRAC version', DIRAC.version)) + + fields = ['Option', 'Value'] + + print() + printTable(fields, records, numbering=False) + print() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py index 15360b22e71..bdf07caf4af 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py @@ -8,8 +8,13 @@ from __future__ import absolute_import from __future__ import division +import sys + +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Utilities.ReturnValues import S_OK +__RCSID__ = "$Id$" + class Params(object): @@ -56,114 +61,112 @@ def setManagerInfo(self, arg): return S_OK() -params = Params() - -import sys - -from DIRAC.Core.Base import Script -Script.registerSwitch("f:", "file=", "File to use as user key", params.setProxyLocation) -Script.registerSwitch("i", "version", "Print version", params.showVersion) -Script.registerSwitch("n", "novoms", "Disable VOMS", params.disableVOMS) -Script.registerSwitch("v", "checkvalid", "Return error if the proxy is invalid", params.validityCheck) -Script.registerSwitch("x", "nocs", "Disable CS", params.disableCS) -Script.registerSwitch("e", "steps", "Show steps info", params.showSteps) -Script.registerSwitch("j", "noclockcheck", "Disable checking if time is ok", params.disableClockCheck) -Script.registerSwitch("m", "uploadedinfo", "Show uploaded proxies info", params.setManagerInfo) - -Script.disableCS() -Script.parseCommandLine() - -from DIRAC.Core.Utilities.NTP import getClockDeviation -from DIRAC import gLogger -from DIRAC.Core.Security.ProxyInfo import getProxyInfo, getProxyStepsInfo, formatProxyInfoAsString, formatProxyStepsInfoAsString -from DIRAC.Core.Security import VOMS -from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager -from DIRAC.ConfigurationSystem.Client.Helpers import Registry - - -__RCSID__ = "$Id$" - - -if params.csEnabled: - retVal = Script.enableCS() - if not retVal['OK']: - print("Cannot contact CS to get user list") - -if params.checkClock: - result = getClockDeviation() - if result['OK']: - deviation = result['Value'] - if deviation > 600: - gLogger.error("Your host clock seems to be off by more than TEN MINUTES! Thats really bad.") - elif deviation > 180: - gLogger.error("Your host clock seems to be off by more than THREE minutes! Thats bad.") - elif deviation > 60: - gLogger.error("Your host clock seems to be off by more than a minute! Thats not good.") - - -result = getProxyInfo(params.proxyLoc, not params.vomsEnabled) -if not result['OK']: - gLogger.error(result['Message']) - sys.exit(1) -infoDict = result['Value'] -gLogger.notice(formatProxyInfoAsString(infoDict)) -if not infoDict['isProxy']: - gLogger.error('==============================\n!!! The proxy is not valid !!!') - -if params.steps: - gLogger.notice("== Steps extended info ==") - chain = infoDict['chain'] - stepInfo = getProxyStepsInfo(chain)['Value'] - gLogger.notice(formatProxyStepsInfoAsString(stepInfo)) - - -def invalidProxy(msg): - gLogger.error("Invalid proxy:", msg) - sys.exit(1) - - -if params.uploadedInfo: - result = gProxyManager.getUserProxiesInfo() +@DIRACScript() +def main(): + params = Params() + + from DIRAC.Core.Base import Script + Script.registerSwitch("f:", "file=", "File to use as user key", params.setProxyLocation) + Script.registerSwitch("i", "version", "Print version", params.showVersion) + Script.registerSwitch("n", "novoms", "Disable VOMS", params.disableVOMS) + Script.registerSwitch("v", "checkvalid", "Return error if the proxy is invalid", params.validityCheck) + Script.registerSwitch("x", "nocs", "Disable CS", params.disableCS) + Script.registerSwitch("e", "steps", "Show steps info", params.showSteps) + Script.registerSwitch("j", "noclockcheck", "Disable checking if time is ok", params.disableClockCheck) + Script.registerSwitch("m", "uploadedinfo", "Show uploaded proxies info", params.setManagerInfo) + + Script.disableCS() + Script.parseCommandLine() + + from DIRAC.Core.Utilities.NTP import getClockDeviation + from DIRAC import gLogger + from DIRAC.Core.Security.ProxyInfo import getProxyInfo, getProxyStepsInfo + from DIRAC.Core.Security.ProxyInfo import formatProxyInfoAsString, formatProxyStepsInfoAsString + from DIRAC.Core.Security import VOMS + from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager + from DIRAC.ConfigurationSystem.Client.Helpers import Registry + + if params.csEnabled: + retVal = Script.enableCS() + if not retVal['OK']: + print("Cannot contact CS to get user list") + + if params.checkClock: + result = getClockDeviation() + if result['OK']: + deviation = result['Value'] + if deviation > 600: + gLogger.error("Your host clock seems to be off by more than TEN MINUTES! Thats really bad.") + elif deviation > 180: + gLogger.error("Your host clock seems to be off by more than THREE minutes! Thats bad.") + elif deviation > 60: + gLogger.error("Your host clock seems to be off by more than a minute! Thats not good.") + + result = getProxyInfo(params.proxyLoc, not params.vomsEnabled) if not result['OK']: - gLogger.error("Could not retrieve the uploaded proxies info", result['Message']) - else: - uploadedInfo = result['Value'] - if not uploadedInfo: - gLogger.notice("== No proxies uploaded ==") - if uploadedInfo: - gLogger.notice("== Proxies uploaded ==") - maxDNLen = 0 - maxGroupLen = 0 - for userDN in uploadedInfo: - maxDNLen = max(maxDNLen, len(userDN)) - for group in uploadedInfo[userDN]: - maxGroupLen = max(maxGroupLen, len(group)) - gLogger.notice(" %s | %s | Until (GMT)" % ("DN".ljust(maxDNLen), "Group".ljust(maxGroupLen))) - for userDN in uploadedInfo: - for group in uploadedInfo[userDN]: - gLogger.notice(" %s | %s | %s" % (userDN.ljust(maxDNLen), - group.ljust(maxGroupLen), - uploadedInfo[userDN][group].strftime("%Y/%m/%d %H:%M"))) - -if params.checkValid: - if infoDict['secondsLeft'] == 0: - invalidProxy("Proxy is expired") - if params.csEnabled and not infoDict['validGroup']: - invalidProxy("Group %s is not valid" % infoDict['group']) - if 'hasVOMS' in infoDict and infoDict['hasVOMS']: - requiredVOMS = Registry.getVOMSAttributeForGroup(infoDict['group']) - if 'VOMS' not in infoDict or not infoDict['VOMS']: - invalidProxy("Unable to retrieve VOMS extension") - if len(infoDict['VOMS']) > 1: - invalidProxy("More than one voms attribute found") - if requiredVOMS not in infoDict['VOMS']: - invalidProxy("Unexpected VOMS extension %s. Extension expected for DIRAC group is %s" % ( - infoDict['VOMS'][0], - requiredVOMS)) - result = VOMS.VOMS().getVOMSProxyInfo(infoDict['chain'], 'actimeleft') + gLogger.error(result['Message']) + sys.exit(1) + infoDict = result['Value'] + gLogger.notice(formatProxyInfoAsString(infoDict)) + if not infoDict['isProxy']: + gLogger.error('==============================\n!!! The proxy is not valid !!!') + + if params.steps: + gLogger.notice("== Steps extended info ==") + chain = infoDict['chain'] + stepInfo = getProxyStepsInfo(chain)['Value'] + gLogger.notice(formatProxyStepsInfoAsString(stepInfo)) + + def invalidProxy(msg): + gLogger.error("Invalid proxy:", msg) + sys.exit(1) + + if params.uploadedInfo: + result = gProxyManager.getUserProxiesInfo() if not result['OK']: - invalidProxy("Cannot determine life time of VOMS attributes: %s" % result['Message']) - if int(result['Value'].strip()) == 0: - invalidProxy("VOMS attributes are expired") - -sys.exit(0) + gLogger.error("Could not retrieve the uploaded proxies info", result['Message']) + else: + uploadedInfo = result['Value'] + if not uploadedInfo: + gLogger.notice("== No proxies uploaded ==") + if uploadedInfo: + gLogger.notice("== Proxies uploaded ==") + maxDNLen = 0 + maxGroupLen = 0 + for userDN in uploadedInfo: + maxDNLen = max(maxDNLen, len(userDN)) + for group in uploadedInfo[userDN]: + maxGroupLen = max(maxGroupLen, len(group)) + gLogger.notice(" %s | %s | Until (GMT)" % ("DN".ljust(maxDNLen), "Group".ljust(maxGroupLen))) + for userDN in uploadedInfo: + for group in uploadedInfo[userDN]: + gLogger.notice(" %s | %s | %s" % (userDN.ljust(maxDNLen), + group.ljust(maxGroupLen), + uploadedInfo[userDN][group].strftime("%Y/%m/%d %H:%M"))) + + if params.checkValid: + if infoDict['secondsLeft'] == 0: + invalidProxy("Proxy is expired") + if params.csEnabled and not infoDict['validGroup']: + invalidProxy("Group %s is not valid" % infoDict['group']) + if 'hasVOMS' in infoDict and infoDict['hasVOMS']: + requiredVOMS = Registry.getVOMSAttributeForGroup(infoDict['group']) + if 'VOMS' not in infoDict or not infoDict['VOMS']: + invalidProxy("Unable to retrieve VOMS extension") + if len(infoDict['VOMS']) > 1: + invalidProxy("More than one voms attribute found") + if requiredVOMS not in infoDict['VOMS']: + invalidProxy("Unexpected VOMS extension %s. Extension expected for DIRAC group is %s" % ( + infoDict['VOMS'][0], + requiredVOMS)) + result = VOMS.VOMS().getVOMSProxyInfo(infoDict['chain'], 'actimeleft') + if not result['OK']: + invalidProxy("Cannot determine life time of VOMS attributes: %s" % result['Message']) + if int(result['Value'].strip()) == 0: + invalidProxy("VOMS attributes are expired") + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py index d5bcf00a2f2..693ad998b7b 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py @@ -17,6 +17,7 @@ from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.FrameworkSystem.Client import ProxyGeneration, ProxyUpload from DIRAC.Core.Security import X509Chain, ProxyInfo, Properties, VOMS from DIRAC.ConfigurationSystem.Client.Helpers import Registry @@ -232,7 +233,9 @@ def doTheMagic(self): return S_OK() -if __name__ == "__main__": +@DIRACScript() +def main(): + global piParams, pI piParams = Params() piParams.registerCLISwitches() @@ -249,3 +252,7 @@ def doTheMagic(self): pI.printInfo() sys.exit(0) + + +if __name__ == "__main__": + main() From df5bae4447a2d44abb7a35d2d0e1ac928a5a8726 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 10 Dec 2020 15:54:41 +0100 Subject: [PATCH 07/27] Replace _ with - when creating script documentation --- docs/diracdoctools/cmd/commandReference.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/diracdoctools/cmd/commandReference.py b/docs/diracdoctools/cmd/commandReference.py index 395c5e938f5..ca7eed12ad7 100644 --- a/docs/diracdoctools/cmd/commandReference.py +++ b/docs/diracdoctools/cmd/commandReference.py @@ -93,6 +93,7 @@ def createFilesAndIndex(self, sectionDict): scriptName = os.path.basename(script) if scriptName.endswith('.py'): scriptName = scriptName[:-3] + scriptName = scriptName.replace("_", "-") prefix = sectionDict[PREFIX].lower() prefix = prefix + '_' if prefix else '' if self.createScriptDocFiles(script, sectionPath, scriptName, referencePrefix=prefix): @@ -120,6 +121,7 @@ def createFiles(self, sectionDict): scriptName = os.path.basename(script) if scriptName.endswith('.py'): scriptName = scriptName[:-3] + scriptName = scriptName.replace("_", "-") prefix = sectionDict[PREFIX].lower() prefix = prefix + '_' if prefix else '' if self.createScriptDocFiles(script, sectionPath, scriptName, referencePrefix=prefix) and \ From 88d5a6efb4c15b7ba0f1c5ea3f7c5a020de0ef6f Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 10 Dec 2020 16:58:37 +0100 Subject: [PATCH 08/27] PY3: Set DIRAC.__version__ using setuptools metadata --- setup.cfg | 3 +++ src/DIRAC/__init__.py | 30 +++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9e993f784f5..d7a02142370 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,6 +30,7 @@ install_requires = pyasn1-modules python-dateutil requests + setuptools six subprocess32 @@ -38,6 +39,8 @@ where=src [options.extras_require] server = + # arc6 doesn't provide a proper Python package with metadata + # (it just installs into site-packages) # arc boto3 botocore diff --git a/src/DIRAC/__init__.py b/src/DIRAC/__init__.py index 0f36d39606d..f3d754a1f8e 100755 --- a/src/DIRAC/__init__.py +++ b/src/DIRAC/__init__.py @@ -70,6 +70,9 @@ import os import platform as pyPlatform from pkgutil import extend_path + +import six + __path__ = extend_path(__path__, __name__) # Set the environment variable such that openssl accepts proxy cert @@ -88,22 +91,31 @@ # Importing _strptime before instantiating the threads seem to be a working workaround import _strptime +# Define Version +if six.PY3: + from pkg_resources import get_distribution, DistributionNotFound -# Define Version, use an unusual structure to minimise conflicts with rel-v7r2 -pythonVersion = pyPlatform.python_version_tuple() -if pythonVersion[0] == "3": - pass + try: + __version__ = get_distribution(__name__).version + version = __version__ + except DistributionNotFound: + # package is not installed + pass else: majorVersion = 7 minorVersion = 2 patchLevel = 0 preVersion = 28 -version = "v%sr%s" % (majorVersion, minorVersion) -if patchLevel: - version = "%sp%s" % (version, patchLevel) -if preVersion: - version = "%s-pre%s" % (version, preVersion) + version = "v%sr%s" % (majorVersion, minorVersion) + # Make it so that __version__ is always PEP-440 style + __version__ = "%s.%s" % (majorVersion, minorVersion) + if patchLevel: + version = "%sp%s" % (version, patchLevel) + __version__ += ".%s" % patchLevel + if preVersion: + version = "%s-pre%s" % (version, preVersion) + __version__ += "a%s" % preVersion errorMail = "dirac.alarms@gmail.com" alarmMail = "dirac.alarms@gmail.com" From 1fa9f792d7324779fe96f2fd9eec93974ea0cb76 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 17 Dec 2020 13:17:26 +0100 Subject: [PATCH 09/27] Apply review comments --- setup.cfg | 15 +++++++------ src/DIRAC/Core/Utilities/DIRACScript.py | 30 +++++-------------------- src/DIRAC/__init__.py | 2 +- 3 files changed, 15 insertions(+), 32 deletions(-) diff --git a/setup.cfg b/setup.cfg index d7a02142370..a1542821abd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,20 +19,27 @@ package_dir= =src packages = find: install_requires = + boto3 + botocore certifi diraccfg + fts3-rest future gfal2-python - M2Crypto + M2Crypto >=0.36 pexpect psutil pyasn1 pyasn1-modules + pyparsing python-dateutil + pytz requests setuptools six + sqlalchemy subprocess32 + tornado =5.1.1+dirac.1 [options.packages.find] where=src @@ -42,23 +49,17 @@ server = # arc6 doesn't provide a proper Python package with metadata # (it just installs into site-packages) # arc - boto3 - botocore CMRESHandler elasticsearch elasticsearch_dsl - fts3-rest GitPython ldap3 matplotlib mysqlclient numpy pillow - pyparsing python-irodsclient python-json-logger - pytz - sqlalchemy stomp.py suds-jurko tornado diff --git a/src/DIRAC/Core/Utilities/DIRACScript.py b/src/DIRAC/Core/Utilities/DIRACScript.py index 58b9da37d6f..87c7618bcd0 100644 --- a/src/DIRAC/Core/Utilities/DIRACScript.py +++ b/src/DIRAC/Core/Utilities/DIRACScript.py @@ -9,8 +9,6 @@ import six -from DIRAC import gLogger - class DIRACScript(object): """Decorator for providing command line executables @@ -23,18 +21,6 @@ def __init__(self): """ pass - @property - def func(self): - if not hasattr(self, "_func"): - raise NotImplementedError("Something is very wrong") - return self._func - - @func.setter - def func(self, value): - if hasattr(self, "_func"): - raise NotImplementedError("Something is very wrong") - self._func = value - def __call__(self, func=None): """Set the wrapped function or call the script @@ -45,12 +31,12 @@ def __call__(self, func=None): """ # If func is provided then the decorator is being applied to a function if func is not None: - self.func = func + self._func = func return functools.wraps(func)(self) # Setuptools based installations aren't supported with Python 2 if six.PY2: - return self.func() # pylint: disable=not-callable + return self._func() # pylint: disable=not-callable # This is only available in Python 3.8+ so it has to be here for now from importlib import metadata # pylint: disable=no-name-in-module @@ -75,7 +61,7 @@ def __call__(self, func=None): if function_name is None: # TODO: This should an error once the integration tests modified to use pip install - return self.func() # pylint: disable=not-callable + return self._func() # pylint: disable=not-callable # raise NotImplementedError("Something is very wrong") # Call the entry_point from the extension with the highest priority @@ -85,7 +71,7 @@ def __call__(self, func=None): key=lambda e: rankedExtensions.index(_entrypointToExtension(e)), ) - return entrypoint.load().func() + return entrypoint.load()._func() def _entrypointToExtension(entrypoint): @@ -109,16 +95,12 @@ def _extensionsByPriority(): extensionName = _entrypointToExtension(entrypoint) extension_metadata = entrypoint.load()() priorties[extension_metadata["priority"]].append(extensionName) - gLogger.verbose( - "Found extension", - "{} with metadata {}".format(extensionName, extension_metadata), - ) extensions = [] for priority, extensionNames in sorted(priorties.items()): if len(extensionNames) != 1: - gLogger.warn( - "Found multiple extensions with priority", + print( + "WARNING: Found multiple extensions with priority", "{} ({})".format(priority, extensionNames), ) # If multiple are passed, sort the extensions so things are deterministic at least diff --git a/src/DIRAC/__init__.py b/src/DIRAC/__init__.py index f3d754a1f8e..e15b7a96597 100755 --- a/src/DIRAC/__init__.py +++ b/src/DIRAC/__init__.py @@ -100,7 +100,7 @@ version = __version__ except DistributionNotFound: # package is not installed - pass + version = "Unknown" else: majorVersion = 7 minorVersion = 2 From 105d82aa95607df2dd71cef9d29c1e8b2cdfaba1 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 13 Jan 2021 10:43:01 +0100 Subject: [PATCH 10/27] Review comment --- src/DIRAC/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/DIRAC/__init__.py b/src/DIRAC/__init__.py index e15b7a96597..c685ba1e25f 100755 --- a/src/DIRAC/__init__.py +++ b/src/DIRAC/__init__.py @@ -218,9 +218,4 @@ def abort(exitCode, *args, **kwargs): def extension_metadata(): return { "priority": 0, - "setups": { - "DIRAC-Certification": { - "ConfigurationServer": "dips://lbcertifdirac70.cern.ch:9135/Configuration/Server", - }, - } } From 5a4cab5bd83828da9f38abc23d60115095fd933b Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 13 Jan 2021 10:43:22 +0100 Subject: [PATCH 11/27] Add documentation for new style scripts --- .../dirac-my-great-script.py | 25 ++++------ .../DevelopingCommands/dirac-ping-info.py | 46 ++++++++++------- .../DevelopingCommands/index.rst | 50 +++++++++++-------- setup.cfg | 2 + 4 files changed, 69 insertions(+), 54 deletions(-) diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-my-great-script.py b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-my-great-script.py index eed6ab5f4b2..ece39fb2217 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-my-great-script.py +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-my-great-script.py @@ -17,11 +17,9 @@ __RCSID__ = '$Id$' from DIRAC import S_OK, S_ERROR, gLogger, exit as DIRACExit +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Base import Script -cliParams = None -switchDict = None - class Params(object): ''' @@ -93,10 +91,17 @@ def parseSwitches(): return switches +@DIRACScript() def main(): ''' This is the script main method, which will hold all the logic. ''' + # Script initialization + registerSwitches() + switchDict = parseSwitches() + + # Import the required DIRAC modules + from DIRAC.Interfaces.API.Dirac import Dirac # let's do something if not len(switchDict['servicesList']): @@ -104,18 +109,8 @@ def main(): DIRACExit(1) gLogger.notice('We are done') + DIRACExit(0) -if __name__ == "__main__": - - # Script initialization - registerSwitches() - switchDict = parseSwitches() - - # Import the required DIRAC modules - from DIRAC.Interfaces.API.Dirac import Dirac - # Run the script +if __name__ == "__main__": main() - - # Bye - DIRACExit(0) diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-ping-info.py b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-ping-info.py index dc7ec21f8c6..5128c8aff0e 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-ping-info.py +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-ping-info.py @@ -12,10 +12,10 @@ from DIRAC import exit as DIRACExit from DIRAC import S_OK, S_ERROR from DIRAC.Core.Base import Script - -# Define a simple class to hold the script parameters +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +# Define a simple class to hold the script parameters class Params(object): def __init__(self): @@ -34,25 +34,33 @@ def setNumOfPingsToDo(self, value): return S_OK() -# Instantiate the params class -cliParams = Params() +@DIRACScript() +def main(): + # Instantiate the params class + cliParams = Params() + + # Register accepted switches and their callbacks + Script.registerSwitch("r", "showRaw", "show raw result from the query", cliParams.setRawResult) + Script.registerSwitch("p:", "numPings=", "Number of pings to do (by default 1)", cliParams.setNumOfPingsToDo) + + # Define a help message + Script.setUsageMessage('\n'.join([__doc__, + 'Usage:', + ' %s [option|cfgfile] +' % Script.scriptName, + ' Specifying a system is mandatory'])) + + # Parse the command line and initialize DIRAC + Script.parseCommandLine(ignoreErrors=False) -# Register accepted switches and their callbacks -Script.registerSwitch("r", "showRaw", "show raw result from the query", cliParams.setRawResult) -Script.registerSwitch("p:", "numPings=", "Number of pings to do (by default 1)", cliParams.setNumOfPingsToDo) + # Get the list of services + servicesList = Script.getPositionalArgs() -# Define a help message -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s [option|cfgfile] +' % Script.scriptName, - ' Specifying a system is mandatory'])) + # Check and process the command line switches and options + if not servicesList: + Script.showHelp(exitCode=1) -# Parse the command line and initialize DIRAC -Script.parseCommandLine(ignoreErrors=False) + # Do something! -# Get the list of services -servicesList = Script.getPositionalArgs() -# Check and process the command line switches and options -if not servicesList: - Script.showHelp(exitCode=1) +if __name__ == "__main__": + main() diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst index ca352e5bace..4e95afa51b5 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst @@ -2,6 +2,10 @@ Developing Commands ====================================== +.. warning:: + This instructions here demonstrate how to support both the legacy (Python 2) and future (Python3) installations of DIRAC. + If only having Python 3 support is acceptable, the requirement for scripts to be in the the *scripts* directory of their parent system will be removed and the only requirement will be for the function to be decorated with the ``@DIRACScript()`` decorator. + Commands are one of the main interface tools for the users. Commands are also called *scripts* in DIRAC lingo. Where to place scripts @@ -31,9 +35,32 @@ The instructions below must be applied as close as possible although some variat which will set the interpreter directive to the python on the environment. -**2.** The next is the documentation line which is describing the command. This same documentation line will be used also the command help information available with the *-h* command switch. +**2.** The next is the documentation line which is describing the command. This same documentation line will be used also the command help information available with the *-h* command switch. + +**3.** The majority of the code should be contained with a function, often called ``main`` though this is not required. This function should be wrapped with the ``@DiracScript()`` decorator to allow the DIRAC plugin mechanism to override the script with the function from the highest priority extension. + +.. code-block:: python + + #Import the required DIRAC modules + from DIRAC.Core.Utilities.DIRACScript import DIRACScript + from DIRAC.Interfaces.API.DIRAC import DIRAC + from DIRAC import gLogger + + @DiracScript() + def main() + # Do stuff + + if __name__ == "__main__": + main() + +**4.** Next the function must be registered as a ``console_scripts`` ``entrypoint`` in the ``setuptools`` metadata (`more details`_). This is done by adding a line to the ``console_scripts`` list like below, where the first string is the name for the script you want to create, the left hand side of ``:`` is the module that contains your function and the right hand side is the object you want to invoke (e.g. a function). -**3.** Users need to specify parameters to scripts to define what they want to do. To do so, they pass arguments when calling the script. The first thing any script has to do is define what options and arguments the script accepts. Once the valid arguments are defined, the script can parse the command line. An example follows which is a typical command description part +.. code-block:: cfg + console_scripts = + dirac-info = DIRAC.Core.scripts.dirac_info:main + dirac-proxy-info = DIRAC.FrameworkSystem.scripts.dirac_proxy_info:main + +**5.** Users need to specify parameters to scripts to define what they want to do. To do so, they pass arguments when calling the script. The first thing any script has to do is define what options and arguments the script accepts. Once the valid arguments are defined, the script can parse the command line. An example follows which is a typical command description part .. literalinclude:: dirac-ping-info.py @@ -61,22 +88,7 @@ All the command line arguments that are not corresponding to the explicitly defi After defining the switches, the *parseCommandLine()* function has to be called. This method not only parses the command line options but also initializes DIRAC collecting all the configuration data. **It is absolutely important to call this function before importing any other DIRAC module**. The callbacks defined for the switches will be called when parsing the command line if necessary. *Even if the switch is not supposed to receive a parameter, the callback has to receive a value*. Switches without callbacks defined can be obtained with *getUnprocessedSwitches()* function. -**4.** Once the command line has been parsed and DIRAC is properly initialized, the rest of the required DIRAC modules can be imported and the script logic can take place: - -.. code-block:: python - - #Import the required DIRAC modules - from DIRAC.Interfaces.API.DIRAC import DIRAC - from DIRAC import gLogger - #Do stuff... depending on cliParams.raw, cliParams.pingsToDo and servicesList - - def executeCommandLogic() - # Do stuff - gLogger.notice('This is the result of the command') - - if __name__ == "__main__": - executeCommandLogic() - +**5.** Once the command line has been parsed and DIRAC is properly initialized, the rest of the required DIRAC modules can be imported and the script logic can take place. Having understood the logic of the script, there are few good practices that must be followed: @@ -84,8 +96,6 @@ Having understood the logic of the script, there are few good practices that mus * Encapsulate the command code into functions / classes so that it can be easily tested * Usage of *gLogger* instead of *print* is mandatory. The information in the normal command execution must be printed out in the NOTICE logging level. -* Use the *if __name__ == "__main__"* close for the actual command execution to avoid running the script - when it is imported. Example command ----------------- diff --git a/setup.cfg b/setup.cfg index a1542821abd..73d69f659d5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -74,6 +74,8 @@ testing = [options.entry_points] dirac = metadata = DIRAC:extension_metadata +# For details on adding additional scripts to vanilla DIRAC or external extensions see +# https://dirac.readthedocs.io/en/integration/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.html console_scripts = dirac-info = DIRAC.Core.scripts.dirac_info:main dirac-proxy-info = DIRAC.FrameworkSystem.scripts.dirac_proxy_info:main From 65a1d991861d08457d0e78227060b41d488aaf89 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 13 Jan 2021 12:05:09 +0100 Subject: [PATCH 12/27] Fix docs --- docs/diracdoctools/cmd/commandReference.py | 5 ++++- .../AddingNewComponents/DevelopingCommands/index.rst | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/diracdoctools/cmd/commandReference.py b/docs/diracdoctools/cmd/commandReference.py index ca7eed12ad7..326da41da3d 100644 --- a/docs/diracdoctools/cmd/commandReference.py +++ b/docs/diracdoctools/cmd/commandReference.py @@ -140,7 +140,10 @@ def cleanExistingIndex(self, sectionDict): If an rst file exists for a command, we move it. An existing entry for a non existing rst file will create a warning when running sphinx. """ - existingCommands = {os.path.basename(com).replace('.py', '') for com in sectionDict[SCRIPTS] + sectionDict[MANUAL]} + existingCommands = { + os.path.basename(com).replace('.py', '').replace("_", "-") + for com in sectionDict[SCRIPTS] + sectionDict[MANUAL] + } sectionPath = os.path.join(self.config.docsPath, sectionDict[SECTION_PATH]) LOG.info('Checking %r for non-existent commands', sectionPath) # read the script index diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst index 4e95afa51b5..884b0a39bba 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst @@ -56,6 +56,7 @@ which will set the interpreter directive to the python on the environment. **4.** Next the function must be registered as a ``console_scripts`` ``entrypoint`` in the ``setuptools`` metadata (`more details`_). This is done by adding a line to the ``console_scripts`` list like below, where the first string is the name for the script you want to create, the left hand side of ``:`` is the module that contains your function and the right hand side is the object you want to invoke (e.g. a function). .. code-block:: cfg + console_scripts = dirac-info = DIRAC.Core.scripts.dirac_info:main dirac-proxy-info = DIRAC.FrameworkSystem.scripts.dirac_proxy_info:main From 5a248b7b772881fbcf6e6ce4cf472393c04a4051 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 13 Jan 2021 13:27:53 +0100 Subject: [PATCH 13/27] Fix docs some more --- docs/diracdoctools/cmd/commandReference.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/diracdoctools/cmd/commandReference.py b/docs/diracdoctools/cmd/commandReference.py index 326da41da3d..31204d01dc8 100644 --- a/docs/diracdoctools/cmd/commandReference.py +++ b/docs/diracdoctools/cmd/commandReference.py @@ -56,8 +56,8 @@ def getScripts(self): continue for mT in self.sectionDicts: - if any(pattern in scriptPath for pattern in mT[PATTERN]) and \ - not any(pattern in scriptPath for pattern in mT[EXCLUDE]): + if any(pattern in scriptPath.replace("_", "-") for pattern in mT[PATTERN]) and \ + not any(pattern in scriptPath.replace("_", "-") for pattern in mT[EXCLUDE]): mT[SCRIPTS].append(scriptPath) return From 35fa23942ef39148a65c1fbd34d07608eaf94ce5 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 13 Jan 2021 15:03:47 +0100 Subject: [PATCH 14/27] Fix docs text formatting --- .../AddingNewComponents/DevelopingCommands/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst index 884b0a39bba..ca2460579f9 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst @@ -53,7 +53,7 @@ which will set the interpreter directive to the python on the environment. if __name__ == "__main__": main() -**4.** Next the function must be registered as a ``console_scripts`` ``entrypoint`` in the ``setuptools`` metadata (`more details`_). This is done by adding a line to the ``console_scripts`` list like below, where the first string is the name for the script you want to create, the left hand side of ``:`` is the module that contains your function and the right hand side is the object you want to invoke (e.g. a function). +**4.** Next the function must be registered as a ``console_scripts`` ``entrypoint`` in the ``setuptools`` metadata (`more details `_). This is done by adding a line to the ``console_scripts`` list like below, where the first string is the name for the script you want to create, the left hand side of ``:`` is the module that contains your function and the right hand side is the object you want to invoke (e.g. a function). .. code-block:: cfg From 6505f58cc8568b7ae75327222b70ed60c80fad95 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 13 Jan 2021 16:15:40 +0100 Subject: [PATCH 15/27] Rename all scripts to use _ instead of - --- ...ounting-decode-fileid.py => dirac_accounting_decode_fileid.py} | 0 ...irac-admin-accounting-cli.py => dirac_admin_accounting_cli.py} | 0 ...{dirac-admin-add-resources.py => dirac_admin_add_resources.py} | 0 .../{dirac-admin-add-shifter.py => dirac_admin_add_shifter.py} | 0 .../scripts/{dirac-admin-add-site.py => dirac_admin_add_site.py} | 0 .../{dirac-admin-bdii-info.py => dirac_admin_bdii_info.py} | 0 ...heck-config-options.py => dirac_admin_check_config_options.py} | 0 ...{dirac-admin-sort-cs-sites.py => dirac_admin_sort_cs_sites.py} | 0 .../{dirac-admin-voms-sync.py => dirac_admin_voms_sync.py} | 0 .../{dirac-configuration-cli.py => dirac_configuration_cli.py} | 0 ...ump-local-cache.py => dirac_configuration_dump_local_cache.py} | 0 ...{dirac-configuration-shell.py => dirac_configuration_shell.py} | 0 .../Tornado/scripts/{tornado-start-CS.py => tornado_start_CS.py} | 0 .../scripts/{tornado-start-all.py => tornado_start_all.py} | 0 src/DIRAC/Core/scripts/{dirac-agent.py => dirac_agent.py} | 0 src/DIRAC/Core/scripts/{dirac-configure.py => dirac_configure.py} | 0 .../scripts/{dirac-deploy-scripts.py => dirac_deploy_scripts.py} | 0 src/DIRAC/Core/scripts/{dirac-executor.py => dirac_executor.py} | 0 ...-externals-requirements.py => dirac_externals_requirements.py} | 0 src/DIRAC/Core/scripts/{dirac-install.py => dirac_install.py} | 0 .../Core/scripts/{dirac-install-db.py => dirac_install_db.py} | 0 .../{dirac-install-extension.py => dirac_install_extension.py} | 0 .../{dirac-install-web-portal.py => dirac_install_web_portal.py} | 0 src/DIRAC/Core/scripts/{dirac-platform.py => dirac_platform.py} | 0 src/DIRAC/Core/scripts/{dirac-service.py => dirac_service.py} | 0 .../Core/scripts/{dirac-setup-site.py => dirac_setup_site.py} | 0 src/DIRAC/Core/scripts/{dirac-version.py => dirac_version.py} | 0 .../scripts/{dirac-admin-allow-se.py => dirac_admin_allow_se.py} | 0 .../scripts/{dirac-admin-ban-se.py => dirac_admin_ban_se.py} | 0 .../{dirac-admin-user-quota.py => dirac_admin_user_quota.py} | 0 .../scripts/{dirac-dms-add-file.py => dirac_dms_add_file.py} | 0 ...irac-dms-catalog-metadata.py => dirac_dms_catalog_metadata.py} | 0 ...hange-replica-status.py => dirac_dms_change_replica_status.py} | 0 ...{dirac-dms-clean-directory.py => dirac_dms_clean_directory.py} | 0 ...ate-archive-request.py => dirac_dms_create_archive_request.py} | 0 ...reate-moving-request.py => dirac_dms_create_moving_request.py} | 0 ...ate-removal-request.py => dirac_dms_create_removal_request.py} | 0 .../scripts/{dirac-dms-data-size.py => dirac_dms_data_size.py} | 0 .../{dirac-dms-directory-sync.py => dirac_dms_directory_sync.py} | 0 ...{dirac-dms-filecatalog-cli.py => dirac_dms_filecatalog_cli.py} | 0 .../scripts/{dirac-dms-find-lfns.py => dirac_dms_find_lfns.py} | 0 ...-move-replica-request.py => dirac_dms_move_replica_request.py} | 0 ...{dirac-dms-protocol-matrix.py => dirac_dms_protocol_matrix.py} | 0 ...-register-request.py => dirac_dms_put_and_register_request.py} | 0 ...-remove-catalog-files.py => dirac_dms_remove_catalog_files.py} | 0 ...e-catalog-replicas.py => dirac_dms_remove_catalog_replicas.py} | 0 .../{dirac-dms-remove-files.py => dirac_dms_remove_files.py} | 0 ...{dirac-dms-remove-replicas.py => dirac_dms_remove_replicas.py} | 0 ...irac-dms-replica-metadata.py => dirac_dms_replica_metadata.py} | 0 ...ter-request.py => dirac_dms_replicate_and_register_request.py} | 0 .../{dirac-dms-resolve-guid.py => dirac_dms_resolve_guid.py} | 0 ...-dms-set-replica-status.py => dirac_dms_set_replica_status.py} | 0 .../{dirac-dms-show-se-status.py => dirac_dms_show_se_status.py} | 0 .../scripts/{dirac-dms-user-lfns.py => dirac_dms_user_lfns.py} | 0 .../scripts/{dirac-dms-user-quota.py => dirac_dms_user_quota.py} | 0 .../scripts/{dirac-admin-get-CAs.py => dirac_admin_get_CAs.py} | 0 .../{dirac-admin-get-proxy.py => dirac_admin_get_proxy.py} | 0 .../{dirac-admin-proxy-upload.py => dirac_admin_proxy_upload.py} | 0 .../{dirac-admin-sysadmin-cli.py => dirac_admin_sysadmin_cli.py} | 0 ...ac-admin-update-instance.py => dirac_admin_update_instance.py} | 0 .../{dirac-admin-update-pilot.py => dirac_admin_update_pilot.py} | 0 ...-admin-users-with-proxy.py => dirac_admin_users_with_proxy.py} | 0 .../{dirac-install-component.py => dirac_install_component.py} | 0 ...nstall-tornado-service.py => dirac_install_tornado_service.py} | 0 ...onents-status.py => dirac_monitoring_get_components_status.py} | 0 .../scripts/{dirac-myproxy-upload.py => dirac_myproxy_upload.py} | 0 ...ac-populate-component-db.py => dirac_populate_component_db.py} | 0 .../scripts/{dirac-proxy-destroy.py => dirac_proxy_destroy.py} | 0 ...roxy-get-uploaded-info.py => dirac_proxy_get_uploaded_info.py} | 0 .../{dirac-restart-component.py => dirac_restart_component.py} | 0 .../{dirac-start-component.py => dirac_start_component.py} | 0 .../{dirac-status-component.py => dirac_status_component.py} | 0 .../scripts/{dirac-stop-component.py => dirac_stop_component.py} | 0 .../scripts/{dirac-sys-sendmail.py => dirac_sys_sendmail.py} | 0 ...{dirac-uninstall-component.py => dirac_uninstall_component.py} | 0 .../{dirac-admin-add-group.py => dirac_admin_add_group.py} | 0 .../scripts/{dirac-admin-add-host.py => dirac_admin_add_host.py} | 0 .../scripts/{dirac-admin-add-user.py => dirac_admin_add_user.py} | 0 .../{dirac-admin-allow-site.py => dirac_admin_allow_site.py} | 0 .../scripts/{dirac-admin-ban-site.py => dirac_admin_ban_site.py} | 0 .../scripts/{dirac-admin-ce-info.py => dirac_admin_ce_info.py} | 0 .../{dirac-admin-delete-user.py => dirac_admin_delete_user.py} | 0 ...-admin-get-banned-sites.py => dirac_admin_get_banned_sites.py} | 0 ...et-job-pilot-output.py => dirac_admin_get_job_pilot_output.py} | 0 ...irac-admin-get-job-pilots.py => dirac_admin_get_job_pilots.py} | 0 ...irac-admin-get-pilot-info.py => dirac_admin_get_pilot_info.py} | 0 ...ilot-logging-info.py => dirac_admin_get_pilot_logging_info.py} | 0 ...-admin-get-pilot-output.py => dirac_admin_get_pilot_output.py} | 0 ...{dirac-admin-get-site-mask.py => dirac_admin_get_site_mask.py} | 0 .../{dirac-admin-list-hosts.py => dirac_admin_list_hosts.py} | 0 .../{dirac-admin-list-users.py => dirac_admin_list_users.py} | 0 .../{dirac-admin-modify-user.py => dirac_admin_modify_user.py} | 0 ...{dirac-admin-pilot-summary.py => dirac_admin_pilot_summary.py} | 0 .../{dirac-admin-reset-job.py => dirac_admin_reset_job.py} | 0 ...{dirac-admin-service-ports.py => dirac_admin_service_ports.py} | 0 ...in-set-site-protocols.py => dirac_admin_set_site_protocols.py} | 0 .../{dirac-admin-site-info.py => dirac_admin_site_info.py} | 0 ...dmin-site-mask-logging.py => dirac_admin_site_mask_logging.py} | 0 ...ync-users-from-file.py => dirac_admin_sync_users_from_file.py} | 0 .../scripts/{dirac-dms-get-file.py => dirac_dms_get_file.py} | 0 .../{dirac-dms-lfn-accessURL.py => dirac_dms_lfn_accessURL.py} | 0 .../{dirac-dms-lfn-metadata.py => dirac_dms_lfn_metadata.py} | 0 .../{dirac-dms-lfn-replicas.py => dirac_dms_lfn_replicas.py} | 0 .../{dirac-dms-pfn-accessURL.py => dirac_dms_pfn_accessURL.py} | 0 .../{dirac-dms-pfn-metadata.py => dirac_dms_pfn_metadata.py} | 0 .../{dirac-dms-replicate-lfn.py => dirac_dms_replicate_lfn.py} | 0 ...-framework-ping-service.py => dirac_framework_ping_service.py} | 0 ...{dirac-framework-self-ping.py => dirac_framework_self_ping.py} | 0 .../scripts/{dirac-repo-monitor.py => dirac_repo_monitor.py} | 0 .../{dirac-utils-file-adler.py => dirac_utils_file_adler.py} | 0 .../scripts/{dirac-utils-file-md5.py => dirac_utils_file_md5.py} | 0 ...d-queue-length.py => dirac_wms_get_normalized_queue_length.py} | 0 ...ueue-normalization.py => dirac_wms_get_queue_normalization.py} | 0 .../{dirac-wms-job-attributes.py => dirac_wms_job_attributes.py} | 0 .../scripts/{dirac-wms-job-delete.py => dirac_wms_job_delete.py} | 0 .../{dirac-wms-job-get-input.py => dirac_wms_job_get_input.py} | 0 .../{dirac-wms-job-get-jdl.py => dirac_wms_job_get_jdl.py} | 0 .../{dirac-wms-job-get-output.py => dirac_wms_job_get_output.py} | 0 ...ms-job-get-output-data.py => dirac_wms_job_get_output_data.py} | 0 .../scripts/{dirac-wms-job-kill.py => dirac_wms_job_kill.py} | 0 ...irac-wms-job-logging-info.py => dirac_wms_job_logging_info.py} | 0 .../{dirac-wms-job-parameters.py => dirac_wms_job_parameters.py} | 0 .../scripts/{dirac-wms-job-peek.py => dirac_wms_job_peek.py} | 0 .../{dirac-wms-job-reschedule.py => dirac_wms_job_reschedule.py} | 0 .../scripts/{dirac-wms-job-status.py => dirac_wms_job_status.py} | 0 .../scripts/{dirac-wms-job-submit.py => dirac_wms_job_submit.py} | 0 ...ct-output-search.py => dirac_wms_jobs_select_output_search.py} | 0 .../{dirac-wms-select-jobs.py => dirac_wms_select_jobs.py} | 0 .../scripts/{dirac-prod-add-trans.py => dirac_prod_add_trans.py} | 0 .../scripts/{dirac-prod-clean.py => dirac_prod_clean.py} | 0 .../scripts/{dirac-prod-delete.py => dirac_prod_delete.py} | 0 .../scripts/{dirac-prod-get.py => dirac_prod_get.py} | 0 .../scripts/{dirac-prod-get-all.py => dirac_prod_get_all.py} | 0 ...irac-prod-get-description.py => dirac_prod_get_description.py} | 0 .../scripts/{dirac-prod-get-trans.py => dirac_prod_get_trans.py} | 0 .../scripts/{dirac-prod-start.py => dirac_prod_start.py} | 0 .../scripts/{dirac-prod-stop.py => dirac_prod_stop.py} | 0 .../{dirac-rms-list-req-cache.py => dirac_rms_list_req_cache.py} | 0 .../{dirac-rms-reqdb-summary.py => dirac_rms_reqdb_summary.py} | 0 .../scripts/{dirac-rms-request.py => dirac_rms_request.py} | 0 .../{dirac-rss-list-status.py => dirac_rss_list_status.py} | 0 .../scripts/{dirac-rss-query-db.py => dirac_rss_query_db.py} | 0 .../{dirac-rss-query-dtcache.py => dirac_rss_query_dtcache.py} | 0 .../scripts/{dirac-rss-set-status.py => dirac_rss_set_status.py} | 0 .../scripts/{dirac-rss-set-token.py => dirac_rss_set_token.py} | 0 .../scripts/{dirac-rss-sync.py => dirac_rss_sync.py} | 0 ...esource-get-parameters.py => dirac_resource_get_parameters.py} | 0 .../scripts/{dirac-resource-info.py => dirac_resource_info.py} | 0 ...{dirac-stager-monitor-file.py => dirac_stager_monitor_file.py} | 0 ...{dirac-stager-monitor-jobs.py => dirac_stager_monitor_jobs.py} | 0 ...-stager-monitor-request.py => dirac_stager_monitor_request.py} | 0 ...tager-monitor-requests.py => dirac_stager_monitor_requests.py} | 0 .../{dirac-stager-show-stats.py => dirac_stager_show_stats.py} | 0 .../{dirac-stager-stage-files.py => dirac_stager_stage_files.py} | 0 ...-production-runjoblocal.py => dirac_production_runjoblocal.py} | 0 ...nsformation-add-files.py => dirac_transformation_add_files.py} | 0 ...-transformation-archive.py => dirac_transformation_archive.py} | 0 ...irac-transformation-clean.py => dirac_transformation_clean.py} | 0 .../{dirac-transformation-cli.py => dirac_transformation_cli.py} | 0 ...nsformation-get-files.py => dirac_transformation_get_files.py} | 0 ...ation-recover-data.py => dirac_transformation_recover_data.py} | 0 ...ion-remove-output.py => dirac_transformation_remove_output.py} | 0 ...rmation-replication.py => dirac_transformation_replication.py} | 0 ...fy-outputdata.py => dirac_transformation_verify_outputdata.py} | 0 .../{dirac-admin-kill-pilot.py => dirac_admin_kill_pilot.py} | 0 ...in-pilot-logging-info.py => dirac_admin_pilot_logging_info.py} | 0 ...-admin-show-task-queues.py => dirac_admin_show_task_queues.py} | 0 .../{dirac-admin-sync-pilot.py => dirac_admin_sync_pilot.py} | 0 .../scripts/{dirac-jobexec.py => dirac_jobexec.py} | 0 ...ac-wms-cpu-normalization.py => dirac_wms_cpu_normalization.py} | 0 ...-wms-get-queue-cpu-time.py => dirac_wms_get_queue_cpu_time.py} | 0 .../scripts/{dirac-wms-get-wn.py => dirac_wms_get_wn.py} | 0 ...ac-wms-get-wn-parameters.py => dirac_wms_get_wn_parameters.py} | 0 .../scripts/{dirac-wms-match.py => dirac_wms_match.py} | 0 .../{dirac-wms-pilot-job-info.py => dirac_wms_pilot_job_info.py} | 0 175 files changed, 0 insertions(+), 0 deletions(-) rename src/DIRAC/AccountingSystem/scripts/{dirac-accounting-decode-fileid.py => dirac_accounting_decode_fileid.py} (100%) rename src/DIRAC/AccountingSystem/scripts/{dirac-admin-accounting-cli.py => dirac_admin_accounting_cli.py} (100%) rename src/DIRAC/ConfigurationSystem/scripts/{dirac-admin-add-resources.py => dirac_admin_add_resources.py} (100%) rename src/DIRAC/ConfigurationSystem/scripts/{dirac-admin-add-shifter.py => dirac_admin_add_shifter.py} (100%) rename src/DIRAC/ConfigurationSystem/scripts/{dirac-admin-add-site.py => dirac_admin_add_site.py} (100%) rename src/DIRAC/ConfigurationSystem/scripts/{dirac-admin-bdii-info.py => dirac_admin_bdii_info.py} (100%) rename src/DIRAC/ConfigurationSystem/scripts/{dirac-admin-check-config-options.py => dirac_admin_check_config_options.py} (100%) rename src/DIRAC/ConfigurationSystem/scripts/{dirac-admin-sort-cs-sites.py => dirac_admin_sort_cs_sites.py} (100%) rename src/DIRAC/ConfigurationSystem/scripts/{dirac-admin-voms-sync.py => dirac_admin_voms_sync.py} (100%) rename src/DIRAC/ConfigurationSystem/scripts/{dirac-configuration-cli.py => dirac_configuration_cli.py} (100%) rename src/DIRAC/ConfigurationSystem/scripts/{dirac-configuration-dump-local-cache.py => dirac_configuration_dump_local_cache.py} (100%) rename src/DIRAC/ConfigurationSystem/scripts/{dirac-configuration-shell.py => dirac_configuration_shell.py} (100%) rename src/DIRAC/Core/Tornado/scripts/{tornado-start-CS.py => tornado_start_CS.py} (100%) rename src/DIRAC/Core/Tornado/scripts/{tornado-start-all.py => tornado_start_all.py} (100%) rename src/DIRAC/Core/scripts/{dirac-agent.py => dirac_agent.py} (100%) rename src/DIRAC/Core/scripts/{dirac-configure.py => dirac_configure.py} (100%) rename src/DIRAC/Core/scripts/{dirac-deploy-scripts.py => dirac_deploy_scripts.py} (100%) rename src/DIRAC/Core/scripts/{dirac-executor.py => dirac_executor.py} (100%) rename src/DIRAC/Core/scripts/{dirac-externals-requirements.py => dirac_externals_requirements.py} (100%) rename src/DIRAC/Core/scripts/{dirac-install.py => dirac_install.py} (100%) rename src/DIRAC/Core/scripts/{dirac-install-db.py => dirac_install_db.py} (100%) rename src/DIRAC/Core/scripts/{dirac-install-extension.py => dirac_install_extension.py} (100%) rename src/DIRAC/Core/scripts/{dirac-install-web-portal.py => dirac_install_web_portal.py} (100%) rename src/DIRAC/Core/scripts/{dirac-platform.py => dirac_platform.py} (100%) rename src/DIRAC/Core/scripts/{dirac-service.py => dirac_service.py} (100%) rename src/DIRAC/Core/scripts/{dirac-setup-site.py => dirac_setup_site.py} (100%) rename src/DIRAC/Core/scripts/{dirac-version.py => dirac_version.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-admin-allow-se.py => dirac_admin_allow_se.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-admin-ban-se.py => dirac_admin_ban_se.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-admin-user-quota.py => dirac_admin_user_quota.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-add-file.py => dirac_dms_add_file.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-catalog-metadata.py => dirac_dms_catalog_metadata.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-change-replica-status.py => dirac_dms_change_replica_status.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-clean-directory.py => dirac_dms_clean_directory.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-create-archive-request.py => dirac_dms_create_archive_request.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-create-moving-request.py => dirac_dms_create_moving_request.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-create-removal-request.py => dirac_dms_create_removal_request.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-data-size.py => dirac_dms_data_size.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-directory-sync.py => dirac_dms_directory_sync.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-filecatalog-cli.py => dirac_dms_filecatalog_cli.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-find-lfns.py => dirac_dms_find_lfns.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-move-replica-request.py => dirac_dms_move_replica_request.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-protocol-matrix.py => dirac_dms_protocol_matrix.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-put-and-register-request.py => dirac_dms_put_and_register_request.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-remove-catalog-files.py => dirac_dms_remove_catalog_files.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-remove-catalog-replicas.py => dirac_dms_remove_catalog_replicas.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-remove-files.py => dirac_dms_remove_files.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-remove-replicas.py => dirac_dms_remove_replicas.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-replica-metadata.py => dirac_dms_replica_metadata.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-replicate-and-register-request.py => dirac_dms_replicate_and_register_request.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-resolve-guid.py => dirac_dms_resolve_guid.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-set-replica-status.py => dirac_dms_set_replica_status.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-show-se-status.py => dirac_dms_show_se_status.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-user-lfns.py => dirac_dms_user_lfns.py} (100%) rename src/DIRAC/DataManagementSystem/scripts/{dirac-dms-user-quota.py => dirac_dms_user_quota.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-admin-get-CAs.py => dirac_admin_get_CAs.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-admin-get-proxy.py => dirac_admin_get_proxy.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-admin-proxy-upload.py => dirac_admin_proxy_upload.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-admin-sysadmin-cli.py => dirac_admin_sysadmin_cli.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-admin-update-instance.py => dirac_admin_update_instance.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-admin-update-pilot.py => dirac_admin_update_pilot.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-admin-users-with-proxy.py => dirac_admin_users_with_proxy.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-install-component.py => dirac_install_component.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-install-tornado-service.py => dirac_install_tornado_service.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-monitoring-get-components-status.py => dirac_monitoring_get_components_status.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-myproxy-upload.py => dirac_myproxy_upload.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-populate-component-db.py => dirac_populate_component_db.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-proxy-destroy.py => dirac_proxy_destroy.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-proxy-get-uploaded-info.py => dirac_proxy_get_uploaded_info.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-restart-component.py => dirac_restart_component.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-start-component.py => dirac_start_component.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-status-component.py => dirac_status_component.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-stop-component.py => dirac_stop_component.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-sys-sendmail.py => dirac_sys_sendmail.py} (100%) rename src/DIRAC/FrameworkSystem/scripts/{dirac-uninstall-component.py => dirac_uninstall_component.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-add-group.py => dirac_admin_add_group.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-add-host.py => dirac_admin_add_host.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-add-user.py => dirac_admin_add_user.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-allow-site.py => dirac_admin_allow_site.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-ban-site.py => dirac_admin_ban_site.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-ce-info.py => dirac_admin_ce_info.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-delete-user.py => dirac_admin_delete_user.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-get-banned-sites.py => dirac_admin_get_banned_sites.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-get-job-pilot-output.py => dirac_admin_get_job_pilot_output.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-get-job-pilots.py => dirac_admin_get_job_pilots.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-get-pilot-info.py => dirac_admin_get_pilot_info.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-get-pilot-logging-info.py => dirac_admin_get_pilot_logging_info.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-get-pilot-output.py => dirac_admin_get_pilot_output.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-get-site-mask.py => dirac_admin_get_site_mask.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-list-hosts.py => dirac_admin_list_hosts.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-list-users.py => dirac_admin_list_users.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-modify-user.py => dirac_admin_modify_user.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-pilot-summary.py => dirac_admin_pilot_summary.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-reset-job.py => dirac_admin_reset_job.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-service-ports.py => dirac_admin_service_ports.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-set-site-protocols.py => dirac_admin_set_site_protocols.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-site-info.py => dirac_admin_site_info.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-site-mask-logging.py => dirac_admin_site_mask_logging.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-admin-sync-users-from-file.py => dirac_admin_sync_users_from_file.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-dms-get-file.py => dirac_dms_get_file.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-dms-lfn-accessURL.py => dirac_dms_lfn_accessURL.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-dms-lfn-metadata.py => dirac_dms_lfn_metadata.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-dms-lfn-replicas.py => dirac_dms_lfn_replicas.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-dms-pfn-accessURL.py => dirac_dms_pfn_accessURL.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-dms-pfn-metadata.py => dirac_dms_pfn_metadata.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-dms-replicate-lfn.py => dirac_dms_replicate_lfn.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-framework-ping-service.py => dirac_framework_ping_service.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-framework-self-ping.py => dirac_framework_self_ping.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-repo-monitor.py => dirac_repo_monitor.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-utils-file-adler.py => dirac_utils_file_adler.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-utils-file-md5.py => dirac_utils_file_md5.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-get-normalized-queue-length.py => dirac_wms_get_normalized_queue_length.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-get-queue-normalization.py => dirac_wms_get_queue_normalization.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-attributes.py => dirac_wms_job_attributes.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-delete.py => dirac_wms_job_delete.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-get-input.py => dirac_wms_job_get_input.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-get-jdl.py => dirac_wms_job_get_jdl.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-get-output.py => dirac_wms_job_get_output.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-get-output-data.py => dirac_wms_job_get_output_data.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-kill.py => dirac_wms_job_kill.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-logging-info.py => dirac_wms_job_logging_info.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-parameters.py => dirac_wms_job_parameters.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-peek.py => dirac_wms_job_peek.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-reschedule.py => dirac_wms_job_reschedule.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-status.py => dirac_wms_job_status.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-job-submit.py => dirac_wms_job_submit.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-jobs-select-output-search.py => dirac_wms_jobs_select_output_search.py} (100%) rename src/DIRAC/Interfaces/scripts/{dirac-wms-select-jobs.py => dirac_wms_select_jobs.py} (100%) rename src/DIRAC/ProductionSystem/scripts/{dirac-prod-add-trans.py => dirac_prod_add_trans.py} (100%) rename src/DIRAC/ProductionSystem/scripts/{dirac-prod-clean.py => dirac_prod_clean.py} (100%) rename src/DIRAC/ProductionSystem/scripts/{dirac-prod-delete.py => dirac_prod_delete.py} (100%) rename src/DIRAC/ProductionSystem/scripts/{dirac-prod-get.py => dirac_prod_get.py} (100%) rename src/DIRAC/ProductionSystem/scripts/{dirac-prod-get-all.py => dirac_prod_get_all.py} (100%) rename src/DIRAC/ProductionSystem/scripts/{dirac-prod-get-description.py => dirac_prod_get_description.py} (100%) rename src/DIRAC/ProductionSystem/scripts/{dirac-prod-get-trans.py => dirac_prod_get_trans.py} (100%) rename src/DIRAC/ProductionSystem/scripts/{dirac-prod-start.py => dirac_prod_start.py} (100%) rename src/DIRAC/ProductionSystem/scripts/{dirac-prod-stop.py => dirac_prod_stop.py} (100%) rename src/DIRAC/RequestManagementSystem/scripts/{dirac-rms-list-req-cache.py => dirac_rms_list_req_cache.py} (100%) rename src/DIRAC/RequestManagementSystem/scripts/{dirac-rms-reqdb-summary.py => dirac_rms_reqdb_summary.py} (100%) rename src/DIRAC/RequestManagementSystem/scripts/{dirac-rms-request.py => dirac_rms_request.py} (100%) rename src/DIRAC/ResourceStatusSystem/scripts/{dirac-rss-list-status.py => dirac_rss_list_status.py} (100%) rename src/DIRAC/ResourceStatusSystem/scripts/{dirac-rss-query-db.py => dirac_rss_query_db.py} (100%) rename src/DIRAC/ResourceStatusSystem/scripts/{dirac-rss-query-dtcache.py => dirac_rss_query_dtcache.py} (100%) rename src/DIRAC/ResourceStatusSystem/scripts/{dirac-rss-set-status.py => dirac_rss_set_status.py} (100%) rename src/DIRAC/ResourceStatusSystem/scripts/{dirac-rss-set-token.py => dirac_rss_set_token.py} (100%) rename src/DIRAC/ResourceStatusSystem/scripts/{dirac-rss-sync.py => dirac_rss_sync.py} (100%) rename src/DIRAC/Resources/scripts/{dirac-resource-get-parameters.py => dirac_resource_get_parameters.py} (100%) rename src/DIRAC/Resources/scripts/{dirac-resource-info.py => dirac_resource_info.py} (100%) rename src/DIRAC/StorageManagementSystem/scripts/{dirac-stager-monitor-file.py => dirac_stager_monitor_file.py} (100%) rename src/DIRAC/StorageManagementSystem/scripts/{dirac-stager-monitor-jobs.py => dirac_stager_monitor_jobs.py} (100%) rename src/DIRAC/StorageManagementSystem/scripts/{dirac-stager-monitor-request.py => dirac_stager_monitor_request.py} (100%) rename src/DIRAC/StorageManagementSystem/scripts/{dirac-stager-monitor-requests.py => dirac_stager_monitor_requests.py} (100%) rename src/DIRAC/StorageManagementSystem/scripts/{dirac-stager-show-stats.py => dirac_stager_show_stats.py} (100%) rename src/DIRAC/StorageManagementSystem/scripts/{dirac-stager-stage-files.py => dirac_stager_stage_files.py} (100%) rename src/DIRAC/TransformationSystem/scripts/{dirac-production-runjoblocal.py => dirac_production_runjoblocal.py} (100%) rename src/DIRAC/TransformationSystem/scripts/{dirac-transformation-add-files.py => dirac_transformation_add_files.py} (100%) rename src/DIRAC/TransformationSystem/scripts/{dirac-transformation-archive.py => dirac_transformation_archive.py} (100%) rename src/DIRAC/TransformationSystem/scripts/{dirac-transformation-clean.py => dirac_transformation_clean.py} (100%) rename src/DIRAC/TransformationSystem/scripts/{dirac-transformation-cli.py => dirac_transformation_cli.py} (100%) rename src/DIRAC/TransformationSystem/scripts/{dirac-transformation-get-files.py => dirac_transformation_get_files.py} (100%) rename src/DIRAC/TransformationSystem/scripts/{dirac-transformation-recover-data.py => dirac_transformation_recover_data.py} (100%) rename src/DIRAC/TransformationSystem/scripts/{dirac-transformation-remove-output.py => dirac_transformation_remove_output.py} (100%) rename src/DIRAC/TransformationSystem/scripts/{dirac-transformation-replication.py => dirac_transformation_replication.py} (100%) rename src/DIRAC/TransformationSystem/scripts/{dirac-transformation-verify-outputdata.py => dirac_transformation_verify_outputdata.py} (100%) rename src/DIRAC/WorkloadManagementSystem/scripts/{dirac-admin-kill-pilot.py => dirac_admin_kill_pilot.py} (100%) rename src/DIRAC/WorkloadManagementSystem/scripts/{dirac-admin-pilot-logging-info.py => dirac_admin_pilot_logging_info.py} (100%) rename src/DIRAC/WorkloadManagementSystem/scripts/{dirac-admin-show-task-queues.py => dirac_admin_show_task_queues.py} (100%) rename src/DIRAC/WorkloadManagementSystem/scripts/{dirac-admin-sync-pilot.py => dirac_admin_sync_pilot.py} (100%) rename src/DIRAC/WorkloadManagementSystem/scripts/{dirac-jobexec.py => dirac_jobexec.py} (100%) rename src/DIRAC/WorkloadManagementSystem/scripts/{dirac-wms-cpu-normalization.py => dirac_wms_cpu_normalization.py} (100%) rename src/DIRAC/WorkloadManagementSystem/scripts/{dirac-wms-get-queue-cpu-time.py => dirac_wms_get_queue_cpu_time.py} (100%) rename src/DIRAC/WorkloadManagementSystem/scripts/{dirac-wms-get-wn.py => dirac_wms_get_wn.py} (100%) rename src/DIRAC/WorkloadManagementSystem/scripts/{dirac-wms-get-wn-parameters.py => dirac_wms_get_wn_parameters.py} (100%) rename src/DIRAC/WorkloadManagementSystem/scripts/{dirac-wms-match.py => dirac_wms_match.py} (100%) rename src/DIRAC/WorkloadManagementSystem/scripts/{dirac-wms-pilot-job-info.py => dirac_wms_pilot_job_info.py} (100%) diff --git a/src/DIRAC/AccountingSystem/scripts/dirac-accounting-decode-fileid.py b/src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py similarity index 100% rename from src/DIRAC/AccountingSystem/scripts/dirac-accounting-decode-fileid.py rename to src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py diff --git a/src/DIRAC/AccountingSystem/scripts/dirac-admin-accounting-cli.py b/src/DIRAC/AccountingSystem/scripts/dirac_admin_accounting_cli.py similarity index 100% rename from src/DIRAC/AccountingSystem/scripts/dirac-admin-accounting-cli.py rename to src/DIRAC/AccountingSystem/scripts/dirac_admin_accounting_cli.py diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-resources.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py similarity index 100% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-resources.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-shifter.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_shifter.py similarity index 100% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-shifter.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_shifter.py diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-site.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py similarity index 100% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-site.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-bdii-info.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_bdii_info.py similarity index 100% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-admin-bdii-info.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_admin_bdii_info.py diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-check-config-options.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py similarity index 100% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-admin-check-config-options.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-sort-cs-sites.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py similarity index 100% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-admin-sort-cs-sites.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-voms-sync.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py similarity index 100% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-admin-voms-sync.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-cli.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_cli.py similarity index 100% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-cli.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_cli.py diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-dump-local-cache.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py similarity index 100% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-dump-local-cache.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-shell.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py similarity index 100% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-shell.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py diff --git a/src/DIRAC/Core/Tornado/scripts/tornado-start-CS.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py similarity index 100% rename from src/DIRAC/Core/Tornado/scripts/tornado-start-CS.py rename to src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py diff --git a/src/DIRAC/Core/Tornado/scripts/tornado-start-all.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py similarity index 100% rename from src/DIRAC/Core/Tornado/scripts/tornado-start-all.py rename to src/DIRAC/Core/Tornado/scripts/tornado_start_all.py diff --git a/src/DIRAC/Core/scripts/dirac-agent.py b/src/DIRAC/Core/scripts/dirac_agent.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-agent.py rename to src/DIRAC/Core/scripts/dirac_agent.py diff --git a/src/DIRAC/Core/scripts/dirac-configure.py b/src/DIRAC/Core/scripts/dirac_configure.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-configure.py rename to src/DIRAC/Core/scripts/dirac_configure.py diff --git a/src/DIRAC/Core/scripts/dirac-deploy-scripts.py b/src/DIRAC/Core/scripts/dirac_deploy_scripts.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-deploy-scripts.py rename to src/DIRAC/Core/scripts/dirac_deploy_scripts.py diff --git a/src/DIRAC/Core/scripts/dirac-executor.py b/src/DIRAC/Core/scripts/dirac_executor.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-executor.py rename to src/DIRAC/Core/scripts/dirac_executor.py diff --git a/src/DIRAC/Core/scripts/dirac-externals-requirements.py b/src/DIRAC/Core/scripts/dirac_externals_requirements.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-externals-requirements.py rename to src/DIRAC/Core/scripts/dirac_externals_requirements.py diff --git a/src/DIRAC/Core/scripts/dirac-install.py b/src/DIRAC/Core/scripts/dirac_install.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-install.py rename to src/DIRAC/Core/scripts/dirac_install.py diff --git a/src/DIRAC/Core/scripts/dirac-install-db.py b/src/DIRAC/Core/scripts/dirac_install_db.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-install-db.py rename to src/DIRAC/Core/scripts/dirac_install_db.py diff --git a/src/DIRAC/Core/scripts/dirac-install-extension.py b/src/DIRAC/Core/scripts/dirac_install_extension.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-install-extension.py rename to src/DIRAC/Core/scripts/dirac_install_extension.py diff --git a/src/DIRAC/Core/scripts/dirac-install-web-portal.py b/src/DIRAC/Core/scripts/dirac_install_web_portal.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-install-web-portal.py rename to src/DIRAC/Core/scripts/dirac_install_web_portal.py diff --git a/src/DIRAC/Core/scripts/dirac-platform.py b/src/DIRAC/Core/scripts/dirac_platform.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-platform.py rename to src/DIRAC/Core/scripts/dirac_platform.py diff --git a/src/DIRAC/Core/scripts/dirac-service.py b/src/DIRAC/Core/scripts/dirac_service.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-service.py rename to src/DIRAC/Core/scripts/dirac_service.py diff --git a/src/DIRAC/Core/scripts/dirac-setup-site.py b/src/DIRAC/Core/scripts/dirac_setup_site.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-setup-site.py rename to src/DIRAC/Core/scripts/dirac_setup_site.py diff --git a/src/DIRAC/Core/scripts/dirac-version.py b/src/DIRAC/Core/scripts/dirac_version.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-version.py rename to src/DIRAC/Core/scripts/dirac_version.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-admin-allow-se.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-admin-allow-se.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-admin-ban-se.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-admin-ban-se.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-admin-user-quota.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-admin-user-quota.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-add-file.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-add-file.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-catalog-metadata.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_catalog_metadata.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-catalog-metadata.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_catalog_metadata.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-change-replica-status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-change-replica-status.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-clean-directory.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-clean-directory.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-archive-request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-archive-request.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-moving-request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-moving-request.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-removal-request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-removal-request.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-data-size.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-data-size.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-directory-sync.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-directory-sync.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-filecatalog-cli.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-filecatalog-cli.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-find-lfns.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-find-lfns.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-move-replica-request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-move-replica-request.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-protocol-matrix.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-protocol-matrix.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-put-and-register-request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-put-and-register-request.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-catalog-files.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-catalog-files.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-catalog-replicas.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-catalog-replicas.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-files.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-files.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-replicas.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_replicas.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-replicas.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_replicas.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-replica-metadata.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-replica-metadata.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-replicate-and-register-request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-replicate-and-register-request.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-resolve-guid.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_resolve_guid.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-resolve-guid.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_resolve_guid.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-set-replica-status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-set-replica-status.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-show-se-status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-show-se-status.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-user-lfns.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-user-lfns.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-user-quota.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py similarity index 100% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-user-quota.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-get-CAs.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_CAs.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-admin-get-CAs.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_CAs.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-get-proxy.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-admin-get-proxy.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-proxy-upload.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_proxy_upload.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-admin-proxy-upload.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_admin_proxy_upload.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-sysadmin-cli.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_sysadmin_cli.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-admin-sysadmin-cli.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_admin_sysadmin_cli.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-update-instance.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_instance.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-admin-update-instance.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_instance.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-update-pilot.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-admin-update-pilot.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-users-with-proxy.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-admin-users-with-proxy.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-install-component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-install-component.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-install-tornado-service.py b/src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-install-tornado-service.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-monitoring-get-components-status.py b/src/DIRAC/FrameworkSystem/scripts/dirac_monitoring_get_components_status.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-monitoring-get-components-status.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_monitoring_get_components_status.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-myproxy-upload.py b/src/DIRAC/FrameworkSystem/scripts/dirac_myproxy_upload.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-myproxy-upload.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_myproxy_upload.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-populate-component-db.py b/src/DIRAC/FrameworkSystem/scripts/dirac_populate_component_db.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-populate-component-db.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_populate_component_db.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-destroy.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_destroy.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-proxy-destroy.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_proxy_destroy.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-get-uploaded-info.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_get_uploaded_info.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-proxy-get-uploaded-info.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_proxy_get_uploaded_info.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-restart-component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_restart_component.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-restart-component.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_restart_component.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-start-component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_start_component.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-start-component.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_start_component.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-status-component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_status_component.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-status-component.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_status_component.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-stop-component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_stop_component.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-stop-component.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_stop_component.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-sys-sendmail.py b/src/DIRAC/FrameworkSystem/scripts/dirac_sys_sendmail.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-sys-sendmail.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_sys_sendmail.py diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-uninstall-component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_uninstall_component.py similarity index 100% rename from src/DIRAC/FrameworkSystem/scripts/dirac-uninstall-component.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_uninstall_component.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-add-group.py b/src/DIRAC/Interfaces/scripts/dirac_admin_add_group.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-add-group.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_add_group.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-add-host.py b/src/DIRAC/Interfaces/scripts/dirac_admin_add_host.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-add-host.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_add_host.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-add-user.py b/src/DIRAC/Interfaces/scripts/dirac_admin_add_user.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-add-user.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_add_user.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-allow-site.py b/src/DIRAC/Interfaces/scripts/dirac_admin_allow_site.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-allow-site.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_allow_site.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-ban-site.py b/src/DIRAC/Interfaces/scripts/dirac_admin_ban_site.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-ban-site.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_ban_site.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-ce-info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_ce_info.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-ce-info.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_ce_info.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-delete-user.py b/src/DIRAC/Interfaces/scripts/dirac_admin_delete_user.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-delete-user.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_delete_user.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-banned-sites.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_banned_sites.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-get-banned-sites.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_get_banned_sites.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-job-pilot-output.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-get-job-pilot-output.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-job-pilots.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-get-job-pilots.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-info.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-logging-info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-logging-info.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-output.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-output.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-site-mask.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-get-site-mask.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-list-hosts.py b/src/DIRAC/Interfaces/scripts/dirac_admin_list_hosts.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-list-hosts.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_list_hosts.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-list-users.py b/src/DIRAC/Interfaces/scripts/dirac_admin_list_users.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-list-users.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_list_users.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-modify-user.py b/src/DIRAC/Interfaces/scripts/dirac_admin_modify_user.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-modify-user.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_modify_user.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-pilot-summary.py b/src/DIRAC/Interfaces/scripts/dirac_admin_pilot_summary.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-pilot-summary.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_pilot_summary.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-reset-job.py b/src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-reset-job.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-service-ports.py b/src/DIRAC/Interfaces/scripts/dirac_admin_service_ports.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-service-ports.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_service_ports.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-set-site-protocols.py b/src/DIRAC/Interfaces/scripts/dirac_admin_set_site_protocols.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-set-site-protocols.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_set_site_protocols.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-site-info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-site-info.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-site-mask-logging.py b/src/DIRAC/Interfaces/scripts/dirac_admin_site_mask_logging.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-site-mask-logging.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_site_mask_logging.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-sync-users-from-file.py b/src/DIRAC/Interfaces/scripts/dirac_admin_sync_users_from_file.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-admin-sync-users-from-file.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_sync_users_from_file.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-get-file.py b/src/DIRAC/Interfaces/scripts/dirac_dms_get_file.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-dms-get-file.py rename to src/DIRAC/Interfaces/scripts/dirac_dms_get_file.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-lfn-accessURL.py b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_accessURL.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-dms-lfn-accessURL.py rename to src/DIRAC/Interfaces/scripts/dirac_dms_lfn_accessURL.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-lfn-metadata.py b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_metadata.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-dms-lfn-metadata.py rename to src/DIRAC/Interfaces/scripts/dirac_dms_lfn_metadata.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-lfn-replicas.py b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_replicas.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-dms-lfn-replicas.py rename to src/DIRAC/Interfaces/scripts/dirac_dms_lfn_replicas.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-pfn-accessURL.py b/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_accessURL.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-dms-pfn-accessURL.py rename to src/DIRAC/Interfaces/scripts/dirac_dms_pfn_accessURL.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-pfn-metadata.py b/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_metadata.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-dms-pfn-metadata.py rename to src/DIRAC/Interfaces/scripts/dirac_dms_pfn_metadata.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-replicate-lfn.py b/src/DIRAC/Interfaces/scripts/dirac_dms_replicate_lfn.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-dms-replicate-lfn.py rename to src/DIRAC/Interfaces/scripts/dirac_dms_replicate_lfn.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-framework-ping-service.py b/src/DIRAC/Interfaces/scripts/dirac_framework_ping_service.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-framework-ping-service.py rename to src/DIRAC/Interfaces/scripts/dirac_framework_ping_service.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-framework-self-ping.py b/src/DIRAC/Interfaces/scripts/dirac_framework_self_ping.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-framework-self-ping.py rename to src/DIRAC/Interfaces/scripts/dirac_framework_self_ping.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-repo-monitor.py b/src/DIRAC/Interfaces/scripts/dirac_repo_monitor.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-repo-monitor.py rename to src/DIRAC/Interfaces/scripts/dirac_repo_monitor.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-utils-file-adler.py b/src/DIRAC/Interfaces/scripts/dirac_utils_file_adler.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-utils-file-adler.py rename to src/DIRAC/Interfaces/scripts/dirac_utils_file_adler.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-utils-file-md5.py b/src/DIRAC/Interfaces/scripts/dirac_utils_file_md5.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-utils-file-md5.py rename to src/DIRAC/Interfaces/scripts/dirac_utils_file_md5.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-get-normalized-queue-length.py b/src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-get-normalized-queue-length.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-get-queue-normalization.py b/src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-get-queue-normalization.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-attributes.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-attributes.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-delete.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_delete.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-delete.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_delete.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-input.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-get-input.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-jdl.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-get-jdl.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-output.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-get-output.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-output-data.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-get-output-data.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-kill.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_kill.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-kill.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_kill.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-logging-info.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-logging-info.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-parameters.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-parameters.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-peek.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-peek.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-reschedule.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_reschedule.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-reschedule.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_reschedule.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-status.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_status.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-status.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_status.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-submit.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-submit.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-jobs-select-output-search.py b/src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-jobs-select-output-search.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-select-jobs.py b/src/DIRAC/Interfaces/scripts/dirac_wms_select_jobs.py similarity index 100% rename from src/DIRAC/Interfaces/scripts/dirac-wms-select-jobs.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_select_jobs.py diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-add-trans.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_add_trans.py similarity index 100% rename from src/DIRAC/ProductionSystem/scripts/dirac-prod-add-trans.py rename to src/DIRAC/ProductionSystem/scripts/dirac_prod_add_trans.py diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-clean.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_clean.py similarity index 100% rename from src/DIRAC/ProductionSystem/scripts/dirac-prod-clean.py rename to src/DIRAC/ProductionSystem/scripts/dirac_prod_clean.py diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-delete.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_delete.py similarity index 100% rename from src/DIRAC/ProductionSystem/scripts/dirac-prod-delete.py rename to src/DIRAC/ProductionSystem/scripts/dirac_prod_delete.py diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-get.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get.py similarity index 100% rename from src/DIRAC/ProductionSystem/scripts/dirac-prod-get.py rename to src/DIRAC/ProductionSystem/scripts/dirac_prod_get.py diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-get-all.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py similarity index 100% rename from src/DIRAC/ProductionSystem/scripts/dirac-prod-get-all.py rename to src/DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-get-description.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_description.py similarity index 100% rename from src/DIRAC/ProductionSystem/scripts/dirac-prod-get-description.py rename to src/DIRAC/ProductionSystem/scripts/dirac_prod_get_description.py diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-get-trans.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py similarity index 100% rename from src/DIRAC/ProductionSystem/scripts/dirac-prod-get-trans.py rename to src/DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-start.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_start.py similarity index 100% rename from src/DIRAC/ProductionSystem/scripts/dirac-prod-start.py rename to src/DIRAC/ProductionSystem/scripts/dirac_prod_start.py diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-stop.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_stop.py similarity index 100% rename from src/DIRAC/ProductionSystem/scripts/dirac-prod-stop.py rename to src/DIRAC/ProductionSystem/scripts/dirac_prod_stop.py diff --git a/src/DIRAC/RequestManagementSystem/scripts/dirac-rms-list-req-cache.py b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_list_req_cache.py similarity index 100% rename from src/DIRAC/RequestManagementSystem/scripts/dirac-rms-list-req-cache.py rename to src/DIRAC/RequestManagementSystem/scripts/dirac_rms_list_req_cache.py diff --git a/src/DIRAC/RequestManagementSystem/scripts/dirac-rms-reqdb-summary.py b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_reqdb_summary.py similarity index 100% rename from src/DIRAC/RequestManagementSystem/scripts/dirac-rms-reqdb-summary.py rename to src/DIRAC/RequestManagementSystem/scripts/dirac_rms_reqdb_summary.py diff --git a/src/DIRAC/RequestManagementSystem/scripts/dirac-rms-request.py b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_request.py similarity index 100% rename from src/DIRAC/RequestManagementSystem/scripts/dirac-rms-request.py rename to src/DIRAC/RequestManagementSystem/scripts/dirac_rms_request.py diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-list-status.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py similarity index 100% rename from src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-list-status.py rename to src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-query-db.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py similarity index 100% rename from src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-query-db.py rename to src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-query-dtcache.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py similarity index 100% rename from src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-query-dtcache.py rename to src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-set-status.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py similarity index 100% rename from src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-set-status.py rename to src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-set-token.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py similarity index 100% rename from src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-set-token.py rename to src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-sync.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py similarity index 100% rename from src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-sync.py rename to src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py diff --git a/src/DIRAC/Resources/scripts/dirac-resource-get-parameters.py b/src/DIRAC/Resources/scripts/dirac_resource_get_parameters.py similarity index 100% rename from src/DIRAC/Resources/scripts/dirac-resource-get-parameters.py rename to src/DIRAC/Resources/scripts/dirac_resource_get_parameters.py diff --git a/src/DIRAC/Resources/scripts/dirac-resource-info.py b/src/DIRAC/Resources/scripts/dirac_resource_info.py similarity index 100% rename from src/DIRAC/Resources/scripts/dirac-resource-info.py rename to src/DIRAC/Resources/scripts/dirac_resource_info.py diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-file.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_file.py similarity index 100% rename from src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-file.py rename to src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_file.py diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-jobs.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_jobs.py similarity index 100% rename from src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-jobs.py rename to src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_jobs.py diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-request.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_request.py similarity index 100% rename from src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-request.py rename to src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_request.py diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-requests.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py similarity index 100% rename from src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-requests.py rename to src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-show-stats.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_show_stats.py similarity index 100% rename from src/DIRAC/StorageManagementSystem/scripts/dirac-stager-show-stats.py rename to src/DIRAC/StorageManagementSystem/scripts/dirac_stager_show_stats.py diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-stage-files.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_stage_files.py similarity index 100% rename from src/DIRAC/StorageManagementSystem/scripts/dirac-stager-stage-files.py rename to src/DIRAC/StorageManagementSystem/scripts/dirac_stager_stage_files.py diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-production-runjoblocal.py b/src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py similarity index 100% rename from src/DIRAC/TransformationSystem/scripts/dirac-production-runjoblocal.py rename to src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-add-files.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_add_files.py similarity index 100% rename from src/DIRAC/TransformationSystem/scripts/dirac-transformation-add-files.py rename to src/DIRAC/TransformationSystem/scripts/dirac_transformation_add_files.py diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-archive.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py similarity index 100% rename from src/DIRAC/TransformationSystem/scripts/dirac-transformation-archive.py rename to src/DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-clean.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py similarity index 100% rename from src/DIRAC/TransformationSystem/scripts/dirac-transformation-clean.py rename to src/DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-cli.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_cli.py similarity index 100% rename from src/DIRAC/TransformationSystem/scripts/dirac-transformation-cli.py rename to src/DIRAC/TransformationSystem/scripts/dirac_transformation_cli.py diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-get-files.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_get_files.py similarity index 100% rename from src/DIRAC/TransformationSystem/scripts/dirac-transformation-get-files.py rename to src/DIRAC/TransformationSystem/scripts/dirac_transformation_get_files.py diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-recover-data.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_recover_data.py similarity index 100% rename from src/DIRAC/TransformationSystem/scripts/dirac-transformation-recover-data.py rename to src/DIRAC/TransformationSystem/scripts/dirac_transformation_recover_data.py diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-remove-output.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py similarity index 100% rename from src/DIRAC/TransformationSystem/scripts/dirac-transformation-remove-output.py rename to src/DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-replication.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_replication.py similarity index 100% rename from src/DIRAC/TransformationSystem/scripts/dirac-transformation-replication.py rename to src/DIRAC/TransformationSystem/scripts/dirac_transformation_replication.py diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-verify-outputdata.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_verify_outputdata.py similarity index 100% rename from src/DIRAC/TransformationSystem/scripts/dirac-transformation-verify-outputdata.py rename to src/DIRAC/TransformationSystem/scripts/dirac_transformation_verify_outputdata.py diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-kill-pilot.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_kill_pilot.py similarity index 100% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-kill-pilot.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_kill_pilot.py diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-pilot-logging-info.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_pilot_logging_info.py similarity index 100% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-pilot-logging-info.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_pilot_logging_info.py diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-show-task-queues.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py similarity index 100% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-show-task-queues.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-sync-pilot.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_sync_pilot.py similarity index 100% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-sync-pilot.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_sync_pilot.py diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-jobexec.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py similarity index 100% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-jobexec.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-cpu-normalization.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_cpu_normalization.py similarity index 100% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-cpu-normalization.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_cpu_normalization.py diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-queue-cpu-time.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_queue_cpu_time.py similarity index 100% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-queue-cpu-time.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_queue_cpu_time.py diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-wn.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn.py similarity index 100% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-wn.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn.py diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-wn-parameters.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn_parameters.py similarity index 100% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-wn-parameters.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn_parameters.py diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-match.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_match.py similarity index 100% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-match.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_match.py diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-pilot-job-info.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py similarity index 100% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-pilot-job-info.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py From d5ec9ff4f591f9c545384ade74eda4ea53b001a2 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 13 Jan 2021 20:10:10 +0100 Subject: [PATCH 16/27] PY3: Convery most scripts to use DiracScript decorator --- setup.cfg | 179 +++- .../scripts/dirac_accounting_decode_fileid.py | 56 +- .../scripts/dirac_admin_accounting_cli.py | 21 +- .../scripts/dirac_admin_add_resources.py | 20 +- .../scripts/dirac_admin_add_shifter.py | 8 +- .../scripts/dirac_admin_add_site.py | 9 +- .../scripts/dirac_admin_bdii_info.py | 12 +- .../dirac_admin_check_config_options.py | 8 +- .../scripts/dirac_admin_sort_cs_sites.py | 179 ++-- .../scripts/dirac_admin_voms_sync.py | 131 +-- .../scripts/dirac_configuration_cli.py | 22 +- .../dirac_configuration_dump_local_cache.py | 56 +- .../scripts/dirac_configuration_shell.py | 10 +- .../{dirac_install.py => dirac-install.py} | 0 src/DIRAC/Core/scripts/dirac_agent.py | 62 +- src/DIRAC/Core/scripts/dirac_configure.py | 648 ++++++------ src/DIRAC/Core/scripts/dirac_executor.py | 82 +- .../scripts/dirac_externals_requirements.py | 106 +- src/DIRAC/Core/scripts/dirac_install_db.py | 71 +- .../Core/scripts/dirac_install_web_portal.py | 23 +- src/DIRAC/Core/scripts/dirac_service.py | 76 +- src/DIRAC/Core/scripts/dirac_setup_site.py | 97 +- src/DIRAC/Core/scripts/dirac_version.py | 11 +- .../scripts/dirac_admin_allow_se.py | 352 +++---- .../scripts/dirac_admin_ban_se.py | 414 ++++---- .../scripts/dirac_admin_user_quota.py | 68 +- .../scripts/dirac_dms_add_file.py | 142 +-- .../scripts/dirac_dms_catalog_metadata.py | 111 +- .../dirac_dms_change_replica_status.py | 104 +- .../scripts/dirac_dms_clean_directory.py | 82 +- .../dirac_dms_create_archive_request.py | 8 +- .../dirac_dms_create_moving_request.py | 8 +- .../dirac_dms_create_removal_request.py | 194 ++-- .../scripts/dirac_dms_data_size.py | 102 +- .../scripts/dirac_dms_directory_sync.py | 973 +++++++++--------- .../scripts/dirac_dms_filecatalog_cli.py | 105 +- .../scripts/dirac_dms_find_lfns.py | 8 +- .../scripts/dirac_dms_move_replica_request.py | 27 +- .../scripts/dirac_dms_protocol_matrix.py | 23 +- .../dirac_dms_put_and_register_request.py | 30 +- .../scripts/dirac_dms_remove_catalog_files.py | 111 +- .../dirac_dms_remove_catalog_replicas.py | 110 +- .../scripts/dirac_dms_remove_files.py | 104 +- .../scripts/dirac_dms_remove_replicas.py | 8 +- .../scripts/dirac_dms_replica_metadata.py | 94 +- ...irac_dms_replicate_and_register_request.py | 39 +- .../scripts/dirac_dms_resolve_guid.py | 68 +- .../scripts/dirac_dms_set_replica_status.py | 112 +- .../scripts/dirac_dms_show_se_status.py | 111 +- .../scripts/dirac_dms_user_lfns.py | 273 ++--- .../scripts/dirac_dms_user_quota.py | 64 +- .../scripts/dirac_admin_get_CAs.py | 61 +- .../scripts/dirac_admin_get_proxy.py | 137 +-- .../scripts/dirac_admin_proxy_upload.py | 9 +- .../scripts/dirac_admin_sysadmin_cli.py | 26 +- .../scripts/dirac_admin_update_instance.py | 420 ++++---- .../scripts/dirac_admin_update_pilot.py | 126 +-- .../scripts/dirac_admin_users_with_proxy.py | 69 +- .../scripts/dirac_install_component.py | 153 +-- .../scripts/dirac_install_tornado_service.py | 124 +-- .../dirac_monitoring_get_components_status.py | 83 +- .../scripts/dirac_myproxy_upload.py | 52 +- .../scripts/dirac_populate_component_db.py | 317 +++--- .../scripts/dirac_proxy_destroy.py | 12 +- .../scripts/dirac_proxy_get_uploaded_info.py | 119 ++- .../scripts/dirac_restart_component.py | 67 +- .../scripts/dirac_start_component.py | 69 +- .../scripts/dirac_status_component.py | 64 +- .../scripts/dirac_stop_component.py | 77 +- .../scripts/dirac_sys_sendmail.py | 90 +- .../scripts/dirac_uninstall_component.py | 122 ++- .../scripts/dirac_admin_add_group.py | 124 ++- .../scripts/dirac_admin_add_host.py | 135 +-- .../scripts/dirac_admin_add_user.py | 111 +- .../scripts/dirac_admin_allow_site.py | 147 +-- .../scripts/dirac_admin_ban_site.py | 149 +-- .../Interfaces/scripts/dirac_admin_ce_info.py | 45 +- .../scripts/dirac_admin_delete_user.py | 84 +- .../scripts/dirac_admin_get_banned_sites.py | 37 +- .../dirac_admin_get_job_pilot_output.py | 78 +- .../scripts/dirac_admin_get_job_pilots.py | 78 +- .../scripts/dirac_admin_get_pilot_info.py | 139 +-- .../dirac_admin_get_pilot_logging_info.py | 72 +- .../scripts/dirac_admin_get_pilot_output.py | 51 +- .../scripts/dirac_admin_get_site_mask.py | 40 +- .../scripts/dirac_admin_list_hosts.py | 52 +- .../scripts/dirac_admin_list_users.py | 95 +- .../scripts/dirac_admin_modify_user.py | 100 +- .../scripts/dirac_admin_pilot_summary.py | 28 +- .../scripts/dirac_admin_reset_job.py | 66 +- .../scripts/dirac_admin_service_ports.py | 48 +- .../scripts/dirac_admin_set_site_protocols.py | 62 +- .../scripts/dirac_admin_site_info.py | 50 +- .../scripts/dirac_admin_site_mask_logging.py | 50 +- .../dirac_admin_sync_users_from_file.py | 85 +- .../Interfaces/scripts/dirac_dms_get_file.py | 64 +- .../scripts/dirac_dms_lfn_accessURL.py | 90 +- .../scripts/dirac_dms_lfn_metadata.py | 66 +- .../scripts/dirac_dms_lfn_replicas.py | 80 +- .../scripts/dirac_dms_pfn_accessURL.py | 76 +- .../scripts/dirac_dms_pfn_metadata.py | 80 +- .../scripts/dirac_dms_replicate_lfn.py | 90 +- .../scripts/dirac_framework_ping_service.py | 81 +- .../scripts/dirac_framework_self_ping.py | 36 +- .../Interfaces/scripts/dirac_repo_monitor.py | 44 +- .../scripts/dirac_utils_file_adler.py | 50 +- .../scripts/dirac_utils_file_md5.py | 54 +- .../dirac_wms_get_normalized_queue_length.py | 49 +- .../dirac_wms_get_queue_normalization.py | 46 +- .../scripts/dirac_wms_job_attributes.py | 50 +- .../scripts/dirac_wms_job_delete.py | 27 +- .../scripts/dirac_wms_job_get_input.py | 80 +- .../scripts/dirac_wms_job_get_jdl.py | 66 +- .../scripts/dirac_wms_job_get_output.py | 178 ++-- .../scripts/dirac_wms_job_get_output_data.py | 78 +- .../Interfaces/scripts/dirac_wms_job_kill.py | 44 +- .../scripts/dirac_wms_job_logging_info.py | 50 +- .../scripts/dirac_wms_job_parameters.py | 50 +- .../Interfaces/scripts/dirac_wms_job_peek.py | 50 +- .../scripts/dirac_wms_job_reschedule.py | 64 +- .../scripts/dirac_wms_job_status.py | 92 +- .../scripts/dirac_wms_job_submit.py | 124 +-- .../dirac_wms_jobs_select_output_search.py | 185 ++-- .../scripts/dirac_wms_select_jobs.py | 206 ++-- .../scripts/dirac_prod_add_trans.py | 104 +- .../scripts/dirac_prod_clean.py | 45 +- .../scripts/dirac_prod_delete.py | 49 +- .../scripts/dirac_prod_get.py | 97 +- .../scripts/dirac_prod_get_all.py | 90 +- .../scripts/dirac_prod_get_description.py | 55 +- .../scripts/dirac_prod_get_trans.py | 156 +-- .../scripts/dirac_prod_start.py | 49 +- .../scripts/dirac_prod_stop.py | 49 +- .../scripts/dirac_rms_list_req_cache.py | 17 +- .../scripts/dirac_rms_reqdb_summary.py | 15 +- .../scripts/dirac_rms_request.py | 60 +- .../scripts/dirac_rss_list_status.py | 9 +- .../scripts/dirac_rss_query_db.py | 10 +- .../scripts/dirac_rss_query_dtcache.py | 13 +- .../scripts/dirac_rss_set_status.py | 10 +- .../scripts/dirac_rss_set_token.py | 22 +- .../scripts/dirac_rss_sync.py | 9 +- .../scripts/dirac_stager_monitor_file.py | 139 +-- .../scripts/dirac_stager_monitor_jobs.py | 110 +- .../scripts/dirac_stager_monitor_request.py | 98 +- .../scripts/dirac_stager_monitor_requests.py | 299 +++--- .../scripts/dirac_stager_show_stats.py | 88 +- .../scripts/dirac_stager_stage_files.py | 100 +- .../scripts/dirac_production_runjoblocal.py | 60 +- .../scripts/dirac_transformation_add_files.py | 98 +- .../scripts/dirac_transformation_archive.py | 39 +- .../scripts/dirac_transformation_clean.py | 39 +- .../scripts/dirac_transformation_cli.py | 16 +- .../scripts/dirac_transformation_get_files.py | 41 +- .../dirac_transformation_recover_data.py | 9 +- .../dirac_transformation_remove_output.py | 39 +- .../dirac_transformation_verify_outputdata.py | 39 +- .../scripts/dirac_admin_kill_pilot.py | 42 +- .../scripts/dirac_admin_pilot_logging_info.py | 45 +- .../scripts/dirac_admin_show_task_queues.py | 121 ++- .../scripts/dirac_admin_sync_pilot.py | 107 +- .../scripts/dirac_jobexec.py | 166 +-- .../scripts/dirac_wms_cpu_normalization.py | 95 +- .../scripts/dirac_wms_get_queue_cpu_time.py | 29 +- .../scripts/dirac_wms_get_wn.py | 8 +- .../scripts/dirac_wms_get_wn_parameters.py | 47 +- .../scripts/dirac_wms_match.py | 54 +- .../scripts/dirac_wms_pilot_job_info.py | 115 ++- 168 files changed, 8192 insertions(+), 6815 deletions(-) rename src/DIRAC/Core/scripts/{dirac_install.py => dirac-install.py} (100%) diff --git a/setup.cfg b/setup.cfg index 73d69f659d5..bf1d9871735 100644 --- a/setup.cfg +++ b/setup.cfg @@ -77,10 +77,187 @@ dirac = # For details on adding additional scripts to vanilla DIRAC or external extensions see # https://dirac.readthedocs.io/en/integration/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.html console_scripts = + # AccountingSystem + dirac-accounting-decode-fileid = DIRAC.AccountingSystem.scripts.dirac_accounting_decode_fileid:main + dirac-admin-accounting-cli = DIRAC.AccountingSystem.scripts.dirac_admin_accounting_cli:main + # ConfigurationSystem + dirac-admin-add-resources = DIRAC.ConfigurationSystem.scripts.dirac_admin_add_resources:main + dirac-admin-add-shifter = DIRAC.ConfigurationSystem.scripts.dirac_admin_add_shifter:main + dirac-admin-add-site = DIRAC.ConfigurationSystem.scripts.dirac_admin_add_site:main + dirac-admin-bdii-info = DIRAC.ConfigurationSystem.scripts.dirac_admin_bdii_info:main + dirac-admin-check-config-options = DIRAC.ConfigurationSystem.scripts.dirac_admin_check_config_options:main + dirac-admin-sort-cs-sites = DIRAC.ConfigurationSystem.scripts.dirac_admin_sort_cs_sites:main + dirac-admin-voms-sync = DIRAC.ConfigurationSystem.scripts.dirac_admin_voms_sync:main + dirac-configuration-cli = DIRAC.ConfigurationSystem.scripts.dirac_configuration_cli:main + dirac-configuration-dump-local-cache = DIRAC.ConfigurationSystem.scripts.dirac_configuration_dump_local_cache:main + dirac-configuration-shell = DIRAC.ConfigurationSystem.scripts.dirac_configuration_shell:main + # Core + dirac-agent = DIRAC.Core.scripts.dirac_agent:main + dirac-configure = DIRAC.Core.scripts.dirac_configure:main + dirac-executor = DIRAC.Core.scripts.dirac_executor:main + dirac-externals-requirements = DIRAC.Core.scripts.dirac_externals_requirements:main dirac-info = DIRAC.Core.scripts.dirac_info:main + dirac-install-db = DIRAC.Core.scripts.dirac_install_db:main + dirac-install-web-portal = DIRAC.Core.scripts.dirac_install_web_portal:main + dirac-service = DIRAC.Core.scripts.dirac_service:main + dirac-setup-site = DIRAC.Core.scripts.dirac_setup_site:main + dirac-version = DIRAC.Core.scripts.dirac_version:main + # DataManagementSystem + dirac-admin-allow-se = DIRAC.DataManagementSystem.scripts.dirac_admin_allow_se:main + dirac-admin-ban-se = DIRAC.DataManagementSystem.scripts.dirac_admin_ban_se:main + dirac-admin-user-quota = DIRAC.DataManagementSystem.scripts.dirac_admin_user_quota:main + dirac-dms-add-file = DIRAC.DataManagementSystem.scripts.dirac_dms_add_file:main + dirac-dms-catalog-metadata = DIRAC.DataManagementSystem.scripts.dirac_dms_catalog_metadata:main + dirac-dms-change-replica-status = DIRAC.DataManagementSystem.scripts.dirac_dms_change_replica_status:main + dirac-dms-clean-directory = DIRAC.DataManagementSystem.scripts.dirac_dms_clean_directory:main + dirac-dms-create-archive-request = DIRAC.DataManagementSystem.scripts.dirac_dms_create_archive_request:main + dirac-dms-create-moving-request = DIRAC.DataManagementSystem.scripts.dirac_dms_create_moving_request:main + dirac-dms-create-removal-request = DIRAC.DataManagementSystem.scripts.dirac_dms_create_removal_request:main + dirac-dms-data-size = DIRAC.DataManagementSystem.scripts.dirac_dms_data_size:main + dirac-dms-directory-sync = DIRAC.DataManagementSystem.scripts.dirac_dms_directory_sync:main + dirac-dms-filecatalog-cli = DIRAC.DataManagementSystem.scripts.dirac_dms_filecatalog_cli:main + dirac-dms-find-lfns = DIRAC.DataManagementSystem.scripts.dirac_dms_find_lfns:main + dirac-dms-move-replica-request = DIRAC.DataManagementSystem.scripts.dirac_dms_move_replica_request:main + dirac-dms-protocol-matrix = DIRAC.DataManagementSystem.scripts.dirac_dms_protocol_matrix:main + dirac-dms-put-and-register-request = DIRAC.DataManagementSystem.scripts.dirac_dms_put_and_register_request:main + dirac-dms-remove-catalog-files = DIRAC.DataManagementSystem.scripts.dirac_dms_remove_catalog_files:main + dirac-dms-remove-catalog-replicas = DIRAC.DataManagementSystem.scripts.dirac_dms_remove_catalog_replicas:main + dirac-dms-remove-files = DIRAC.DataManagementSystem.scripts.dirac_dms_remove_files:main + dirac-dms-remove-replicas = DIRAC.DataManagementSystem.scripts.dirac_dms_remove_replicas:main + dirac-dms-replica-metadata = DIRAC.DataManagementSystem.scripts.dirac_dms_replica_metadata:main + dirac-dms-replicate-and-register-request = DIRAC.DataManagementSystem.scripts.dirac_dms_replicate_and_register_request:main + dirac-dms-resolve-guid = DIRAC.DataManagementSystem.scripts.dirac_dms_resolve_guid:main + dirac-dms-set-replica-status = DIRAC.DataManagementSystem.scripts.dirac_dms_set_replica_status:main + dirac-dms-show-se-status = DIRAC.DataManagementSystem.scripts.dirac_dms_show_se_status:main + dirac-dms-user-lfns = DIRAC.DataManagementSystem.scripts.dirac_dms_user_lfns:main + dirac-dms-user-quota = DIRAC.DataManagementSystem.scripts.dirac_dms_user_quota:main + # FrameworkSystem + dirac-admin-get-CAs = DIRAC.FrameworkSystem.scripts.dirac_admin_get_CAs:main + dirac-admin-get-proxy = DIRAC.FrameworkSystem.scripts.dirac_admin_get_proxy:main + dirac-admin-proxy-upload = DIRAC.FrameworkSystem.scripts.dirac_admin_proxy_upload:main + dirac-admin-sysadmin-cli = DIRAC.FrameworkSystem.scripts.dirac_admin_sysadmin_cli:main + dirac-admin-update-instance = DIRAC.FrameworkSystem.scripts.dirac_admin_update_instance:main + dirac-admin-update-pilot = DIRAC.FrameworkSystem.scripts.dirac_admin_update_pilot:main + dirac-admin-users-with-proxy = DIRAC.FrameworkSystem.scripts.dirac_admin_users_with_proxy:main + dirac-install-component = DIRAC.FrameworkSystem.scripts.dirac_install_component:main + dirac-install-tornado-service = DIRAC.FrameworkSystem.scripts.dirac_install_tornado_service:main + dirac-monitoring-get-components-status = DIRAC.FrameworkSystem.scripts.dirac_monitoring_get_components_status:main + dirac-myproxy-upload = DIRAC.FrameworkSystem.scripts.dirac_myproxy_upload:main + dirac-populate-component-db = DIRAC.FrameworkSystem.scripts.dirac_populate_component_db:main + dirac-proxy-destroy = DIRAC.FrameworkSystem.scripts.dirac_proxy_destroy:main + dirac-proxy-get-uploaded-info = DIRAC.FrameworkSystem.scripts.dirac_proxy_get_uploaded_info:main dirac-proxy-info = DIRAC.FrameworkSystem.scripts.dirac_proxy_info:main dirac-proxy-init = DIRAC.FrameworkSystem.scripts.dirac_proxy_init:main - + dirac-restart-component = DIRAC.FrameworkSystem.scripts.dirac_restart_component:main + dirac-start-component = DIRAC.FrameworkSystem.scripts.dirac_start_component:main + dirac-status-component = DIRAC.FrameworkSystem.scripts.dirac_status_component:main + dirac-stop-component = DIRAC.FrameworkSystem.scripts.dirac_stop_component:main + dirac-sys-sendmail = DIRAC.FrameworkSystem.scripts.dirac_sys_sendmail:main + dirac-uninstall-component = DIRAC.FrameworkSystem.scripts.dirac_uninstall_component:main + # Interfaces + dirac-admin-add-group = DIRAC.Interfaces.scripts.dirac_admin_add_group:main + dirac-admin-add-host = DIRAC.Interfaces.scripts.dirac_admin_add_host:main + dirac-admin-add-user = DIRAC.Interfaces.scripts.dirac_admin_add_user:main + dirac-admin-allow-site = DIRAC.Interfaces.scripts.dirac_admin_allow_site:main + dirac-admin-ban-site = DIRAC.Interfaces.scripts.dirac_admin_ban_site:main + dirac-admin-ce-info = DIRAC.Interfaces.scripts.dirac_admin_ce_info:main + dirac-admin-delete-user = DIRAC.Interfaces.scripts.dirac_admin_delete_user:main + dirac-admin-get-banned-sites = DIRAC.Interfaces.scripts.dirac_admin_get_banned_sites:main + dirac-admin-get-job-pilot-output = DIRAC.Interfaces.scripts.dirac_admin_get_job_pilot_output:main + dirac-admin-get-job-pilots = DIRAC.Interfaces.scripts.dirac_admin_get_job_pilots:main + dirac-admin-get-pilot-info = DIRAC.Interfaces.scripts.dirac_admin_get_pilot_info:main + dirac-admin-get-pilot-logging-info = DIRAC.Interfaces.scripts.dirac_admin_get_pilot_logging_info:main + dirac-admin-get-pilot-output = DIRAC.Interfaces.scripts.dirac_admin_get_pilot_output:main + dirac-admin-get-site-mask = DIRAC.Interfaces.scripts.dirac_admin_get_site_mask:main + dirac-admin-list-hosts = DIRAC.Interfaces.scripts.dirac_admin_list_hosts:main + dirac-admin-list-users = DIRAC.Interfaces.scripts.dirac_admin_list_users:main + dirac-admin-modify-user = DIRAC.Interfaces.scripts.dirac_admin_modify_user:main + dirac-admin-pilot-summary = DIRAC.Interfaces.scripts.dirac_admin_pilot_summary:main + dirac-admin-reset-job = DIRAC.Interfaces.scripts.dirac_admin_reset_job:main + dirac-admin-service-ports = DIRAC.Interfaces.scripts.dirac_admin_service_ports:main + dirac-admin-set-site-protocols = DIRAC.Interfaces.scripts.dirac_admin_set_site_protocols:main + dirac-admin-site-info = DIRAC.Interfaces.scripts.dirac_admin_site_info:main + dirac-admin-site-mask-logging = DIRAC.Interfaces.scripts.dirac_admin_site_mask_logging:main + dirac-admin-sync-users-from-file = DIRAC.Interfaces.scripts.dirac_admin_sync_users_from_file:main + dirac-dms-get-file = DIRAC.Interfaces.scripts.dirac_dms_get_file:main + dirac-dms-lfn-accessURL = DIRAC.Interfaces.scripts.dirac_dms_lfn_accessURL:main + dirac-dms-lfn-metadata = DIRAC.Interfaces.scripts.dirac_dms_lfn_metadata:main + dirac-dms-lfn-replicas = DIRAC.Interfaces.scripts.dirac_dms_lfn_replicas:main + dirac-dms-pfn-accessURL = DIRAC.Interfaces.scripts.dirac_dms_pfn_accessURL:main + dirac-dms-pfn-metadata = DIRAC.Interfaces.scripts.dirac_dms_pfn_metadata:main + dirac-dms-replicate-lfn = DIRAC.Interfaces.scripts.dirac_dms_replicate_lfn:main + dirac-framework-ping-service = DIRAC.Interfaces.scripts.dirac_framework_ping_service:main + dirac-framework-self-ping = DIRAC.Interfaces.scripts.dirac_framework_self_ping:main + dirac-repo-monitor = DIRAC.Interfaces.scripts.dirac_repo_monitor:main + dirac-utils-file-adler = DIRAC.Interfaces.scripts.dirac_utils_file_adler:main + dirac-utils-file-md5 = DIRAC.Interfaces.scripts.dirac_utils_file_md5:main + dirac-wms-get-normalized-queue-length = DIRAC.Interfaces.scripts.dirac_wms_get_normalized_queue_length:main + dirac-wms-get-queue-normalization = DIRAC.Interfaces.scripts.dirac_wms_get_queue_normalization:main + dirac-wms-job-attributes = DIRAC.Interfaces.scripts.dirac_wms_job_attributes:main + dirac-wms-job-delete = DIRAC.Interfaces.scripts.dirac_wms_job_delete:main + dirac-wms-job-get-input = DIRAC.Interfaces.scripts.dirac_wms_job_get_input:main + dirac-wms-job-get-jdl = DIRAC.Interfaces.scripts.dirac_wms_job_get_jdl:main + dirac-wms-job-get-output = DIRAC.Interfaces.scripts.dirac_wms_job_get_output:main + dirac-wms-job-get-output-data = DIRAC.Interfaces.scripts.dirac_wms_job_get_output_data:main + dirac-wms-job-kill = DIRAC.Interfaces.scripts.dirac_wms_job_kill:main + dirac-wms-job-logging-info = DIRAC.Interfaces.scripts.dirac_wms_job_logging_info:main + dirac-wms-job-parameters = DIRAC.Interfaces.scripts.dirac_wms_job_parameters:main + dirac-wms-job-peek = DIRAC.Interfaces.scripts.dirac_wms_job_peek:main + dirac-wms-job-reschedule = DIRAC.Interfaces.scripts.dirac_wms_job_reschedule:main + dirac-wms-job-status = DIRAC.Interfaces.scripts.dirac_wms_job_status:main + dirac-wms-job-submit = DIRAC.Interfaces.scripts.dirac_wms_job_submit:main + dirac-wms-jobs-select-output-search = DIRAC.Interfaces.scripts.dirac_wms_jobs_select_output_search:main + dirac-wms-select-jobs = DIRAC.Interfaces.scripts.dirac_wms_select_jobs:main + # ProductionSystem + dirac-prod-add-trans = DIRAC.ProductionSystem.scripts.dirac_prod_add_trans:main + dirac-prod-clean = DIRAC.ProductionSystem.scripts.dirac_prod_clean:main + dirac-prod-delete = DIRAC.ProductionSystem.scripts.dirac_prod_delete:main + dirac-prod-get = DIRAC.ProductionSystem.scripts.dirac_prod_get:main + dirac-prod-get-all = DIRAC.ProductionSystem.scripts.dirac_prod_get_all:main + dirac-prod-get-description = DIRAC.ProductionSystem.scripts.dirac_prod_get_description:main + dirac-prod-get-trans = DIRAC.ProductionSystem.scripts.dirac_prod_get_trans:main + dirac-prod-start = DIRAC.ProductionSystem.scripts.dirac_prod_start:main + dirac-prod-stop = DIRAC.ProductionSystem.scripts.dirac_prod_stop:main + # RequestManagementSystem + dirac-rms-list-req-cache = DIRAC.RequestManagementSystem.scripts.dirac_rms_list_req_cache:main + dirac-rms-reqdb-summary = DIRAC.RequestManagementSystem.scripts.dirac_rms_reqdb_summary:main + dirac-rms-request = DIRAC.RequestManagementSystem.scripts.dirac_rms_request:main + # ResourceStatusSystem + dirac-rss-list-status = DIRAC.ResourceStatusSystem.scripts.dirac_rss_list_status:main + dirac-rss-query-db = DIRAC.ResourceStatusSystem.scripts.dirac_rss_query_db:main + dirac-rss-query-dtcache = DIRAC.ResourceStatusSystem.scripts.dirac_rss_query_dtcache:main + dirac-rss-set-status = DIRAC.ResourceStatusSystem.scripts.dirac_rss_set_status:main + dirac-rss-set-token = DIRAC.ResourceStatusSystem.scripts.dirac_rss_set_token:main + dirac-rss-sync = DIRAC.ResourceStatusSystem.scripts.dirac_rss_sync:main + # StorageManagementSystem + dirac-stager-monitor-file = DIRAC.StorageManagementSystem.scripts.dirac_stager_monitor_file:main + dirac-stager-monitor-jobs = DIRAC.StorageManagementSystem.scripts.dirac_stager_monitor_jobs:main + dirac-stager-monitor-request = DIRAC.StorageManagementSystem.scripts.dirac_stager_monitor_request:main + dirac-stager-monitor-requests = DIRAC.StorageManagementSystem.scripts.dirac_stager_monitor_requests:main + dirac-stager-show-stats = DIRAC.StorageManagementSystem.scripts.dirac_stager_show_stats:main + dirac-stager-stage-files = DIRAC.StorageManagementSystem.scripts.dirac_stager_stage_files:main + # TransformationSystem + dirac-production-runjoblocal = DIRAC.TransformationSystem.scripts.dirac_production_runjoblocal:main + dirac-transformation-add-files = DIRAC.TransformationSystem.scripts.dirac_transformation_add_files:main + dirac-transformation-archive = DIRAC.TransformationSystem.scripts.dirac_transformation_archive:main + dirac-transformation-clean = DIRAC.TransformationSystem.scripts.dirac_transformation_clean:main + dirac-transformation-cli = DIRAC.TransformationSystem.scripts.dirac_transformation_cli:main + dirac-transformation-get-files = DIRAC.TransformationSystem.scripts.dirac_transformation_get_files:main + dirac-transformation-recover-data = DIRAC.TransformationSystem.scripts.dirac_transformation_recover_data:main + dirac-transformation-remove-output = DIRAC.TransformationSystem.scripts.dirac_transformation_remove_output:main + dirac-transformation-verify-outputdata = DIRAC.TransformationSystem.scripts.dirac_transformation_verify_outputdata:main + # WorkloadManagementSystem + dirac-admin-kill-pilot = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_kill_pilot:main + dirac-admin-pilot-logging-info = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_pilot_logging_info:main + dirac-admin-show-task-queues = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_show_task_queues:main + dirac-admin-sync-pilot = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_sync_pilot:main + dirac-jobexec = DIRAC.WorkloadManagementSystem.scripts.dirac_jobexec:main + dirac-wms-cpu-normalization = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_cpu_normalization:main + dirac-wms-get-queue-cpu-time = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_get_queue_cpu_time:main + dirac-wms-get-wn = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_get_wn:main + dirac-wms-get-wn-parameters = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_get_wn_parameters:main + dirac-wms-match = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_match:main + dirac-wms-pilot-job-info = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_pilot_job_info:main ############################################################################### # Linting diff --git a/src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py b/src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py index 0dd71abdacf..3c749e2ce47 100755 --- a/src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py +++ b/src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py @@ -17,29 +17,37 @@ import cgi from DIRAC import gLogger from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Utilities.Plotting.FileCoding import extractRequestFromFileId -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... URL ...' % Script.scriptName, - 'Arguments:', - ' URL: encoded URL of a DIRAC Accounting plot'])) -Script.parseCommandLine() - -fileIds = Script.getPositionalArgs() - -for fileId in fileIds: - # Try to find if it's a url - parseRes = urlparse.urlparse(fileId) - if parseRes.query: - queryRes = cgi.parse_qs(parseRes.query) - if 'file' in queryRes: - fileId = queryRes['file'][0] - # Decode - result = extractRequestFromFileId(fileId) - if not result['OK']: - gLogger.error("Could not decode fileId", "'%s', error was %s" % (fileId, result['Message'])) - sys.exit(1) - gLogger.notice("Decode for '%s' is:\n%s" % (fileId, pprint.pformat(result['Value']))) - -sys.exit(0) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... URL ...' % Script.scriptName, + 'Arguments:', + ' URL: encoded URL of a DIRAC Accounting plot'])) + Script.parseCommandLine() + + fileIds = Script.getPositionalArgs() + + for fileId in fileIds: + # Try to find if it's a url + parseRes = urlparse.urlparse(fileId) + if parseRes.query: + queryRes = cgi.parse_qs(parseRes.query) + if 'file' in queryRes: + fileId = queryRes['file'][0] + # Decode + result = extractRequestFromFileId(fileId) + if not result['OK']: + gLogger.error("Could not decode fileId", "'%s', error was %s" % (fileId, result['Message'])) + sys.exit(1) + gLogger.notice("Decode for '%s' is:\n%s" % (fileId, pprint.pformat(result['Value']))) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/AccountingSystem/scripts/dirac_admin_accounting_cli.py b/src/DIRAC/AccountingSystem/scripts/dirac_admin_accounting_cli.py index 838fe5d5c67..1847340c011 100755 --- a/src/DIRAC/AccountingSystem/scripts/dirac_admin_accounting_cli.py +++ b/src/DIRAC/AccountingSystem/scripts/dirac_admin_accounting_cli.py @@ -13,15 +13,22 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.localCfg.addDefaultEntry("LogLevel", "info") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % Script.scriptName, ])) -Script.parseCommandLine() -from DIRAC.AccountingSystem.Client.AccountingCLI import AccountingCLI +@DIRACScript() +def main(): + Script.localCfg.addDefaultEntry("LogLevel", "info") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % Script.scriptName, ])) + Script.parseCommandLine() + + from DIRAC.AccountingSystem.Client.AccountingCLI import AccountingCLI -if __name__ == "__main__": acli = AccountingCLI() acli.start() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py index b8282ff6931..5f7fe0107ac 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py @@ -16,6 +16,7 @@ import shlex from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import gLogger, exit as DIRACExit from DIRAC.ConfigurationSystem.Client.Utilities import getGridCEs, getSiteUpdates from DIRAC.Core.Utilities.Subprocess import systemCall @@ -264,15 +265,12 @@ def handler(signum, frame): DIRACExit(-1) -if __name__ == "__main__": - +@DIRACScript() +def main(): signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGINT, handler) - vo = '' - dry = False - doCEs = False - ceBdiiDict = None + global vo, dry, doCEs, ceBdiiDict processScriptSwitches() @@ -280,7 +278,6 @@ def handler(signum, frame): gLogger.error('No VO specified') DIRACExit(-1) - diracVO = vo vo = getVOOption(vo, 'VOMSName', vo) if doCEs: @@ -293,3 +290,12 @@ def handler(signum, frame): yn = yn.strip() if yn == '' or yn.lower().startswith('y'): updateSites() + + +if __name__ == "__main__": + vo = '' + dry = False + doCEs = False + ceBdiiDict = None + + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_shifter.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_shifter.py index 161d2130bba..59fa8c6d731 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_shifter.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_shifter.py @@ -11,11 +11,13 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC import exit as DIRACExit, gLogger -if __name__ == "__main__": +@DIRACScript() +def main(): Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... ShifterRole UserName DIRACGroup ...' % Script.scriptName, @@ -44,3 +46,7 @@ gLogger.error("Could not add shifter", ": " + res['Message']) DIRACExit(1) gLogger.notice("Added shifter %s as user %s with group %s" % (shifterRole, userName, diracGroup)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py index f3a4b94cf4f..d6e7ed85fc5 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py @@ -14,13 +14,14 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import exit as DIRACExit, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getDIRACSiteName from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI -if __name__ == "__main__": - +@DIRACScript() +def main(): Script.setUsageMessage( '\n'.join( [ @@ -99,3 +100,7 @@ if not res['OK']: gLogger.error("Failure committing to CS", res['Message']) DIRACExit(3) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_bdii_info.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_bdii_info.py index 2619ffd5e5c..cb65a3d167c 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_bdii_info.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_bdii_info.py @@ -14,6 +14,7 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup @@ -131,10 +132,8 @@ def showInfo(result, info): print("}") -# ............................................................................... - -if __name__ == "__main__": - +@DIRACScript() +def main(): # Script initialization registerSwitches() # registerUsageMessage() @@ -144,5 +143,6 @@ def showInfo(result, info): DIRAC.exit(0) -################################################################################ -# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py index 087fd876853..53b9f5fc011 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py @@ -22,6 +22,7 @@ from diraccfg import CFG from DIRAC import gLogger, S_ERROR, S_OK, gConfig from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Utilities.List import fromChar LOG = gLogger @@ -202,5 +203,10 @@ def run(self): return S_OK() -if __name__ == "__main__": +@DIRACScript() +def main(): CheckConfig().run() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py index a9faf2e3028..113c766fc48 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py @@ -25,6 +25,7 @@ from DIRAC import gLogger, exit as DIRACExit from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getPropertiesForGroup from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI @@ -53,94 +54,98 @@ def country(arg): return cb[2] -Script.registerSwitch( - "C", - "country", - "Sort site names by country postfix (i.e. LCG.IHEP.cn, LCG.IN2P3.fr, LCG.IHEP.su)", - sortBy) -Script.registerSwitch("R", "reverse", "Reverse the sort order", isReverse) - -Script.setUsageMessage( - "\n".join( - [ - __doc__.split("\n")[1], - "Usage:", - " %s [option|cfgfile]
" % - Script.scriptName, - "Optional arguments:", - " Section: Name of the subsection in '/Resources/Sites/' for sort (i.e. LCG DIRAC)", - "Example:", - " dirac-admin-sort-cs-sites -C CLOUDS DIRAC", - " sort site names by country postfix in '/Resources/Sites/CLOUDS' and '/Resources/Sites/DIRAC' subsection", - ""])) - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -result = getProxyInfo() -if not result["OK"]: - gLogger.error("Failed to get proxy information", result["Message"]) - DIRACExit(2) -proxy = result["Value"] -if proxy["secondsLeft"] < 1: - gLogger.error("Your proxy has expired, please create new one") - DIRACExit(2) -group = proxy["group"] -if "CSAdministrator" not in getPropertiesForGroup(group): - gLogger.error("You must be CSAdministrator user to execute this script") - gLogger.notice("Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'") - DIRACExit(2) - -cs = CSAPI() -result = cs.getCurrentCFG() -if not result["OK"]: - gLogger.error("Failed to get copy of CS", result["Message"]) - DIRACExit(2) -cfg = result["Value"] - -if not cfg.isSection("Resources"): - gLogger.error("Section '/Resources' is absent in CS") - DIRACExit(2) - -if not cfg.isSection("Resources/Sites"): - gLogger.error("Subsection '/Resources/Sites' is absent in CS") - DIRACExit(2) - - -if args and len(args) > 0: - resultList = args[:] -else: - resultList = cfg["Resources"]["Sites"].listSections() - -hasRun = False -isDirty = False -for i in resultList: - if not cfg.isSection("Resources/Sites/%s" % i): - gLogger.error("Subsection /Resources/Sites/%s does not exists" % i) - continue - hasRun = True - if SORTBYNAME: - dirty = cfg["Resources"]["Sites"][i].sortAlphabetically(ascending=not REVERSE) +@DIRACScript() +def main(): + Script.registerSwitch( + "C", + "country", + "Sort site names by country postfix (i.e. LCG.IHEP.cn, LCG.IN2P3.fr, LCG.IHEP.su)", + sortBy) + Script.registerSwitch("R", "reverse", "Reverse the sort order", isReverse) + + Script.setUsageMessage("\n".join([ + __doc__.split("\n")[1], + "Usage:", + " %s [option|cfgfile]
" % + Script.scriptName, + "Optional arguments:", + " Section: Name of the subsection in '/Resources/Sites/' for sort (i.e. LCG DIRAC)", + "Example:", + " dirac-admin-sort-cs-sites -C CLOUDS DIRAC", + " sort site names by country postfix in '/Resources/Sites/CLOUDS' and '/Resources/Sites/DIRAC' subsection", + "" + ])) + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + result = getProxyInfo() + if not result["OK"]: + gLogger.error("Failed to get proxy information", result["Message"]) + DIRACExit(2) + proxy = result["Value"] + if proxy["secondsLeft"] < 1: + gLogger.error("Your proxy has expired, please create new one") + DIRACExit(2) + group = proxy["group"] + if "CSAdministrator" not in getPropertiesForGroup(group): + gLogger.error("You must be CSAdministrator user to execute this script") + gLogger.notice("Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'") + DIRACExit(2) + + cs = CSAPI() + result = cs.getCurrentCFG() + if not result["OK"]: + gLogger.error("Failed to get copy of CS", result["Message"]) + DIRACExit(2) + cfg = result["Value"] + + if not cfg.isSection("Resources"): + gLogger.error("Section '/Resources' is absent in CS") + DIRACExit(2) + + if not cfg.isSection("Resources/Sites"): + gLogger.error("Subsection '/Resources/Sites' is absent in CS") + DIRACExit(2) + + if args and len(args) > 0: + resultList = args[:] else: - dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country, reverse=REVERSE) - if dirty: - isDirty = True - -if not hasRun: - gLogger.notice("Failed to find suitable subsections with site names to sort") - DIRACExit(0) - -if not isDirty: - gLogger.notice("Nothing to do, site names are already sorted") + resultList = cfg["Resources"]["Sites"].listSections() + + hasRun = False + isDirty = False + for i in resultList: + if not cfg.isSection("Resources/Sites/%s" % i): + gLogger.error("Subsection /Resources/Sites/%s does not exists" % i) + continue + hasRun = True + if SORTBYNAME: + dirty = cfg["Resources"]["Sites"][i].sortAlphabetically(ascending=not REVERSE) + else: + dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country, reverse=REVERSE) + if dirty: + isDirty = True + + if not hasRun: + gLogger.notice("Failed to find suitable subsections with site names to sort") + DIRACExit(0) + + if not isDirty: + gLogger.notice("Nothing to do, site names are already sorted") + DIRACExit(0) + + timestamp = toString(dateTime()) + stamp = "Site names are sorted by %s script at %s" % (Script.scriptName, timestamp) + cs.setOptionComment("/Resources/Sites", stamp) + + result = cs.commit() + if not result["OK"]: + gLogger.error("Failed to commit changes to CS", result["Message"]) + DIRACExit(2) + gLogger.notice("Site names are sorted and committed to CS") DIRACExit(0) -timestamp = toString(dateTime()) -stamp = "Site names are sorted by %s script at %s" % (Script.scriptName, timestamp) -cs.setOptionComment("/Resources/Sites", stamp) -result = cs.commit() -if not result["OK"]: - gLogger.error("Failed to commit changes to CS", result["Message"]) - DIRACExit(2) -gLogger.notice("Site names are sorted and committed to CS") -DIRACExit(0) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py index a00df5d0ba5..c7b2aeeadf6 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py @@ -14,12 +14,14 @@ from DIRAC import gLogger, exit as DIRACExit, S_OK from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.ConfigurationSystem.Client.VOMS2CSSynchronizer import VOMS2CSSynchronizer from DIRAC.Core.Utilities.Proxy import executeWithUserProxy from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOOption dryRun = False +voName = None def setDryRun(value): @@ -28,77 +30,78 @@ def setDryRun(value): return S_OK() -voName = None - - def setVO(value): global voName voName = value return S_OK() -Script.registerSwitch("V:", "vo=", "VO name", setVO) -Script.registerSwitch("D", "dryRun", "Dry run", setDryRun) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % Script.scriptName] - )) - -Script.parseCommandLine(ignoreErrors=True) - - -@executeWithUserProxy -def syncCSWithVOMS(vomsSync): - return vomsSync.syncCSWithVOMS() - - -def getVOAdmin(voName): - voAdminUser = getVOOption(voName, "VOAdmin") - voAdminGroup = getVOOption(voName, "VOAdminGroup", getVOOption(voName, "DefaultGroup")) - return voAdminUser, voAdminGroup - - -voAdminUser, voAdminGroup = getVOAdmin(voName) - -vomsSync = VOMS2CSSynchronizer(voName) -result = syncCSWithVOMS(vomsSync, # pylint: disable=unexpected-keyword-arg - proxyUserName=voAdminUser, - proxyUserGroup=voAdminGroup) -if not result['OK']: - gLogger.error("Failed to synchronize user data") - DIRACExit(-1) - -resultDict = result['Value'] -newUsers = resultDict.get("NewUsers", []) -modUsers = resultDict.get("ModifiedUsers", []) -delUsers = resultDict.get("DeletedUsers", []) -susUsers = resultDict.get("SuspendedUsers", []) -gLogger.notice("\nUser results: new %d, modified %d, deleted %d, new/suspended %d" % - (len(newUsers), len(modUsers), len(delUsers), len(susUsers))) +@DIRACScript() +def main(): + Script.registerSwitch("V:", "vo=", "VO name", setVO) + Script.registerSwitch("D", "dryRun", "Dry run", setDryRun) + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % Script.scriptName + ])) + + Script.parseCommandLine(ignoreErrors=True) + + @executeWithUserProxy + def syncCSWithVOMS(vomsSync): + return vomsSync.syncCSWithVOMS() + + def getVOAdmin(voName): + voAdminUser = getVOOption(voName, "VOAdmin") + voAdminGroup = getVOOption(voName, "VOAdminGroup", getVOOption(voName, "DefaultGroup")) + return voAdminUser, voAdminGroup + + voAdminUser, voAdminGroup = getVOAdmin(voName) + + vomsSync = VOMS2CSSynchronizer(voName) + result = syncCSWithVOMS(vomsSync, # pylint: disable=unexpected-keyword-arg + proxyUserName=voAdminUser, + proxyUserGroup=voAdminGroup) + if not result['OK']: + gLogger.error("Failed to synchronize user data") + DIRACExit(-1) + + resultDict = result['Value'] + newUsers = resultDict.get("NewUsers", []) + modUsers = resultDict.get("ModifiedUsers", []) + delUsers = resultDict.get("DeletedUsers", []) + susUsers = resultDict.get("SuspendedUsers", []) + gLogger.notice("\nUser results: new %d, modified %d, deleted %d, new/suspended %d" % + (len(newUsers), len(modUsers), len(delUsers), len(susUsers))) + + for msg in resultDict["AdminMessages"]["Info"]: + gLogger.notice(msg) + + csapi = resultDict.get("CSAPI") + if csapi and csapi.csModified: + if dryRun: + gLogger.notice("There are changes to Registry ready to commit, skipped because of dry run") + else: + yn = raw_input("There are changes to Registry ready to commit, do you want to proceed ? [Y|n]:") + if yn == '' or yn[0].lower() == 'y': + result = csapi.commitChanges() + if not result['OK']: + gLogger.error("Could not commit configuration changes", result['Message']) + else: + gLogger.notice("Registry changes committed for VO %s" % voName) + else: + gLogger.notice("Registry changes are not committed") + else: + gLogger.notice("No changes to Registry for VO %s" % voName) -for msg in resultDict["AdminMessages"]["Info"]: - gLogger.notice(msg) + result = vomsSync.getVOUserReport() + if not result['OK']: + gLogger.error('Failed to generate user data report') + DIRACExit(-1) -csapi = resultDict.get("CSAPI") -if csapi and csapi.csModified: - if dryRun: - gLogger.notice("There are changes to Registry ready to commit, skipped because of dry run") - else: - yn = raw_input("There are changes to Registry ready to commit, do you want to proceed ? [Y|n]:") - if yn == '' or yn[0].lower() == 'y': - result = csapi.commitChanges() - if not result['OK']: - gLogger.error("Could not commit configuration changes", result['Message']) - else: - gLogger.notice("Registry changes committed for VO %s" % voName) - else: - gLogger.notice("Registry changes are not committed") -else: - gLogger.notice("No changes to Registry for VO %s" % voName) + gLogger.notice("\n" + result['Value']) -result = vomsSync.getVOUserReport() -if not result['OK']: - gLogger.error('Failed to generate user data report') - DIRACExit(-1) -gLogger.notice("\n" + result['Value']) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_cli.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_cli.py index d26a8be2ae7..6641f2e44a2 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_cli.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_cli.py @@ -12,12 +12,22 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.ConfigurationSystem.Client.CSCLI import CSCLI -Script.localCfg.addDefaultEntry("LogLevel", "fatal") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % Script.scriptName, ])) -Script.parseCommandLine() -CSCLI().start() +@DIRACScript() +def main(): + Script.localCfg.addDefaultEntry("LogLevel", "fatal") + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % Script.scriptName, + ])) + Script.parseCommandLine() + + CSCLI().start() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py index 75fde14489c..33e6f80caf2 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py @@ -15,41 +15,45 @@ import sys import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.localCfg.addDefaultEntry("LogLevel", "fatal") -fileName = "" +@DIRACScript() +def main(): + Script.localCfg.addDefaultEntry("LogLevel", "fatal") + fileName = "" -def setFilename(args): - global fileName - fileName = args - return DIRAC.S_OK() + def setFilename(args): + global fileName + fileName = args + return DIRAC.S_OK() + raw = False -raw = False + def setRaw(args): + global raw + raw = True + return DIRAC.S_OK() + Script.registerSwitch("f:", "file=", "Dump Configuration data into ", setFilename) + Script.registerSwitch("r", "raw", "Do not make any modification to the data", setRaw) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % Script.scriptName, ])) + Script.parseCommandLine() -def setRaw(args): - global raw - raw = True - return DIRAC.S_OK() + from DIRAC import gConfig, gLogger + result = gConfig.dumpCFGAsLocalCache(fileName, raw) + if not result['OK']: + print("Error: %s" % result['Message']) + sys.exit(1) + if not fileName: + print(result['Value']) -Script.registerSwitch("f:", "file=", "Dump Configuration data into ", setFilename) -Script.registerSwitch("r", "raw", "Do not make any modification to the data", setRaw) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % Script.scriptName, ])) -Script.parseCommandLine() + sys.exit(0) -from DIRAC import gConfig, gLogger -result = gConfig.dumpCFGAsLocalCache(fileName, raw) -if not result['OK']: - print("Error: %s" % result['Message']) - sys.exit(1) -if not fileName: - print(result['Value']) - -sys.exit(0) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py index 27b65491393..a70a2a9bd6c 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py @@ -9,20 +9,20 @@ import sys from DIRAC.Core.Base import Script - -Script.parseCommandLine() - -from DIRAC.ConfigurationSystem.Client.CSShellCLI import CSShellCLI +from DIRAC.Core.Utilities.DIRACScript import DIRACScript # Invariants: # * root does not end with "/" or root is "/" # * root starts with "/" +@DIRACScript def main(): + Script.parseCommandLine() + from DIRAC.ConfigurationSystem.Client.CSShellCLI import CSShellCLI shell = CSShellCLI() shell.cmdloop() if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/src/DIRAC/Core/scripts/dirac_install.py b/src/DIRAC/Core/scripts/dirac-install.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac_install.py rename to src/DIRAC/Core/scripts/dirac-install.py diff --git a/src/DIRAC/Core/scripts/dirac_agent.py b/src/DIRAC/Core/scripts/dirac_agent.py index 6bd5b3e43a2..e449967ed6c 100755 --- a/src/DIRAC/Core/scripts/dirac_agent.py +++ b/src/DIRAC/Core/scripts/dirac_agent.py @@ -16,31 +16,39 @@ from DIRAC import gLogger from DIRAC.Core.Base.AgentReactor import AgentReactor from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -localCfg = LocalConfiguration() - -positionalArgs = localCfg.getPositionalArguments() -if len(positionalArgs) == 0: - gLogger.fatal("You must specify which agent to run!") - sys.exit(1) - -agentName = positionalArgs[0] -localCfg.setConfigurationForAgent(agentName) -localCfg.addMandatoryEntry("/DIRAC/Setup") -localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") -localCfg.addDefaultEntry("LogLevel", "INFO") -localCfg.addDefaultEntry("LogColor", True) -resultDict = localCfg.loadUserData() -if not resultDict['OK']: - gLogger.error("There were errors when loading configuration", resultDict['Message']) - sys.exit(1) - -includeExtensionErrors() - -agentReactor = AgentReactor(positionalArgs[0]) -result = agentReactor.loadAgentModules(positionalArgs) -if result['OK']: - agentReactor.go() -else: - gLogger.error("Error while loading agent module", result['Message']) - sys.exit(2) + +@DIRACScript() +def main(): + localCfg = LocalConfiguration() + + positionalArgs = localCfg.getPositionalArguments() + if len(positionalArgs) == 0: + gLogger.fatal("You must specify which agent to run!") + sys.exit(1) + + agentName = positionalArgs[0] + localCfg.setConfigurationForAgent(agentName) + localCfg.addMandatoryEntry("/DIRAC/Setup") + localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") + localCfg.addDefaultEntry("LogLevel", "INFO") + localCfg.addDefaultEntry("LogColor", True) + resultDict = localCfg.loadUserData() + if not resultDict['OK']: + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + + includeExtensionErrors() + + agentReactor = AgentReactor(positionalArgs[0]) + result = agentReactor.loadAgentModules(positionalArgs) + if result['OK']: + agentReactor.go() + else: + gLogger.error("Error while loading agent module", result['Message']) + sys.exit(2) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_configure.py b/src/DIRAC/Core/scripts/dirac_configure.py index 6127d6c066d..e87350e4590 100755 --- a/src/DIRAC/Core/scripts/dirac_configure.py +++ b/src/DIRAC/Core/scripts/dirac_configure.py @@ -75,6 +75,7 @@ import DIRAC from DIRAC.Core.Utilities.File import mkDir from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.ConfigurationSystem.Client.Helpers import cfgInstallPath, cfgPath, Registry from DIRAC.Core.Utilities.SiteSEMapping import getSEsForSite @@ -228,335 +229,358 @@ def setExtensions(optionValue): return DIRAC.S_OK() -Script.disableCS() - -Script.registerSwitch("S:", "Setup=", "Set as DIRAC setup", setSetup) -Script.registerSwitch("e:", "Extensions=", "Set as DIRAC extensions", setExtensions) -Script.registerSwitch("C:", "ConfigurationServer=", "Set as DIRAC configuration server", setServer) -Script.registerSwitch("I", "IncludeAllServers", "include all Configuration Servers", setAllServers) -Script.registerSwitch("n:", "SiteName=", "Set as DIRAC Site Name", setSiteName) -Script.registerSwitch("N:", "CEName=", "Determiner from ", setCEName) -Script.registerSwitch("V:", "VO=", "Set the VO name", setVO) - -Script.registerSwitch("W:", "gateway=", "Configure as DIRAC Gateway for the site", setGateway) - -Script.registerSwitch("U", "UseServerCertificate", "Configure to use Server Certificate", setServerCert) -Script.registerSwitch("H", "SkipCAChecks", "Configure to skip check of CAs", setSkipCAChecks) -Script.registerSwitch("D", "SkipCADownload", "Configure to skip download of CAs", setSkipCADownload) -Script.registerSwitch("M", "SkipVOMSDownload", "Configure to skip download of VOMS info", setSkipVOMSDownload) - -Script.registerSwitch("v", "UseVersionsDir", "Use versions directory", setUseVersionsDir) - -Script.registerSwitch("A:", "Architecture=", "Configure /Architecture=", setArchitecture) -Script.registerSwitch("L:", "LocalSE=", "Configure LocalSite/LocalSE=", setLocalSE) - -Script.registerSwitch( - "F", - "ForceUpdate", - "Force Update of cfg file (i.e. dirac.cfg) (otherwise nothing happens if dirac.cfg already exists)", - forceUpdate) - -Script.registerSwitch("O:", "output=", "output configuration file", setOutput) - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - '\nUsage:', - ' %s [option|cfgfile] ...\n' % Script.scriptName])) - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getExtraCLICFGFiles() - -if not logLevel: - logLevel = DIRAC.gConfig.getValue(cfgInstallPath('LogLevel'), '') - if logLevel: - DIRAC.gLogger.setLevel(logLevel) -else: - DIRAC.gConfig.setOptionValue(cfgInstallPath('LogLevel'), logLevel) - -if not gatewayServer: - newGatewayServer = DIRAC.gConfig.getValue(cfgInstallPath('Gateway'), '') - if newGatewayServer: - setGateway(newGatewayServer) - -if not configurationServer: - newConfigurationServer = DIRAC.gConfig.getValue(cfgInstallPath('ConfigurationServer'), '') - if newConfigurationServer: - setServer(newConfigurationServer) - -if not includeAllServers: - newIncludeAllServer = DIRAC.gConfig.getValue(cfgInstallPath('IncludeAllServers'), False) - if newIncludeAllServer: - setAllServers(True) - -if not setup: - newSetup = DIRAC.gConfig.getValue(cfgInstallPath('Setup'), '') - if newSetup: - setSetup(newSetup) - -if not siteName: - newSiteName = DIRAC.gConfig.getValue(cfgInstallPath('SiteName'), '') - if newSiteName: - setSiteName(newSiteName) - -if not ceName: - newCEName = DIRAC.gConfig.getValue(cfgInstallPath('CEName'), '') - if newCEName: - setCEName(newCEName) - -if not useServerCert: - newUserServerCert = DIRAC.gConfig.getValue(cfgInstallPath('UseServerCertificate'), False) - if newUserServerCert: - setServerCert(newUserServerCert) - -if not skipCAChecks: - newSkipCAChecks = DIRAC.gConfig.getValue(cfgInstallPath('SkipCAChecks'), False) - if newSkipCAChecks: - setSkipCAChecks(newSkipCAChecks) - -if not skipCADownload: - newSkipCADownload = DIRAC.gConfig.getValue(cfgInstallPath('SkipCADownload'), False) - if newSkipCADownload: - setSkipCADownload(newSkipCADownload) - -if not useVersionsDir: - newUseVersionsDir = DIRAC.gConfig.getValue(cfgInstallPath('UseVersionsDir'), False) - if newUseVersionsDir: - setUseVersionsDir(newUseVersionsDir) - # Set proper Defaults in configuration (even if they will be properly overwrite by gComponentInstaller - instancePath = os.path.dirname(os.path.dirname(DIRAC.rootPath)) - rootPath = os.path.join(instancePath, 'pro') - DIRAC.gConfig.setOptionValue(cfgInstallPath('InstancePath'), instancePath) - DIRAC.gConfig.setOptionValue(cfgInstallPath('RootPath'), rootPath) - -if not architecture: - newArchitecture = DIRAC.gConfig.getValue(cfgInstallPath('Architecture'), '') - if newArchitecture: - setArchitecture(newArchitecture) - -if not vo: - newVO = DIRAC.gConfig.getValue(cfgInstallPath('VirtualOrganization'), '') - if newVO: - setVO(newVO) - -if not extensions: - newExtensions = DIRAC.gConfig.getValue(cfgInstallPath('Extensions'), '') - if newExtensions: - setExtensions(newExtensions) - -DIRAC.gLogger.notice('Executing: %s ' % (' '.join(sys.argv))) -DIRAC.gLogger.notice('Checking DIRAC installation at "%s"' % DIRAC.rootPath) - -if update: - if outputFile: - DIRAC.gLogger.notice('Will update the output file %s' % outputFile) +@DIRACScript() +def main(): + global logLevel + global setup + global configurationServer + global includeAllServers + global gatewayServer + global siteName + global useServerCert + global skipCAChecks + global skipCADownload + global useVersionsDir + global architecture + global localSE + global ceName + global vo + global update + global outputFile + global skipVOMSDownload + global extensions + + Script.disableCS() + + Script.registerSwitch("S:", "Setup=", "Set as DIRAC setup", setSetup) + Script.registerSwitch("e:", "Extensions=", "Set as DIRAC extensions", setExtensions) + Script.registerSwitch("C:", "ConfigurationServer=", "Set as DIRAC configuration server", setServer) + Script.registerSwitch("I", "IncludeAllServers", "include all Configuration Servers", setAllServers) + Script.registerSwitch("n:", "SiteName=", "Set as DIRAC Site Name", setSiteName) + Script.registerSwitch("N:", "CEName=", "Determiner from ", setCEName) + Script.registerSwitch("V:", "VO=", "Set the VO name", setVO) + + Script.registerSwitch("W:", "gateway=", "Configure as DIRAC Gateway for the site", setGateway) + + Script.registerSwitch("U", "UseServerCertificate", "Configure to use Server Certificate", setServerCert) + Script.registerSwitch("H", "SkipCAChecks", "Configure to skip check of CAs", setSkipCAChecks) + Script.registerSwitch("D", "SkipCADownload", "Configure to skip download of CAs", setSkipCADownload) + Script.registerSwitch("M", "SkipVOMSDownload", "Configure to skip download of VOMS info", setSkipVOMSDownload) + + Script.registerSwitch("v", "UseVersionsDir", "Use versions directory", setUseVersionsDir) + + Script.registerSwitch("A:", "Architecture=", "Configure /Architecture=", setArchitecture) + Script.registerSwitch("L:", "LocalSE=", "Configure LocalSite/LocalSE=", setLocalSE) + + Script.registerSwitch( + "F", + "ForceUpdate", + "Force Update of cfg file (i.e. dirac.cfg) (otherwise nothing happens if dirac.cfg already exists)", + forceUpdate) + + Script.registerSwitch("O:", "output=", "output configuration file", setOutput) + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + '\nUsage:', + ' %s [option|cfgfile] ...\n' % Script.scriptName])) + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getExtraCLICFGFiles() + + if not logLevel: + logLevel = DIRAC.gConfig.getValue(cfgInstallPath('LogLevel'), '') + if logLevel: + DIRAC.gLogger.setLevel(logLevel) else: - DIRAC.gLogger.notice('Will update %s' % DIRAC.gConfig.diracConfigFilePath) - -if setup: - DIRAC.gLogger.verbose('/DIRAC/Setup =', setup) -if vo: - DIRAC.gLogger.verbose('/DIRAC/VirtualOrganization =', vo) -if configurationServer: - DIRAC.gLogger.verbose('/DIRAC/Configuration/Servers =', configurationServer) - -if siteName: - DIRAC.gLogger.verbose('/LocalSite/Site =', siteName) -if architecture: - DIRAC.gLogger.verbose('/LocalSite/Architecture =', architecture) -if localSE: - DIRAC.gLogger.verbose('/LocalSite/localSE =', localSE) - -if not useServerCert: - DIRAC.gLogger.verbose('/DIRAC/Security/UseServerCertificate =', 'no') - # Being sure it was not there before - Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') - Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'no') -else: - DIRAC.gLogger.verbose('/DIRAC/Security/UseServerCertificate =', 'yes') - # Being sure it was not there before - Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') - Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'yes') - -host = DIRAC.gConfig.getValue(cfgInstallPath("Host"), "") -if host: - DIRAC.gConfig.setOptionValue(cfgPath("DIRAC", "Hostname"), host) - -if skipCAChecks: - DIRAC.gLogger.verbose('/DIRAC/Security/SkipCAChecks =', 'yes') - # Being sure it was not there before - Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') - Script.localCfg.addDefaultEntry('/DIRAC/Security/SkipCAChecks', 'yes') -else: - # Necessary to allow initial download of CA's - if not skipCADownload: - DIRAC.gConfig.setOptionValue('/DIRAC/Security/SkipCAChecks', 'yes') -if not skipCADownload: - Script.enableCS() - try: - dirName = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'certificates') - mkDir(dirName) - except BaseException: - DIRAC.gLogger.exception() - DIRAC.gLogger.fatal('Fail to create directory:', dirName) - DIRAC.exit(-1) - try: - bdc = BundleDeliveryClient() - result = bdc.syncCAs() - if result['OK']: - result = bdc.syncCRLs() - except Exception as e: - DIRAC.gLogger.error('Failed to sync CAs and CRLs: %s' % str(e)) + DIRAC.gConfig.setOptionValue(cfgInstallPath('LogLevel'), logLevel) + + if not gatewayServer: + newGatewayServer = DIRAC.gConfig.getValue(cfgInstallPath('Gateway'), '') + if newGatewayServer: + setGateway(newGatewayServer) + + if not configurationServer: + newConfigurationServer = DIRAC.gConfig.getValue(cfgInstallPath('ConfigurationServer'), '') + if newConfigurationServer: + setServer(newConfigurationServer) + + if not includeAllServers: + newIncludeAllServer = DIRAC.gConfig.getValue(cfgInstallPath('IncludeAllServers'), False) + if newIncludeAllServer: + setAllServers(True) + + if not setup: + newSetup = DIRAC.gConfig.getValue(cfgInstallPath('Setup'), '') + if newSetup: + setSetup(newSetup) + + if not siteName: + newSiteName = DIRAC.gConfig.getValue(cfgInstallPath('SiteName'), '') + if newSiteName: + setSiteName(newSiteName) + + if not ceName: + newCEName = DIRAC.gConfig.getValue(cfgInstallPath('CEName'), '') + if newCEName: + setCEName(newCEName) + + if not useServerCert: + newUserServerCert = DIRAC.gConfig.getValue(cfgInstallPath('UseServerCertificate'), False) + if newUserServerCert: + setServerCert(newUserServerCert) if not skipCAChecks: - Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') + newSkipCAChecks = DIRAC.gConfig.getValue(cfgInstallPath('SkipCAChecks'), False) + if newSkipCAChecks: + setSkipCAChecks(newSkipCAChecks) -if ceName or siteName: - # This is used in the pilot context, we should have a proxy, or a certificate, and access to CS - if useServerCert: - # Being sure it was not there before - Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') - Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'yes') - Script.enableCS() - # Get the site resource section - gridSections = DIRAC.gConfig.getSections('/Resources/Sites/') - if not gridSections['OK']: - DIRAC.gLogger.warn('Could not get grid sections list') - grids = [] - else: - grids = gridSections['Value'] - # try to get siteName from ceName or Local SE from siteName using Remote Configuration - for grid in grids: - siteSections = DIRAC.gConfig.getSections('/Resources/Sites/%s/' % grid) - if not siteSections['OK']: - DIRAC.gLogger.warn('Could not get %s site list' % grid) - sites = [] + if not skipCADownload: + newSkipCADownload = DIRAC.gConfig.getValue(cfgInstallPath('SkipCADownload'), False) + if newSkipCADownload: + setSkipCADownload(newSkipCADownload) + + if not useVersionsDir: + newUseVersionsDir = DIRAC.gConfig.getValue(cfgInstallPath('UseVersionsDir'), False) + if newUseVersionsDir: + setUseVersionsDir(newUseVersionsDir) + # Set proper Defaults in configuration (even if they will be properly overwrite by gComponentInstaller + instancePath = os.path.dirname(os.path.dirname(DIRAC.rootPath)) + rootPath = os.path.join(instancePath, 'pro') + DIRAC.gConfig.setOptionValue(cfgInstallPath('InstancePath'), instancePath) + DIRAC.gConfig.setOptionValue(cfgInstallPath('RootPath'), rootPath) + + if not architecture: + newArchitecture = DIRAC.gConfig.getValue(cfgInstallPath('Architecture'), '') + if newArchitecture: + setArchitecture(newArchitecture) + + if not vo: + newVO = DIRAC.gConfig.getValue(cfgInstallPath('VirtualOrganization'), '') + if newVO: + setVO(newVO) + + if not extensions: + newExtensions = DIRAC.gConfig.getValue(cfgInstallPath('Extensions'), '') + if newExtensions: + setExtensions(newExtensions) + + DIRAC.gLogger.notice('Executing: %s ' % (' '.join(sys.argv))) + DIRAC.gLogger.notice('Checking DIRAC installation at "%s"' % DIRAC.rootPath) + + if update: + if outputFile: + DIRAC.gLogger.notice('Will update the output file %s' % outputFile) else: - sites = siteSections['Value'] - - if not siteName: - if ceName: - for site in sites: - res = DIRAC.gConfig.getSections('/Resources/Sites/%s/%s/CEs/' % (grid, site), []) - if not res['OK']: - DIRAC.gLogger.warn('Could not get %s CEs list' % site) - if ceName in res['Value']: - siteName = site - break - if siteName: - DIRAC.gLogger.notice('Setting /LocalSite/Site = %s' % siteName) - Script.localCfg.addDefaultEntry('/LocalSite/Site', siteName) - DIRAC.__siteName = False - if ceName: - DIRAC.gLogger.notice('Setting /LocalSite/GridCE = %s' % ceName) - Script.localCfg.addDefaultEntry('/LocalSite/GridCE', ceName) - - if not localSE and siteName in sites: - localSE = getSEsForSite(siteName) - if localSE['OK'] and localSE['Value']: - localSE = ','.join(localSE['Value']) - DIRAC.gLogger.notice('Setting /LocalSite/LocalSE =', localSE) - Script.localCfg.addDefaultEntry('/LocalSite/LocalSE', localSE) - break - -if gatewayServer: - DIRAC.gLogger.verbose('/DIRAC/Gateways/%s =' % DIRAC.siteName(), gatewayServer) - Script.localCfg.addDefaultEntry('/DIRAC/Gateways/%s' % DIRAC.siteName(), gatewayServer) - -# Create the local cfg if it is not yet there -if not outputFile: - outputFile = DIRAC.gConfig.diracConfigFilePath -outputFile = os.path.abspath(outputFile) -if not os.path.exists(outputFile): - configDir = os.path.dirname(outputFile) - mkDir(configDir) - update = True - DIRAC.gConfig.dumpLocalCFGToFile(outputFile) + DIRAC.gLogger.notice('Will update %s' % DIRAC.gConfig.diracConfigFilePath) + + if setup: + DIRAC.gLogger.verbose('/DIRAC/Setup =', setup) + if vo: + DIRAC.gLogger.verbose('/DIRAC/VirtualOrganization =', vo) + if configurationServer: + DIRAC.gLogger.verbose('/DIRAC/Configuration/Servers =', configurationServer) + + if siteName: + DIRAC.gLogger.verbose('/LocalSite/Site =', siteName) + if architecture: + DIRAC.gLogger.verbose('/LocalSite/Architecture =', architecture) + if localSE: + DIRAC.gLogger.verbose('/LocalSite/localSE =', localSE) -if includeAllServers: - # We need user proxy or server certificate to continue in order to get all the CS URLs if not useServerCert: - Script.enableCS() - result = getProxyInfo() - if not result['OK']: - DIRAC.gLogger.notice('Configuration is not completed because no user proxy is available') - DIRAC.gLogger.notice('Create one using dirac-proxy-init and execute again with -F option') - sys.exit(1) + DIRAC.gLogger.verbose('/DIRAC/Security/UseServerCertificate =', 'no') + # Being sure it was not there before + Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') + Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'no') else: + DIRAC.gLogger.verbose('/DIRAC/Security/UseServerCertificate =', 'yes') + # Being sure it was not there before Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') - # When using Server Certs CA's will be checked, the flag only disables initial download - # this will be replaced by the use of SkipCADownload Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'yes') - Script.enableCS() - DIRAC.gConfig.setOptionValue('/DIRAC/Configuration/Servers', ','.join(DIRAC.gConfig.getServersList())) - DIRAC.gLogger.verbose('/DIRAC/Configuration/Servers =', ','.join(DIRAC.gConfig.getServersList())) + host = DIRAC.gConfig.getValue(cfgInstallPath("Host"), "") + if host: + DIRAC.gConfig.setOptionValue(cfgPath("DIRAC", "Hostname"), host) -if useServerCert: - # always removing before dumping - Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') - Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') - Script.localCfg.deleteOption('/DIRAC/Security/SkipVOMSDownload') + if skipCAChecks: + DIRAC.gLogger.verbose('/DIRAC/Security/SkipCAChecks =', 'yes') + # Being sure it was not there before + Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') + Script.localCfg.addDefaultEntry('/DIRAC/Security/SkipCAChecks', 'yes') + else: + # Necessary to allow initial download of CA's + if not skipCADownload: + DIRAC.gConfig.setOptionValue('/DIRAC/Security/SkipCAChecks', 'yes') + if not skipCADownload: + Script.enableCS() + try: + dirName = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'certificates') + mkDir(dirName) + except BaseException: + DIRAC.gLogger.exception() + DIRAC.gLogger.fatal('Fail to create directory:', dirName) + DIRAC.exit(-1) + try: + bdc = BundleDeliveryClient() + result = bdc.syncCAs() + if result['OK']: + result = bdc.syncCRLs() + except Exception as e: + DIRAC.gLogger.error('Failed to sync CAs and CRLs: %s' % str(e)) + + if not skipCAChecks: + Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') + + if ceName or siteName: + # This is used in the pilot context, we should have a proxy, or a certificate, and access to CS + if useServerCert: + # Being sure it was not there before + Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') + Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'yes') + Script.enableCS() + # Get the site resource section + gridSections = DIRAC.gConfig.getSections('/Resources/Sites/') + if not gridSections['OK']: + DIRAC.gLogger.warn('Could not get grid sections list') + grids = [] + else: + grids = gridSections['Value'] + # try to get siteName from ceName or Local SE from siteName using Remote Configuration + for grid in grids: + siteSections = DIRAC.gConfig.getSections('/Resources/Sites/%s/' % grid) + if not siteSections['OK']: + DIRAC.gLogger.warn('Could not get %s site list' % grid) + sites = [] + else: + sites = siteSections['Value'] + + if not siteName: + if ceName: + for site in sites: + res = DIRAC.gConfig.getSections('/Resources/Sites/%s/%s/CEs/' % (grid, site), []) + if not res['OK']: + DIRAC.gLogger.warn('Could not get %s CEs list' % site) + if ceName in res['Value']: + siteName = site + break + if siteName: + DIRAC.gLogger.notice('Setting /LocalSite/Site = %s' % siteName) + Script.localCfg.addDefaultEntry('/LocalSite/Site', siteName) + DIRAC.__siteName = False + if ceName: + DIRAC.gLogger.notice('Setting /LocalSite/GridCE = %s' % ceName) + Script.localCfg.addDefaultEntry('/LocalSite/GridCE', ceName) + + if not localSE and siteName in sites: + localSE = getSEsForSite(siteName) + if localSE['OK'] and localSE['Value']: + localSE = ','.join(localSE['Value']) + DIRAC.gLogger.notice('Setting /LocalSite/LocalSE =', localSE) + Script.localCfg.addDefaultEntry('/LocalSite/LocalSE', localSE) + break + + if gatewayServer: + DIRAC.gLogger.verbose('/DIRAC/Gateways/%s =' % DIRAC.siteName(), gatewayServer) + Script.localCfg.addDefaultEntry('/DIRAC/Gateways/%s' % DIRAC.siteName(), gatewayServer) + + # Create the local cfg if it is not yet there + if not outputFile: + outputFile = DIRAC.gConfig.diracConfigFilePath + outputFile = os.path.abspath(outputFile) + if not os.path.exists(outputFile): + configDir = os.path.dirname(outputFile) + mkDir(configDir) + update = True + DIRAC.gConfig.dumpLocalCFGToFile(outputFile) + + if includeAllServers: + # We need user proxy or server certificate to continue in order to get all the CS URLs + if not useServerCert: + Script.enableCS() + result = getProxyInfo() + if not result['OK']: + DIRAC.gLogger.notice('Configuration is not completed because no user proxy is available') + DIRAC.gLogger.notice('Create one using dirac-proxy-init and execute again with -F option') + sys.exit(1) + else: + Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') + # When using Server Certs CA's will be checked, the flag only disables initial download + # this will be replaced by the use of SkipCADownload + Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'yes') + Script.enableCS() -if update: - DIRAC.gConfig.dumpLocalCFGToFile(outputFile) + DIRAC.gConfig.setOptionValue('/DIRAC/Configuration/Servers', ','.join(DIRAC.gConfig.getServersList())) + DIRAC.gLogger.verbose('/DIRAC/Configuration/Servers =', ','.join(DIRAC.gConfig.getServersList())) + if useServerCert: + # always removing before dumping + Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') + Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') + Script.localCfg.deleteOption('/DIRAC/Security/SkipVOMSDownload') + + if update: + DIRAC.gConfig.dumpLocalCFGToFile(outputFile) + + # ## LAST PART: do the vomsdir/vomses magic + + # This has to be done for all VOs in the installation + + if skipVOMSDownload: + # We stop here + sys.exit(0) + + result = Registry.getVOMSServerInfo() + if not result['OK']: + sys.exit(1) + + error = '' + vomsDict = result['Value'] + for vo in vomsDict: + voName = vomsDict[vo]['VOMSName'] + vomsDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'vomsdir', voName) + vomsesDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'vomses') + for path in (vomsDirPath, vomsesDirPath): + mkDir(path) + vomsesLines = [] + for vomsHost in vomsDict[vo].get('Servers', {}): + hostFilePath = os.path.join(vomsDirPath, "%s.lsc" % vomsHost) + try: + DN = vomsDict[vo]['Servers'][vomsHost]['DN'] + CA = vomsDict[vo]['Servers'][vomsHost]['CA'] + port = vomsDict[vo]['Servers'][vomsHost]['Port'] + if not DN or not CA or not port: + DIRAC.gLogger.error('DN = %s' % DN) + DIRAC.gLogger.error('CA = %s' % CA) + DIRAC.gLogger.error('Port = %s' % port) + DIRAC.gLogger.error('Missing Parameter for %s' % vomsHost) + continue + with open(hostFilePath, "wt") as fd: + fd.write("%s\n%s\n" % (DN, CA)) + vomsesLines.append('"%s" "%s" "%s" "%s" "%s" "24"' % (voName, vomsHost, port, DN, voName)) + DIRAC.gLogger.notice("Created vomsdir file %s" % hostFilePath) + except Exception: + DIRAC.gLogger.exception("Could not generate vomsdir file for host", vomsHost) + error = "Could not generate vomsdir file for VO %s, host %s" % (voName, vomsHost) + try: + vomsesFilePath = os.path.join(vomsesDirPath, voName) + with open(vomsesFilePath, "wt") as fd: + fd.write("%s\n" % "\n".join(vomsesLines)) + DIRAC.gLogger.notice("Created vomses file %s" % vomsesFilePath) + except Exception: + DIRAC.gLogger.exception("Could not generate vomses file") + error = "Could not generate vomses file for VO %s" % voName -# ## LAST PART: do the vomsdir/vomses magic + if useServerCert: + Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') + # When using Server Certs CA's will be checked, the flag only disables initial download + # this will be replaced by the use of SkipCADownload + Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') -# This has to be done for all VOs in the installation + if error: + sys.exit(1) -if skipVOMSDownload: - # We stop here sys.exit(0) -result = Registry.getVOMSServerInfo() -if not result['OK']: - sys.exit(1) - -error = '' -vomsDict = result['Value'] -for vo in vomsDict: - voName = vomsDict[vo]['VOMSName'] - vomsDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'vomsdir', voName) - vomsesDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'vomses') - for path in (vomsDirPath, vomsesDirPath): - mkDir(path) - vomsesLines = [] - for vomsHost in vomsDict[vo].get('Servers', {}): - hostFilePath = os.path.join(vomsDirPath, "%s.lsc" % vomsHost) - try: - DN = vomsDict[vo]['Servers'][vomsHost]['DN'] - CA = vomsDict[vo]['Servers'][vomsHost]['CA'] - port = vomsDict[vo]['Servers'][vomsHost]['Port'] - if not DN or not CA or not port: - DIRAC.gLogger.error('DN = %s' % DN) - DIRAC.gLogger.error('CA = %s' % CA) - DIRAC.gLogger.error('Port = %s' % port) - DIRAC.gLogger.error('Missing Parameter for %s' % vomsHost) - continue - with open(hostFilePath, "wt") as fd: - fd.write("%s\n%s\n" % (DN, CA)) - vomsesLines.append('"%s" "%s" "%s" "%s" "%s" "24"' % (voName, vomsHost, port, DN, voName)) - DIRAC.gLogger.notice("Created vomsdir file %s" % hostFilePath) - except Exception: - DIRAC.gLogger.exception("Could not generate vomsdir file for host", vomsHost) - error = "Could not generate vomsdir file for VO %s, host %s" % (voName, vomsHost) - try: - vomsesFilePath = os.path.join(vomsesDirPath, voName) - with open(vomsesFilePath, "wt") as fd: - fd.write("%s\n" % "\n".join(vomsesLines)) - DIRAC.gLogger.notice("Created vomses file %s" % vomsesFilePath) - except Exception: - DIRAC.gLogger.exception("Could not generate vomses file") - error = "Could not generate vomses file for VO %s" % voName - -if useServerCert: - Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') - # When using Server Certs CA's will be checked, the flag only disables initial download - # this will be replaced by the use of SkipCADownload - Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') - -if error: - sys.exit(1) - -sys.exit(0) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_executor.py b/src/DIRAC/Core/scripts/dirac_executor.py index 76b578c5017..d9a2970b24f 100755 --- a/src/DIRAC/Core/scripts/dirac_executor.py +++ b/src/DIRAC/Core/scripts/dirac_executor.py @@ -16,41 +16,49 @@ from DIRAC import gLogger from DIRAC.Core.Base.ExecutorReactor import ExecutorReactor from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -localCfg = LocalConfiguration() - -positionalArgs = localCfg.getPositionalArguments() -if len(positionalArgs) == 0: - gLogger.fatal("You must specify which executor to run!") - sys.exit(1) - -if len(positionalArgs) == 1 and positionalArgs[0].find("/") > -1: - mainName = positionalArgs[0] -else: - mainName = "Framework/MultiExecutor" - -localCfg.setConfigurationForExecutor(mainName) -localCfg.addMandatoryEntry("/DIRAC/Setup") -localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") -localCfg.addDefaultEntry("LogLevel", "INFO") -localCfg.addDefaultEntry("LogColor", True) -resultDict = localCfg.loadUserData() -if not resultDict['OK']: - gLogger.fatal("There were errors when loading configuration", resultDict['Message']) - sys.exit(1) - -includeExtensionErrors() -executorReactor = ExecutorReactor() - -result = executorReactor.loadModules(positionalArgs) -if not result['OK']: - gLogger.fatal("Error while loading executor", result['Message']) - sys.exit(1) - -result = executorReactor.go() -if not result['OK']: - gLogger.fatal(result['Message']) - sys.exit(1) - -gLogger.notice("Graceful exit. Bye!") -sys.exit(0) + +@DIRACScript() +def main(): + localCfg = LocalConfiguration() + + positionalArgs = localCfg.getPositionalArguments() + if len(positionalArgs) == 0: + gLogger.fatal("You must specify which executor to run!") + sys.exit(1) + + if len(positionalArgs) == 1 and positionalArgs[0].find("/") > -1: + mainName = positionalArgs[0] + else: + mainName = "Framework/MultiExecutor" + + localCfg.setConfigurationForExecutor(mainName) + localCfg.addMandatoryEntry("/DIRAC/Setup") + localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") + localCfg.addDefaultEntry("LogLevel", "INFO") + localCfg.addDefaultEntry("LogColor", True) + resultDict = localCfg.loadUserData() + if not resultDict['OK']: + gLogger.fatal("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + + includeExtensionErrors() + executorReactor = ExecutorReactor() + + result = executorReactor.loadModules(positionalArgs) + if not result['OK']: + gLogger.fatal("Error while loading executor", result['Message']) + sys.exit(1) + + result = executorReactor.go() + if not result['OK']: + gLogger.fatal(result['Message']) + sys.exit(1) + + gLogger.notice("Graceful exit. Bye!") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_externals_requirements.py b/src/DIRAC/Core/scripts/dirac_externals_requirements.py index 71753e17d7e..9817f819d29 100755 --- a/src/DIRAC/Core/scripts/dirac_externals_requirements.py +++ b/src/DIRAC/Core/scripts/dirac_externals_requirements.py @@ -24,8 +24,7 @@ from diraccfg import CFG from DIRAC.Core.Base import Script -Script.disableCS() - +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import gLogger, rootPath, S_OK __RCSID__ = "$Id$" @@ -40,10 +39,6 @@ def setInstallType(val): return S_OK() -Script.registerSwitch("t:", "type=", "Installation type. 'server' by default.", setInstallType) -Script.parseCommandLine(ignoreErrors=True) - - def pipInstall(package, switches=""): # The right pip should be in the PATH, which is the case after sourcing the DIRAC bashrc cmd = "pip install --trusted-host pypi.python.org %s %s" % (switches, package) @@ -51,49 +46,58 @@ def pipInstall(package, switches=""): return commands.getstatusoutput(cmd) -# Collect all the requested python modules to install -reqDict = {} - - -for entry in os.listdir(rootPath): - if len(entry) < 5 or entry.find("DIRAC") != len(entry) - 5: - continue - reqFile = os.path.join(rootPath, entry, "releases.cfg") - try: - with open(reqFile, "r") as extfd: - reqCFG = CFG().loadFromBuffer(extfd.read()) - except BaseException: - gLogger.verbose("%s not found" % reqFile) - continue - reqList = reqCFG.getOption("/RequiredExternals/%s" % instType.capitalize(), []) - if not reqList: - gLogger.verbose("%s does not have requirements for %s installation" % (entry, instType)) - continue - for req in reqList: - reqName = False - reqCond = "" - for cond in ("==", ">="): - iP = cond.find(req) - if iP > 0: - reqName = req[:iP] - reqCond = req[iP:] - break - if not reqName: - reqName = req - if reqName not in reqDict: - reqDict[reqName] = (reqCond, entry) +@DIRACScript() +def main(): + Script.disableCS() + + Script.registerSwitch("t:", "type=", "Installation type. 'server' by default.", setInstallType) + Script.parseCommandLine(ignoreErrors=True) + + # Collect all the requested python modules to install + reqDict = {} + for entry in os.listdir(rootPath): + if len(entry) < 5 or entry.find("DIRAC") != len(entry) - 5: + continue + reqFile = os.path.join(rootPath, entry, "releases.cfg") + try: + with open(reqFile, "r") as extfd: + reqCFG = CFG().loadFromBuffer(extfd.read()) + except BaseException: + gLogger.verbose("%s not found" % reqFile) + continue + reqList = reqCFG.getOption("/RequiredExternals/%s" % instType.capitalize(), []) + if not reqList: + gLogger.verbose("%s does not have requirements for %s installation" % (entry, instType)) + continue + for req in reqList: + reqName = False + reqCond = "" + for cond in ("==", ">="): + iP = cond.find(req) + if iP > 0: + reqName = req[:iP] + reqCond = req[iP:] + break + if not reqName: + reqName = req + if reqName not in reqDict: + reqDict[reqName] = (reqCond, entry) + else: + gLogger.notice("Skipping %s, it's already requested by %s" % (reqName, reqDict[reqName][1])) + + if not reqDict: + gLogger.notice("No extra python module requested to be installed") + sys.exit(0) + + for reqName in reqDict: + package = "%s%s" % (reqName, reqDict[reqName][0]) + gLogger.notice("Requesting installation of %s" % package) + status, output = pipInstall(package) + if status != 0: + gLogger.error(output) else: - gLogger.notice("Skipping %s, it's already requested by %s" % (reqName, reqDict[reqName][1])) - -if not reqDict: - gLogger.notice("No extra python module requested to be installed") - sys.exit(0) - -for reqName in reqDict: - package = "%s%s" % (reqName, reqDict[reqName][0]) - gLogger.notice("Requesting installation of %s" % package) - status, output = pipInstall(package) - if status != 0: - gLogger.error(output) - else: - gLogger.notice("Successfully installed %s" % package) + gLogger.notice("Successfully installed %s" % package) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_install_db.py b/src/DIRAC/Core/scripts/dirac_install_db.py index 1b7a1b62061..0778e809c4f 100755 --- a/src/DIRAC/Core/scripts/dirac_install_db.py +++ b/src/DIRAC/Core/scripts/dirac_install_db.py @@ -7,34 +7,43 @@ from __future__ import absolute_import from __future__ import division from DIRAC.Core.Base import Script -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgFile] ... DB ...' % Script.scriptName, - 'Arguments:', - ' DB: Name of the Database (mandatory)'])) -Script.parseCommandLine() -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) - -# Script imports -from DIRAC import gConfig -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller -from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities - -__RCSID__ = "$Id$" - -gComponentInstaller.exitOnError = True -gComponentInstaller.getMySQLPasswords() -for db in args: - result = gComponentInstaller.installDatabase(db) - if not result['OK']: - print("ERROR: failed to correctly install %s" % db, result['Message']) - else: - extension, system = result['Value'] - gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, db, overwrite=True) - - if db != 'InstalledComponentsDB': - result = MonitoringUtilities.monitorInstallation('DB', system, db) - if not result['OK']: - print("ERROR: failed to register installation in database: %s" % result['Message']) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgFile] ... DB ...' % Script.scriptName, + 'Arguments:', + ' DB: Name of the Database (mandatory)'])) + Script.parseCommandLine() + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) + + # Script imports + from DIRAC import gConfig + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities + + __RCSID__ = "$Id$" + + gComponentInstaller.exitOnError = True + gComponentInstaller.getMySQLPasswords() + for db in args: + result = gComponentInstaller.installDatabase(db) + if not result['OK']: + print("ERROR: failed to correctly install %s" % db, result['Message']) + else: + extension, system = result['Value'] + gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, db, overwrite=True) + + if db != 'InstalledComponentsDB': + result = MonitoringUtilities.monitorInstallation('DB', system, db) + if not result['OK']: + print("ERROR: failed to register installation in database: %s" % result['Message']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_install_web_portal.py b/src/DIRAC/Core/scripts/dirac_install_web_portal.py index 77cbf31161d..4e8d7f81af8 100755 --- a/src/DIRAC/Core/scripts/dirac_install_web_portal.py +++ b/src/DIRAC/Core/scripts/dirac_install_web_portal.py @@ -12,16 +12,23 @@ __RCSID__ = "$Id$" # from DIRAC.Core.Base import Script -Script.disableCS() -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % Script.scriptName])) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.parseCommandLine() -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller +@DIRACScript() +def main(): + Script.disableCS() + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % Script.scriptName])) -gComponentInstaller.exitOnError = True + Script.parseCommandLine() + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller -gComponentInstaller.setupPortal() + gComponentInstaller.exitOnError = True + gComponentInstaller.setupPortal() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_service.py b/src/DIRAC/Core/scripts/dirac_service.py index 49db27aa5fd..7626ce51622 100755 --- a/src/DIRAC/Core/scripts/dirac_service.py +++ b/src/DIRAC/Core/scripts/dirac_service.py @@ -13,41 +13,47 @@ from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.Core.DISET.ServiceReactor import ServiceReactor from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +from DIRAC.Core.Utilities.DIRACScript import DIRACScript __RCSID__ = "$Id$" -localCfg = LocalConfiguration() - - -positionalArgs = localCfg.getPositionalArguments() -if len(positionalArgs) == 0: - gLogger.fatal("You must specify which server to run!") - sys.exit(1) - -serverName = positionalArgs[0] -localCfg.setConfigurationForServer(serverName) -localCfg.addMandatoryEntry("Port") -# localCfg.addMandatoryEntry( "HandlerPath" ) -localCfg.addMandatoryEntry("/DIRAC/Setup") -localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") -localCfg.addDefaultEntry("LogLevel", "INFO") -localCfg.addDefaultEntry("LogColor", True) -resultDict = localCfg.loadUserData() -if not resultDict['OK']: - gLogger.initialize(serverName, "/") - gLogger.error("There were errors when loading configuration", resultDict['Message']) - sys.exit(1) - -includeExtensionErrors() - - -serverToLaunch = ServiceReactor() -result = serverToLaunch.initialize(positionalArgs) -if not result['OK']: - gLogger.error(result['Message']) - sys.exit(1) - -result = serverToLaunch.serve() -if not result['OK']: - gLogger.error(result['Message']) - sys.exit(1) + +@DIRACScript() +def main(): + localCfg = LocalConfiguration() + + positionalArgs = localCfg.getPositionalArguments() + if len(positionalArgs) == 0: + gLogger.fatal("You must specify which server to run!") + sys.exit(1) + + serverName = positionalArgs[0] + localCfg.setConfigurationForServer(serverName) + localCfg.addMandatoryEntry("Port") + # localCfg.addMandatoryEntry( "HandlerPath" ) + localCfg.addMandatoryEntry("/DIRAC/Setup") + localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") + localCfg.addDefaultEntry("LogLevel", "INFO") + localCfg.addDefaultEntry("LogColor", True) + resultDict = localCfg.loadUserData() + if not resultDict['OK']: + gLogger.initialize(serverName, "/") + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + + includeExtensionErrors() + + serverToLaunch = ServiceReactor() + result = serverToLaunch.initialize(positionalArgs) + if not result['OK']: + gLogger.error(result['Message']) + sys.exit(1) + + result = serverToLaunch.serve() + if not result['OK']: + gLogger.error(result['Message']) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_setup_site.py b/src/DIRAC/Core/scripts/dirac_setup_site.py index 7945086a73d..a99d454ff84 100755 --- a/src/DIRAC/Core/scripts/dirac_setup_site.py +++ b/src/DIRAC/Core/scripts/dirac_setup_site.py @@ -13,6 +13,7 @@ from DIRAC import S_OK from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript class Params(object): @@ -25,48 +26,54 @@ def setExitOnError(self, value): return S_OK() -cliParams = Params() - -Script.disableCS() -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option] ... [cfgfile]' % Script.scriptName, - 'Arguments:', - ' cfgfile: DIRAC Cfg with description of the configuration (optional)'])) - -Script.registerSwitch( - "e", - "exitOnError", - "flag to exit on error of any component installation", - cliParams.setExitOnError) - -Script.addDefaultOptionValue('/DIRAC/Security/UseServerCertificate', 'yes') -Script.addDefaultOptionValue('LogLevel', 'INFO') -Script.parseCommandLine() -args = Script.getExtraCLICFGFiles() -# -if len(args) > 1: - Script.showHelp(exitCode=1) -# -cfg = None -if len(args): - cfg = args[0] -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller -# -gComponentInstaller.exitOnError = cliParams.exitOnError -# -result = gComponentInstaller.setupSite(Script.localCfg, cfg) -if not result['OK']: - print("ERROR:", result['Message']) - exit(-1) -# -result = gComponentInstaller.getStartupComponentStatus([]) -if not result['OK']: - print('ERROR:', result['Message']) - exit(-1) - -print("\nStatus of installed components:\n") -result = gComponentInstaller.printStartupStatus(result['Value']) -if not result['OK']: - print('ERROR:', result['Message']) - exit(-1) +@DIRACScript() +def main(): + cliParams = Params() + + Script.disableCS() + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option] ... [cfgfile]' % Script.scriptName, + 'Arguments:', + ' cfgfile: DIRAC Cfg with description of the configuration (optional)'])) + + Script.registerSwitch( + "e", + "exitOnError", + "flag to exit on error of any component installation", + cliParams.setExitOnError) + + Script.addDefaultOptionValue('/DIRAC/Security/UseServerCertificate', 'yes') + Script.addDefaultOptionValue('LogLevel', 'INFO') + Script.parseCommandLine() + args = Script.getExtraCLICFGFiles() + + if len(args) > 1: + Script.showHelp(exitCode=1) + + cfg = None + if len(args): + cfg = args[0] + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + + gComponentInstaller.exitOnError = cliParams.exitOnError + + result = gComponentInstaller.setupSite(Script.localCfg, cfg) + if not result['OK']: + print("ERROR:", result['Message']) + exit(-1) + + result = gComponentInstaller.getStartupComponentStatus([]) + if not result['OK']: + print('ERROR:', result['Message']) + exit(-1) + + print("\nStatus of installed components:\n") + result = gComponentInstaller.printStartupStatus(result['Value']) + if not result['OK']: + print('ERROR:', result['Message']) + exit(-1) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_version.py b/src/DIRAC/Core/scripts/dirac_version.py index 09dce990d0c..6cd1072a1b4 100755 --- a/src/DIRAC/Core/scripts/dirac_version.py +++ b/src/DIRAC/Core/scripts/dirac_version.py @@ -11,4 +11,13 @@ from __future__ import division import DIRAC -print(DIRAC.version) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + print(DIRAC.version) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py index 1baafd4d3f0..a8ce614b4f6 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py @@ -8,192 +8,196 @@ import DIRAC from DIRAC.Core.Base import Script - -read = False -write = False -check = False -remove = False -site = '' -mute = False - -Script.setUsageMessage(""" -Enable using one or more Storage Elements - -Usage: - %s SE1 [SE2 ...] -""" % Script.scriptName) - -Script.registerSwitch("r", "AllowRead", " Allow only reading from the storage element") -Script.registerSwitch("w", "AllowWrite", " Allow only writing to the storage element") -Script.registerSwitch("k", "AllowCheck", " Allow only check access to the storage element") -Script.registerSwitch("v", "AllowRemove", " Allow only remove access to the storage element") -Script.registerSwitch("a", "All", " Allow all access to the storage element") -Script.registerSwitch("m", "Mute", " Do not send email") -Script.registerSwitch("S:", "Site=", " Allow all SEs associated to site") - -Script.parseCommandLine(ignoreErrors=True) - -ses = Script.getPositionalArgs() -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() in ("r", "allowread"): - read = True - if switch[0].lower() in ("w", "allowwrite"): - write = True - if switch[0].lower() in ("k", "allowcheck"): - check = True - if switch[0].lower() in ("v", "allowremove"): - remove = True - if switch[0].lower() in ("a", "all"): +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + read = False + write = False + check = False + remove = False + site = '' + mute = False + + Script.setUsageMessage(""" + Enable using one or more Storage Elements + + Usage: + %s SE1 [SE2 ...] + """ % Script.scriptName) + + Script.registerSwitch("r", "AllowRead", " Allow only reading from the storage element") + Script.registerSwitch("w", "AllowWrite", " Allow only writing to the storage element") + Script.registerSwitch("k", "AllowCheck", " Allow only check access to the storage element") + Script.registerSwitch("v", "AllowRemove", " Allow only remove access to the storage element") + Script.registerSwitch("a", "All", " Allow all access to the storage element") + Script.registerSwitch("m", "Mute", " Do not send email") + Script.registerSwitch("S:", "Site=", " Allow all SEs associated to site") + + Script.parseCommandLine(ignoreErrors=True) + + ses = Script.getPositionalArgs() + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() in ("r", "allowread"): + read = True + if switch[0].lower() in ("w", "allowwrite"): + write = True + if switch[0].lower() in ("k", "allowcheck"): + check = True + if switch[0].lower() in ("v", "allowremove"): + remove = True + if switch[0].lower() in ("a", "all"): + read = True + write = True + check = True + remove = True + if switch[0].lower() in ("m", "mute"): + mute = True + if switch[0].lower() in ("s", "site"): + site = switch[1] + + # imports + from DIRAC import gConfig, gLogger + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus + + if not (read or write or check or remove): + # No switch was specified, means we need all of them + gLogger.notice("No option given, all accesses will be allowed if they were not") read = True write = True check = True remove = True - if switch[0].lower() in ("m", "mute"): - mute = True - if switch[0].lower() in ("s", "site"): - site = switch[1] - -# imports -from DIRAC import gConfig, gLogger -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus - -if not (read or write or check or remove): - # No switch was specified, means we need all of them - gLogger.notice("No option given, all accesses will be allowed if they were not") - read = True - write = True - check = True - remove = True - -ses = resolveSEGroup(ses) -diracAdmin = DiracAdmin() -errorList = [] -setup = gConfig.getValue('/DIRAC/Setup', '') -if not setup: - print('ERROR: Could not contact Configuration Service') - DIRAC.exit(2) - -res = getProxyInfo() -if not res['OK']: - gLogger.error('Failed to get proxy information', res['Message']) - DIRAC.exit(2) - -userName = res['Value'].get('username') -if not userName: - gLogger.error('Failed to get username for proxy') - DIRAC.exit(2) - -if site: - res = getSites() - if not res['OK']: - gLogger.error(res['Message']) - DIRAC.exit(-1) - if site not in res['Value']: - gLogger.error('The provided site (%s) is not known.' % site) - DIRAC.exit(-1) - ses.extend(res['Value']['SE'].replace(' ', '').split(',')) -if not ses: - gLogger.error('There were no SEs provided') - DIRAC.exit() -STATUS_TYPES = ["ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess"] -ALLOWED_STATUSES = ["Unknown", "InActive", "Banned", "Probing", "Degraded"] + ses = resolveSEGroup(ses) + diracAdmin = DiracAdmin() + errorList = [] + setup = gConfig.getValue('/DIRAC/Setup', '') + if not setup: + print('ERROR: Could not contact Configuration Service') + DIRAC.exit(2) -statusAllowedDict = {} -for statusType in STATUS_TYPES: - statusAllowedDict[statusType] = [] - -statusFlagDict = {} -statusFlagDict['ReadAccess'] = read -statusFlagDict['WriteAccess'] = write -statusFlagDict['CheckAccess'] = check -statusFlagDict['RemoveAccess'] = remove + res = getProxyInfo() + if not res['OK']: + gLogger.error('Failed to get proxy information', res['Message']) + DIRAC.exit(2) + + userName = res['Value'].get('username') + if not userName: + gLogger.error('Failed to get username for proxy') + DIRAC.exit(2) + + if site: + res = getSites() + if not res['OK']: + gLogger.error(res['Message']) + DIRAC.exit(-1) + if site not in res['Value']: + gLogger.error('The provided site (%s) is not known.' % site) + DIRAC.exit(-1) + ses.extend(res['Value']['SE'].replace(' ', '').split(',')) + if not ses: + gLogger.error('There were no SEs provided') + DIRAC.exit() + + STATUS_TYPES = ["ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess"] + ALLOWED_STATUSES = ["Unknown", "InActive", "Banned", "Probing", "Degraded"] + + statusAllowedDict = {} + for statusType in STATUS_TYPES: + statusAllowedDict[statusType] = [] -resourceStatus = ResourceStatus() + statusFlagDict = {} + statusFlagDict['ReadAccess'] = read + statusFlagDict['WriteAccess'] = write + statusFlagDict['CheckAccess'] = check + statusFlagDict['RemoveAccess'] = remove -res = resourceStatus.getElementStatus(ses, "StorageElement") -if not res['OK']: - gLogger.error('Storage Element %s does not exist' % ses) - DIRAC.exit(-1) + resourceStatus = ResourceStatus() -reason = 'Forced with dirac-admin-allow-se by %s' % userName + res = resourceStatus.getElementStatus(ses, "StorageElement") + if not res['OK']: + gLogger.error('Storage Element %s does not exist' % ses) + DIRAC.exit(-1) -for se, seOptions in res['Value'].items(): - # InActive is used on the CS model, Banned is the equivalent in RSS - for statusType in STATUS_TYPES: - if statusFlagDict[statusType]: - if seOptions.get(statusType) == "Active": - gLogger.notice('%s status of %s is already Active' % (statusType, se)) - continue - if statusType in seOptions: - if not seOptions[statusType] in ALLOWED_STATUSES: - gLogger.notice('%s option for %s is %s, instead of %s' % - (statusType, se, seOptions['ReadAccess'], ALLOWED_STATUSES)) - gLogger.notice('Try specifying the command switches') - else: - resR = resourceStatus.setElementStatus(se, "StorageElement", statusType, 'Active', reason, userName) - if not resR['OK']: - gLogger.fatal("Failed to update %s %s to Active, exit -" % (se, statusType), resR['Message']) - DIRAC.exit(-1) + reason = 'Forced with dirac-admin-allow-se by %s' % userName + + for se, seOptions in res['Value'].items(): + # InActive is used on the CS model, Banned is the equivalent in RSS + for statusType in STATUS_TYPES: + if statusFlagDict[statusType]: + if seOptions.get(statusType) == "Active": + gLogger.notice('%s status of %s is already Active' % (statusType, se)) + continue + if statusType in seOptions: + if not seOptions[statusType] in ALLOWED_STATUSES: + gLogger.notice('%s option for %s is %s, instead of %s' % + (statusType, se, seOptions['ReadAccess'], ALLOWED_STATUSES)) + gLogger.notice('Try specifying the command switches') else: - gLogger.notice("Successfully updated %s %s to Active" % (se, statusType)) - statusAllowedDict[statusType].append(se) - -totalAllowed = 0 -totalAllowedSEs = [] -for statusType in STATUS_TYPES: - totalAllowed += len(statusAllowedDict[statusType]) - totalAllowedSEs += statusAllowedDict[statusType] -totalAllowedSEs = list(set(totalAllowedSEs)) - -if not totalAllowed: - gLogger.info("No storage elements were allowed") - DIRAC.exit(-1) - -if mute: - gLogger.notice('Email is muted by script switch') - DIRAC.exit(0) + resR = resourceStatus.setElementStatus(se, "StorageElement", statusType, 'Active', reason, userName) + if not resR['OK']: + gLogger.fatal("Failed to update %s %s to Active, exit -" % (se, statusType), resR['Message']) + DIRAC.exit(-1) + else: + gLogger.notice("Successfully updated %s %s to Active" % (se, statusType)) + statusAllowedDict[statusType].append(se) + + totalAllowed = 0 + totalAllowedSEs = [] + for statusType in STATUS_TYPES: + totalAllowed += len(statusAllowedDict[statusType]) + totalAllowedSEs += statusAllowedDict[statusType] + totalAllowedSEs = list(set(totalAllowedSEs)) -subject = '%s storage elements allowed for use' % len(totalAllowedSEs) -addressPath = 'EMail/Production' -address = Operations().getValue(addressPath, '') - - -body = '' -if read: - body = "%s\n\nThe following storage elements were allowed for reading:" % body - for se in statusAllowedDict['ReadAccess']: - body = "%s\n%s" % (body, se) -if write: - body = "%s\n\nThe following storage elements were allowed for writing:" % body - for se in statusAllowedDict['WriteAccess']: - body = "%s\n%s" % (body, se) -if check: - body = "%s\n\nThe following storage elements were allowed for checking:" % body - for se in statusAllowedDict['CheckAccess']: - body = "%s\n%s" % (body, se) -if remove: - body = "%s\n\nThe following storage elements were allowed for removing:" % body - for se in statusAllowedDict['RemoveAccess']: - body = "%s\n%s" % (body, se) - -if not address: - gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) - DIRAC.exit(0) + if not totalAllowed: + gLogger.info("No storage elements were allowed") + DIRAC.exit(-1) -res = diracAdmin.sendMail(address, subject, body) -gLogger.notice('Notifying %s' % address) -if res['OK']: - gLogger.notice(res['Value']) -else: - gLogger.notice(res['Message']) + if mute: + gLogger.notice('Email is muted by script switch') + DIRAC.exit(0) + + subject = '%s storage elements allowed for use' % len(totalAllowedSEs) + addressPath = 'EMail/Production' + address = Operations().getValue(addressPath, '') + + body = '' + if read: + body = "%s\n\nThe following storage elements were allowed for reading:" % body + for se in statusAllowedDict['ReadAccess']: + body = "%s\n%s" % (body, se) + if write: + body = "%s\n\nThe following storage elements were allowed for writing:" % body + for se in statusAllowedDict['WriteAccess']: + body = "%s\n%s" % (body, se) + if check: + body = "%s\n\nThe following storage elements were allowed for checking:" % body + for se in statusAllowedDict['CheckAccess']: + body = "%s\n%s" % (body, se) + if remove: + body = "%s\n\nThe following storage elements were allowed for removing:" % body + for se in statusAllowedDict['RemoveAccess']: + body = "%s\n%s" % (body, se) + + if not address: + gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) + DIRAC.exit(0) + + res = diracAdmin.sendMail(address, subject, body) + gLogger.notice('Notifying %s' % address) + if res['OK']: + gLogger.notice(res['Value']) + else: + gLogger.notice(res['Message']) + + DIRAC.exit(0) -DIRAC.exit(0) -################################################################################ -# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py index 04519a5cd21..5bc1392e1b3 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py @@ -9,226 +9,234 @@ import DIRAC from DIRAC.Core.Base import Script - -read = True -write = True -check = True -remove = True -site = '' -mute = False - -Script.setUsageMessage(""" -Ban one or more Storage Elements for usage - -Usage: - %s SE1 [SE2 ...] -""" % Script.scriptName) - -Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element") -Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element") -Script.registerSwitch("k", "BanCheck", " Ban check access to the storage element") -Script.registerSwitch("v", "BanRemove", " Ban remove access to the storage element") -Script.registerSwitch("a", "All", " Ban all access to the storage element") -Script.registerSwitch("m", "Mute", " Do not send email") -Script.registerSwitch( - "S:", - "Site=", - " Ban all SEs associate to site (note that if writing is allowed, check is always allowed)") -Script.parseCommandLine(ignoreErrors=True) - -ses = Script.getPositionalArgs() -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() in ("r", "banread"): - write = False - check = False - remove = False - if switch[0].lower() in ("w", "banwrite"): - read = False - check = False - remove = False - if switch[0].lower() in ("k", "bancheck"): - read = False - write = False - remove = False - if switch[0].lower() in ("v", "banremove"): - read = False - write = False - check = False - if switch[0].lower() in ("a", "all"): - pass - if switch[0].lower() in ("m", "mute"): - mute = True - if switch[0].lower() in ("s", "site"): - site = switch[1] - -# from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI -from DIRAC import gConfig, gLogger -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus -from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup - -ses = resolveSEGroup(ses) -diracAdmin = DiracAdmin() -errorList = [] -setup = gConfig.getValue('/DIRAC/Setup', '') -if not setup: - print('ERROR: Could not contact Configuration Service') - DIRAC.exit(2) - -res = getProxyInfo() -if not res['OK']: - gLogger.error('Failed to get proxy information', res['Message']) - DIRAC.exit(2) - -userName = res['Value'].get('username') -if not userName: - gLogger.error('Failed to get username for proxy') - DIRAC.exit(2) - -if site: - res = getSites() +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + read = True + write = True + check = True + remove = True + site = '' + mute = False + + Script.setUsageMessage(""" + Ban one or more Storage Elements for usage + + Usage: + %s SE1 [SE2 ...] + """ % Script.scriptName) + + Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element") + Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element") + Script.registerSwitch("k", "BanCheck", " Ban check access to the storage element") + Script.registerSwitch("v", "BanRemove", " Ban remove access to the storage element") + Script.registerSwitch("a", "All", " Ban all access to the storage element") + Script.registerSwitch("m", "Mute", " Do not send email") + Script.registerSwitch( + "S:", + "Site=", + " Ban all SEs associate to site (note that if writing is allowed, check is always allowed)") + Script.parseCommandLine(ignoreErrors=True) + + ses = Script.getPositionalArgs() + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() in ("r", "banread"): + write = False + check = False + remove = False + if switch[0].lower() in ("w", "banwrite"): + read = False + check = False + remove = False + if switch[0].lower() in ("k", "bancheck"): + read = False + write = False + remove = False + if switch[0].lower() in ("v", "banremove"): + read = False + write = False + check = False + if switch[0].lower() in ("a", "all"): + pass + if switch[0].lower() in ("m", "mute"): + mute = True + if switch[0].lower() in ("s", "site"): + site = switch[1] + + # from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI + from DIRAC import gConfig, gLogger + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus + from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup + + ses = resolveSEGroup(ses) + diracAdmin = DiracAdmin() + errorList = [] + setup = gConfig.getValue('/DIRAC/Setup', '') + if not setup: + print('ERROR: Could not contact Configuration Service') + DIRAC.exit(2) + + res = getProxyInfo() if not res['OK']: - gLogger.error(res['Message']) - DIRAC.exit(-1) - if site not in res['Value']: - gLogger.error('The provided site (%s) is not known.' % site) + gLogger.error('Failed to get proxy information', res['Message']) + DIRAC.exit(2) + + userName = res['Value'].get('username') + if not userName: + gLogger.error('Failed to get username for proxy') + DIRAC.exit(2) + + if site: + res = getSites() + if not res['OK']: + gLogger.error(res['Message']) + DIRAC.exit(-1) + if site not in res['Value']: + gLogger.error('The provided site (%s) is not known.' % site) + DIRAC.exit(-1) + ses.extend(res['Value']['SE'].replace(' ', '').split(',')) + + if not ses: + gLogger.error('There were no SEs provided') DIRAC.exit(-1) - ses.extend(res['Value']['SE'].replace(' ', '').split(',')) - -if not ses: - gLogger.error('There were no SEs provided') - DIRAC.exit(-1) - -readBanned = [] -writeBanned = [] -checkBanned = [] -removeBanned = [] -resourceStatus = ResourceStatus() + readBanned = [] + writeBanned = [] + checkBanned = [] + removeBanned = [] -res = resourceStatus.getElementStatus(ses, "StorageElement") -if not res['OK']: - gLogger.error("Storage Element %s does not exist" % ses) - DIRAC.exit(-1) + resourceStatus = ResourceStatus() -reason = 'Forced with dirac-admin-ban-se by %s' % userName + res = resourceStatus.getElementStatus(ses, "StorageElement") + if not res['OK']: + gLogger.error("Storage Element %s does not exist" % ses) + DIRAC.exit(-1) -for se, seOptions in res['Value'].items(): + reason = 'Forced with dirac-admin-ban-se by %s' % userName - resW = resC = resR = {'OK': False} + for se, seOptions in res['Value'].items(): - # Eventually, we will get rid of the notion of InActive, as we always write Banned. - if read and 'ReadAccess' in seOptions: + resW = resC = resR = {'OK': False} - if not seOptions['ReadAccess'] in ['Active', 'Degraded', 'Probing']: - gLogger.notice('Read option for %s is %s, instead of %s' % - (se, seOptions['ReadAccess'], ['Active', 'Degraded', 'Probing'])) - gLogger.notice('Try specifying the command switches') - else: + # Eventually, we will get rid of the notion of InActive, as we always write Banned. + if read and 'ReadAccess' in seOptions: - resR = resourceStatus.setElementStatus(se, 'StorageElement', 'ReadAccess', 'Banned', reason, userName) - # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) - if not resR['OK']: - gLogger.error('Failed to update %s read access to Banned' % se) + if not seOptions['ReadAccess'] in ['Active', 'Degraded', 'Probing']: + gLogger.notice('Read option for %s is %s, instead of %s' % + (se, seOptions['ReadAccess'], ['Active', 'Degraded', 'Probing'])) + gLogger.notice('Try specifying the command switches') else: - gLogger.notice('Successfully updated %s read access to Banned' % se) - readBanned.append(se) - - # Eventually, we will get rid of the notion of InActive, as we always write Banned. - if write and 'WriteAccess' in seOptions: - - if not seOptions['WriteAccess'] in ['Active', 'Degraded', 'Probing']: - gLogger.notice('Write option for %s is %s, instead of %s' % - (se, seOptions['WriteAccess'], ['Active', 'Degraded', 'Probing'])) - gLogger.notice('Try specifying the command switches') - else: - - resW = resourceStatus.setElementStatus(se, 'StorageElement', 'WriteAccess', 'Banned', reason, userName) - # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) - if not resW['OK']: - gLogger.error("Failed to update %s write access to Banned" % se) + + resR = resourceStatus.setElementStatus(se, 'StorageElement', 'ReadAccess', 'Banned', reason, userName) + # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) + if not resR['OK']: + gLogger.error('Failed to update %s read access to Banned' % se) + else: + gLogger.notice('Successfully updated %s read access to Banned' % se) + readBanned.append(se) + + # Eventually, we will get rid of the notion of InActive, as we always write Banned. + if write and 'WriteAccess' in seOptions: + + if not seOptions['WriteAccess'] in ['Active', 'Degraded', 'Probing']: + gLogger.notice('Write option for %s is %s, instead of %s' % + (se, seOptions['WriteAccess'], ['Active', 'Degraded', 'Probing'])) + gLogger.notice('Try specifying the command switches') else: - gLogger.notice("Successfully updated %s write access to Banned" % se) - writeBanned.append(se) - - # Eventually, we will get rid of the notion of InActive, as we always write Banned. - if check and 'CheckAccess' in seOptions: - - if not seOptions['CheckAccess'] in ['Active', 'Degraded', 'Probing']: - gLogger.notice('Check option for %s is %s, instead of %s' % - (se, seOptions['CheckAccess'], ['Active', 'Degraded', 'Probing'])) - gLogger.notice('Try specifying the command switches') - else: - - resC = resourceStatus.setElementStatus(se, 'StorageElement', 'CheckAccess', 'Banned', reason, userName) - # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) - if not resC['OK']: - gLogger.error("Failed to update %s check access to Banned" % se) + + resW = resourceStatus.setElementStatus(se, 'StorageElement', 'WriteAccess', 'Banned', reason, userName) + # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) + if not resW['OK']: + gLogger.error("Failed to update %s write access to Banned" % se) + else: + gLogger.notice("Successfully updated %s write access to Banned" % se) + writeBanned.append(se) + + # Eventually, we will get rid of the notion of InActive, as we always write Banned. + if check and 'CheckAccess' in seOptions: + + if not seOptions['CheckAccess'] in ['Active', 'Degraded', 'Probing']: + gLogger.notice('Check option for %s is %s, instead of %s' % + (se, seOptions['CheckAccess'], ['Active', 'Degraded', 'Probing'])) + gLogger.notice('Try specifying the command switches') else: - gLogger.notice("Successfully updated %s check access to Banned" % se) - checkBanned.append(se) - - # Eventually, we will get rid of the notion of InActive, as we always write Banned. - if remove and 'RemoveAccess' in seOptions: - - if not seOptions['RemoveAccess'] in ['Active', 'Degraded', 'Probing']: - gLogger.notice('Remove option for %s is %s, instead of %s' % - (se, seOptions['RemoveAccess'], ['Active', 'Degraded', 'Probing'])) - gLogger.notice('Try specifying the command switches') - else: - - resC = resourceStatus.setElementStatus(se, 'StorageElement', 'RemoveAccess', 'Banned', reason, userName) - # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) - if not resC['OK']: - gLogger.error("Failed to update %s remove access to Banned" % se) + + resC = resourceStatus.setElementStatus(se, 'StorageElement', 'CheckAccess', 'Banned', reason, userName) + # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) + if not resC['OK']: + gLogger.error("Failed to update %s check access to Banned" % se) + else: + gLogger.notice("Successfully updated %s check access to Banned" % se) + checkBanned.append(se) + + # Eventually, we will get rid of the notion of InActive, as we always write Banned. + if remove and 'RemoveAccess' in seOptions: + + if not seOptions['RemoveAccess'] in ['Active', 'Degraded', 'Probing']: + gLogger.notice('Remove option for %s is %s, instead of %s' % + (se, seOptions['RemoveAccess'], ['Active', 'Degraded', 'Probing'])) + gLogger.notice('Try specifying the command switches') else: - gLogger.notice("Successfully updated %s remove access to Banned" % se) - removeBanned.append(se) - if not(resR['OK'] or resW['OK'] or resC['OK']): - DIRAC.exit(-1) + resC = resourceStatus.setElementStatus(se, 'StorageElement', 'RemoveAccess', 'Banned', reason, userName) + # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) + if not resC['OK']: + gLogger.error("Failed to update %s remove access to Banned" % se) + else: + gLogger.notice("Successfully updated %s remove access to Banned" % se) + removeBanned.append(se) -if not (writeBanned or readBanned or checkBanned or removeBanned): - gLogger.notice("No storage elements were banned") - DIRAC.exit(-1) + if not(resR['OK'] or resW['OK'] or resC['OK']): + DIRAC.exit(-1) -if mute: - gLogger.notice('Email is muted by script switch') - DIRAC.exit(0) + if not (writeBanned or readBanned or checkBanned or removeBanned): + gLogger.notice("No storage elements were banned") + DIRAC.exit(-1) -subject = '%s storage elements banned for use' % len(writeBanned + readBanned + checkBanned + removeBanned) -addressPath = 'EMail/Production' -address = Operations().getValue(addressPath, '') - -body = '' -if read: - body = "%s\n\nThe following storage elements were banned for reading:" % body - for se in readBanned: - body = "%s\n%s" % (body, se) -if write: - body = "%s\n\nThe following storage elements were banned for writing:" % body - for se in writeBanned: - body = "%s\n%s" % (body, se) -if check: - body = "%s\n\nThe following storage elements were banned for check access:" % body - for se in checkBanned: - body = "%s\n%s" % (body, se) -if remove: - body = "%s\n\nThe following storage elements were banned for remove access:" % body - for se in removeBanned: - body = "%s\n%s" % (body, se) - -if not address: - gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) + if mute: + gLogger.notice('Email is muted by script switch') + DIRAC.exit(0) + + subject = '%s storage elements banned for use' % len(writeBanned + readBanned + checkBanned + removeBanned) + addressPath = 'EMail/Production' + address = Operations().getValue(addressPath, '') + + body = '' + if read: + body = "%s\n\nThe following storage elements were banned for reading:" % body + for se in readBanned: + body = "%s\n%s" % (body, se) + if write: + body = "%s\n\nThe following storage elements were banned for writing:" % body + for se in writeBanned: + body = "%s\n%s" % (body, se) + if check: + body = "%s\n\nThe following storage elements were banned for check access:" % body + for se in checkBanned: + body = "%s\n%s" % (body, se) + if remove: + body = "%s\n\nThe following storage elements were banned for remove access:" % body + for se in removeBanned: + body = "%s\n%s" % (body, se) + + if not address: + gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) + DIRAC.exit(0) + + res = diracAdmin.sendMail(address, subject, body) + gLogger.notice('Notifying %s' % address) + if res['OK']: + gLogger.notice(res['Value']) + else: + gLogger.notice(res['Message']) DIRAC.exit(0) -res = diracAdmin.sendMail(address, subject, body) -gLogger.notice('Notifying %s' % address) -if res['OK']: - gLogger.notice(res['Value']) -else: - gLogger.notice(res['Message']) -DIRAC.exit(0) + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py index cb7977e868b..e1ef550cda9 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py @@ -8,34 +8,42 @@ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage(""" -Show storage quotas for specified users or for all registered users if nobody is specified - -Usage: - %s [user1 ...] -""" % Script.scriptName) - -Script.parseCommandLine() -users = Script.getPositionalArgs() - -from DIRAC import gLogger, gConfig -from DIRAC.Core.Security.ProxyInfo import getProxyInfo - -if not users: - res = gConfig.getSections('/Registry/Users') - if not res['OK']: - gLogger.error("Failed to retrieve user list from CS", res['Message']) - DIRAC.exit(2) - users = res['Value'] - -gLogger.notice("-" * 30) -gLogger.notice("%s|%s" % ('Username'.ljust(15), 'Quota (GB)'.rjust(15))) -gLogger.notice("-" * 30) -for user in sorted(users): - quota = gConfig.getValue('/Registry/Users/%s/Quota' % user, 0) - if not quota: - quota = gConfig.getValue('/Registry/DefaultStorageQuota') - gLogger.notice("%s|%s" % (user.ljust(15), str(quota).rjust(15))) -gLogger.notice("-" * 30) -DIRAC.exit(0) + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Show storage quotas for specified users or for all registered users if nobody is specified + + Usage: + %s [user1 ...] + """ % Script.scriptName) + + Script.parseCommandLine() + users = Script.getPositionalArgs() + + from DIRAC import gLogger, gConfig + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + + if not users: + res = gConfig.getSections('/Registry/Users') + if not res['OK']: + gLogger.error("Failed to retrieve user list from CS", res['Message']) + DIRAC.exit(2) + users = res['Value'] + + gLogger.notice("-" * 30) + gLogger.notice("%s|%s" % ('Username'.ljust(15), 'Quota (GB)'.rjust(15))) + gLogger.notice("-" * 30) + for user in sorted(users): + quota = gConfig.getValue('/Registry/Users/%s/Quota' % user, 0) + if not quota: + quota = gConfig.getValue('/Registry/DefaultStorageQuota') + gLogger.notice("%s|%s" % (user.ljust(15), str(quota).rjust(15))) + gLogger.notice("-" * 30) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py index cb4c6fedd57..097596f2ece 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py @@ -14,24 +14,8 @@ from DIRAC.Core.Base import Script from DIRAC import S_OK import os -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - '\nUsage:', - ' %s [option|cfgfile] ... LFN Path SE [GUID]' % Script.scriptName, - '\nArguments:', - ' LFN: Logical File Name', - ' Path: Local path to the file', - ' SE: DIRAC Storage Element', - ' GUID: GUID to use in the registration (optional)', - '', - '**OR**', - '', - 'Usage:', - ' %s [option|cfgfile] ... LocalFile' % Script.scriptName, - '\nArguments:', - ' LocalFile: Path to local file containing all the above, i.e.:', - ' lfn1 localfile1 SE [GUID1]', - ' lfn2 localfile2 SE [GUID2]']) - ) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + overwrite = False @@ -41,13 +25,6 @@ def setOverwrite(arg): return S_OK() -Script.registerSwitch("f", "force", "Force overwrite of existing file", setOverwrite) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 1 or len(args) > 4: - Script.showHelp(exitCode=1) - - def getDict(item_list): """ From the input list, populate the dictionary @@ -63,46 +40,79 @@ def getDict(item_list): return lfn_dict -from DIRAC.DataManagementSystem.Client.DataManager import DataManager -from DIRAC import gLogger -import DIRAC -exitCode = 0 - -lfns = [] -if len(args) == 1: - inputFileName = args[0] - if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - for line in inputFile: - line = line.rstrip() - items = line.split() - items[0] = items[0].replace('LFN:', '').replace('lfn:', '') - lfns.append(getDict(items)) - inputFile.close() - else: - gLogger.error("Error: LFN list '%s' missing." % inputFileName) - exitCode = 4 -else: - lfns.append(getDict(args)) - -dm = DataManager() -for lfn in lfns: - if not os.path.exists(lfn['localfile']): - gLogger.error("File %s must exist locally" % lfn['localfile']) - exitCode = 1 - continue - if not os.path.isfile(lfn['localfile']): - gLogger.error("%s is not a file" % lfn['localfile']) - exitCode = 2 - continue - - gLogger.notice("\nUploading %s" % lfn['lfn']) - res = dm.putAndRegister(lfn['lfn'], lfn['localfile'], lfn['SE'], lfn['guid'], overwrite=overwrite) - if not res['OK']: - exitCode = 3 - gLogger.error('Error: failed to upload %s to %s' % (lfn['lfn'], lfn['SE'])) - continue +@DIRACScript() +def main(): + global overwrite + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + '\nUsage:', + ' %s [option|cfgfile] ... LFN Path SE [GUID]' % Script.scriptName, + '\nArguments:', + ' LFN: Logical File Name', + ' Path: Local path to the file', + ' SE: DIRAC Storage Element', + ' GUID: GUID to use in the registration (optional)', + '', + '**OR**', + '', + 'Usage:', + ' %s [option|cfgfile] ... LocalFile' % Script.scriptName, + '\nArguments:', + ' LocalFile: Path to local file containing all the above, i.e.:', + ' lfn1 localfile1 SE [GUID1]', + ' lfn2 localfile2 SE [GUID2]' + ])) + + Script.registerSwitch("f", "force", "Force overwrite of existing file", setOverwrite) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + if len(args) < 1 or len(args) > 4: + Script.showHelp(exitCode=1) + + from DIRAC.DataManagementSystem.Client.DataManager import DataManager + from DIRAC import gLogger + import DIRAC + exitCode = 0 + + lfns = [] + if len(args) == 1: + inputFileName = args[0] + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + for line in inputFile: + line = line.rstrip() + items = line.split() + items[0] = items[0].replace('LFN:', '').replace('lfn:', '') + lfns.append(getDict(items)) + inputFile.close() + else: + gLogger.error("Error: LFN list '%s' missing." % inputFileName) + exitCode = 4 else: - gLogger.notice('Successfully uploaded file to %s' % lfn['SE']) + lfns.append(getDict(args)) + + dm = DataManager() + for lfn in lfns: + if not os.path.exists(lfn['localfile']): + gLogger.error("File %s must exist locally" % lfn['localfile']) + exitCode = 1 + continue + if not os.path.isfile(lfn['localfile']): + gLogger.error("%s is not a file" % lfn['localfile']) + exitCode = 2 + continue + + gLogger.notice("\nUploading %s" % lfn['lfn']) + res = dm.putAndRegister(lfn['lfn'], lfn['localfile'], lfn['SE'], lfn['guid'], overwrite=overwrite) + if not res['OK']: + exitCode = 3 + gLogger.error('Error: failed to upload %s to %s' % (lfn['lfn'], lfn['SE'])) + continue + else: + gLogger.notice('Successfully uploaded file to %s' % lfn['SE']) + + DIRAC.exit(exitCode) + -DIRAC.exit(exitCode) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_catalog_metadata.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_catalog_metadata.py index 7fc6a429271..83305171614 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_catalog_metadata.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_catalog_metadata.py @@ -7,67 +7,72 @@ from DIRAC import exit as DIRACExit from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage(""" -Get metadata for the given file specified by its Logical File Name or for a list of files -contained in the specifed file -Usage: - %s [Catalog] -""" % Script.scriptName) +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Get metadata for the given file specified by its Logical File Name or for a list of files + contained in the specifed file -Script.parseCommandLine() + Usage: + %s [Catalog] + """ % Script.scriptName) -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + Script.parseCommandLine() -import os -args = Script.getPositionalArgs() + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -if not len(args) >= 1: - Script.showHelp(exitCode=1) -else: - inputFileName = args[0] - catalogs = [] - if len(args) == 2: - catalogs = [args[1]] + import os + args = Script.getPositionalArgs() + if not len(args) >= 1: + Script.showHelp(exitCode=1) + else: + inputFileName = args[0] + catalogs = [] + if len(args) == 2: + catalogs = [args[1]] -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - lfns = string.splitlines() - inputFile.close() -else: - lfns = [inputFileName] + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + lfns = string.splitlines() + inputFile.close() + else: + lfns = [inputFileName] -res = FileCatalog(catalogs=catalogs).getFileMetadata(lfns) -if not res['OK']: - print("ERROR:", res['Message']) - DIRACExit(-1) + res = FileCatalog(catalogs=catalogs).getFileMetadata(lfns) + if not res['OK']: + print("ERROR:", res['Message']) + DIRACExit(-1) -print( - '%s %s %s %s %s' % - ('FileName'.ljust(100), - 'Size'.ljust(10), - 'GUID'.ljust(40), - 'Status'.ljust(8), - 'Checksum'.ljust(10))) -for lfn in sorted(res['Value']['Successful'].keys()): - metadata = res['Value']['Successful'][lfn] - checksum = '' - if 'Checksum' in metadata: - checksum = str(metadata['Checksum']) - size = '' - if 'Size' in metadata: - size = str(metadata['Size']) - guid = '' - if 'GUID' in metadata: - guid = str(metadata['GUID']) - status = '' - if 'Status' in metadata: - status = str(metadata['Status']) - print('%s %s %s %s %s' % (lfn.ljust(100), size.ljust(10), guid.ljust(40), status.ljust(8), checksum.ljust(10))) + print('FileName'.ljust(100), + 'Size'.ljust(10), + 'GUID'.ljust(40), + 'Status'.ljust(8), + 'Checksum'.ljust(10)) + for lfn in sorted(res['Value']['Successful'].keys()): + metadata = res['Value']['Successful'][lfn] + checksum = '' + if 'Checksum' in metadata: + checksum = str(metadata['Checksum']) + size = '' + if 'Size' in metadata: + size = str(metadata['Size']) + guid = '' + if 'GUID' in metadata: + guid = str(metadata['GUID']) + status = '' + if 'Status' in metadata: + status = str(metadata['Status']) + print('%s %s %s %s %s' % (lfn.ljust(100), size.ljust(10), guid.ljust(40), status.ljust(8), checksum.ljust(10))) -for lfn in sorted(res['Value']['Failed'].keys()): - message = res['Value']['Failed'][lfn] - print(lfn, message) + for lfn in sorted(res['Value']['Failed'].keys()): + message = res['Value']['Failed'][lfn] + print(lfn, message) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py index 8cc76bcfa25..19365d6a50f 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py @@ -9,52 +9,60 @@ from DIRAC import exit as DIRACExit from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage(""" -Change status of replica of a given file or a list of files at a given Storage Element - -Usage: - %s -""" % Script.scriptName) - -Script.parseCommandLine() - -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -catalog = FileCatalog() -import os -args = Script.getPositionalArgs() -if not len(args) == 3: - Script.showHelp(exitCode=1) -else: - inputFileName = args[0] - se = args[1] - newStatus = args[2] - -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - lfns = string.splitlines() - inputFile.close() -else: - lfns = [inputFileName] - -res = catalog.getReplicas(lfns, True) -if not res['OK']: - print(res['Message']) - DIRACExit(-1) -replicas = res['Value']['Successful'] - -lfnDict = {} -for lfn in lfns: - lfnDict[lfn] = {} - lfnDict[lfn]['SE'] = se - lfnDict[lfn]['Status'] = newStatus - lfnDict[lfn]['PFN'] = replicas[lfn][se] - -res = catalog.setReplicaStatus(lfnDict) -if not res['OK']: - print("ERROR:", res['Message']) -if res['Value']['Failed']: - print("Failed to update %d replica status" % len(res['Value']['Failed'])) -if res['Value']['Successful']: - print("Successfully updated %d replica status" % len(res['Value']['Successful'])) + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Change status of replica of a given file or a list of files at a given Storage Element + + Usage: + %s + """ % Script.scriptName) + + Script.parseCommandLine() + + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + catalog = FileCatalog() + import os + args = Script.getPositionalArgs() + if not len(args) == 3: + Script.showHelp(exitCode=1) + else: + inputFileName = args[0] + se = args[1] + newStatus = args[2] + + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + lfns = string.splitlines() + inputFile.close() + else: + lfns = [inputFileName] + + res = catalog.getReplicas(lfns, True) + if not res['OK']: + print(res['Message']) + DIRACExit(-1) + replicas = res['Value']['Successful'] + + lfnDict = {} + for lfn in lfns: + lfnDict[lfn] = {} + lfnDict[lfn]['SE'] = se + lfnDict[lfn]['Status'] = newStatus + lfnDict[lfn]['PFN'] = replicas[lfn][se] + + res = catalog.setReplicaStatus(lfnDict) + if not res['OK']: + print("ERROR:", res['Message']) + if res['Value']['Failed']: + print("Failed to update %d replica status" % len(res['Value']['Failed'])) + if res['Value']['Successful']: + print("Successfully updated %d replica status" % len(res['Value']['Successful'])) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py index 63e041ff8ea..02e8aeaa21c 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py @@ -9,44 +9,52 @@ from DIRAC import exit as DIRACExit, gLogger from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage(""" -Clean the given directory or a list of directories by removing it and all the -contained files and subdirectories from the physical storage and from the -file catalogs. - -Usage: - %s -""" % Script.scriptName) - -Script.parseCommandLine() - -args = Script.getPositionalArgs() -if len(args) != 1: - Script.showHelp(exitCode=1) - -inputFileName = args[0] - -if os.path.exists(inputFileName): - lfns = [lfn.strip().split()[0] for lfn in sorted(open(inputFileName, 'r').read().splitlines())] -else: - lfns = [inputFileName] - -from DIRAC.DataManagementSystem.Client.DataManager import DataManager -dm = DataManager() -retVal = 0 -for lfn in [lfn for lfn in lfns if lfn]: - gLogger.notice("Cleaning directory %r ... " % lfn) - result = dm.cleanLogicalDirectory(lfn) - if not result['OK']: - gLogger.error('Failed to clean directory', result['Message']) - retVal = -1 + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Clean the given directory or a list of directories by removing it and all the + contained files and subdirectories from the physical storage and from the + file catalogs. + + Usage: + %s + """ % Script.scriptName) + + Script.parseCommandLine() + + args = Script.getPositionalArgs() + if len(args) != 1: + Script.showHelp(exitCode=1) + + inputFileName = args[0] + + if os.path.exists(inputFileName): + lfns = [lfn.strip().split()[0] for lfn in sorted(open(inputFileName, 'r').read().splitlines())] else: - if not result['Value']['Failed']: - gLogger.notice('OK') + lfns = [inputFileName] + + from DIRAC.DataManagementSystem.Client.DataManager import DataManager + dm = DataManager() + retVal = 0 + for lfn in [lfn for lfn in lfns if lfn]: + gLogger.notice("Cleaning directory %r ... " % lfn) + result = dm.cleanLogicalDirectory(lfn) + if not result['OK']: + gLogger.error('Failed to clean directory', result['Message']) + retVal = -1 else: - for folder, message in result['Value']['Failed'].items(): - gLogger.error('Failed to clean folder', "%r: %s" % (folder, message)) - retVal = -1 + if not result['Value']['Failed']: + gLogger.notice('OK') + else: + for folder, message in result['Value']['Failed'].items(): + gLogger.error('Failed to clean folder', "%r: %s" % (folder, message)) + retVal = -1 + + DIRACExit(retVal) + - DIRACExit(retVal) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py index 85f97755344..9f2542db535 100644 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py @@ -42,6 +42,7 @@ from DIRAC.Core.Utilities import DEncode from DIRAC.Core.Utilities.ReturnValues import returnSingleResult from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.FrameworkSystem.private.standardLogging.LogLevels import LogLevels from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.Client.Request import Request @@ -520,7 +521,8 @@ def _replicateSourceFiles(self, request, lfns): request.addOperation(registerSource) -if __name__ == '__main__': +@DIRACScript() +def main(): try: CAR = CreateArchiveRequest() CAR.run() @@ -531,3 +533,7 @@ def _replicateSourceFiles(self, request, lfns): sLog.error('ERROR: Failed to create Archive Request:', str(e)) exit(1) exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py index b25e3d57c63..d160dfe3d8b 100644 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py @@ -17,6 +17,7 @@ import DIRAC from DIRAC import gLogger from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Utilities.List import breakListIntoChunks from DIRAC.Core.Utilities.ReturnValues import returnSingleResult from DIRAC.FrameworkSystem.private.standardLogging.LogLevels import LogLevels @@ -292,7 +293,8 @@ def putOrRunRequests(self): return 1 -if __name__ == '__main__': +@DIRACScript() +def main(): try: CMR = CreateMovingRequest() CMR.run() @@ -303,3 +305,7 @@ def putOrRunRequests(self): sLog.error('ERROR: Failed to create Moving Request:', str(e)) exit(1) exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py index f85b8e8f6f2..76d09d4f5e0 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py @@ -10,100 +10,108 @@ from hashlib import md5 import time from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Utilities.List import breakListIntoChunks -Script.setUsageMessage('\n'.join([__doc__.split('\n')[0], - __doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... SE LFN ...' % Script.scriptName, - 'Arguments:', - ' SE: StorageElement|All', - ' LFN: LFN or file containing a List of LFNs'])) - -Script.parseCommandLine(ignoreErrors=False) - -args = Script.getPositionalArgs() -if len(args) < 2: - Script.showHelp() - -targetSE = args.pop(0) - -lfns = [] -for inputFileName in args: - if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - inputFile.close() - lfns.extend([lfn.strip() for lfn in string.splitlines()]) - else: - lfns.append(inputFileName) - -from DIRAC.Resources.Storage.StorageElement import StorageElement -import DIRAC -# Check is provided SE is OK -if targetSE != 'All': - se = StorageElement(targetSE) - if not se.valid: - print(se.errorReason) - print() + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[0], + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... SE LFN ...' % Script.scriptName, + 'Arguments:', + ' SE: StorageElement|All', + ' LFN: LFN or file containing a List of LFNs'])) + + Script.parseCommandLine(ignoreErrors=False) + + args = Script.getPositionalArgs() + if len(args) < 2: Script.showHelp() -from DIRAC.RequestManagementSystem.Client.Request import Request -from DIRAC.RequestManagementSystem.Client.Operation import Operation -from DIRAC.RequestManagementSystem.Client.File import File -from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient -from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog - -reqClient = ReqClient() -fc = FileCatalog() - -requestOperation = 'RemoveReplica' -if targetSE == 'All': - requestOperation = 'RemoveFile' - -for lfnList in breakListIntoChunks(lfns, 100): - - oRequest = Request() - requestName = "%s_%s" % ( - md5(repr(time.time()).encode()).hexdigest()[:16], - md5(repr(time.time()).encode()).hexdigest()[:16], - ) - oRequest.RequestName = requestName - - oOperation = Operation() - oOperation.Type = requestOperation - oOperation.TargetSE = targetSE - - res = fc.getFileMetadata(lfnList) - if not res['OK']: - print("Can't get file metadata: %s" % res['Message']) - DIRAC.exit(1) - if res['Value']['Failed']: - print("Could not get the file metadata of the following, so skipping them:") - for fFile in res['Value']['Failed']: - print(fFile) - - lfnMetadata = res['Value']['Successful'] - - for lfn in lfnMetadata: - rarFile = File() - rarFile.LFN = lfn - rarFile.Size = lfnMetadata[lfn]['Size'] - rarFile.Checksum = lfnMetadata[lfn]['Checksum'] - rarFile.GUID = lfnMetadata[lfn]['GUID'] - rarFile.ChecksumType = 'ADLER32' - oOperation.addFile(rarFile) - - oRequest.addOperation(oOperation) - - isValid = RequestValidator().validate(oRequest) - if not isValid['OK']: - print("Request is not valid: ", isValid['Message']) - DIRAC.exit(1) - - result = reqClient.putRequest(oRequest) - if result['OK']: - print('Request %d Submitted' % result['Value']) - else: - print('Failed to submit Request: ', result['Message']) + targetSE = args.pop(0) + + lfns = [] + for inputFileName in args: + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + inputFile.close() + lfns.extend([lfn.strip() for lfn in string.splitlines()]) + else: + lfns.append(inputFileName) + + from DIRAC.Resources.Storage.StorageElement import StorageElement + import DIRAC + # Check is provided SE is OK + if targetSE != 'All': + se = StorageElement(targetSE) + if not se.valid: + print(se.errorReason) + print() + Script.showHelp() + + from DIRAC.RequestManagementSystem.Client.Request import Request + from DIRAC.RequestManagementSystem.Client.Operation import Operation + from DIRAC.RequestManagementSystem.Client.File import File + from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient + from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + + reqClient = ReqClient() + fc = FileCatalog() + + requestOperation = 'RemoveReplica' + if targetSE == 'All': + requestOperation = 'RemoveFile' + + for lfnList in breakListIntoChunks(lfns, 100): + + oRequest = Request() + requestName = "%s_%s" % ( + md5(repr(time.time()).encode()).hexdigest()[:16], + md5(repr(time.time()).encode()).hexdigest()[:16], + ) + oRequest.RequestName = requestName + + oOperation = Operation() + oOperation.Type = requestOperation + oOperation.TargetSE = targetSE + + res = fc.getFileMetadata(lfnList) + if not res['OK']: + print("Can't get file metadata: %s" % res['Message']) + DIRAC.exit(1) + if res['Value']['Failed']: + print("Could not get the file metadata of the following, so skipping them:") + for fFile in res['Value']['Failed']: + print(fFile) + + lfnMetadata = res['Value']['Successful'] + + for lfn in lfnMetadata: + rarFile = File() + rarFile.LFN = lfn + rarFile.Size = lfnMetadata[lfn]['Size'] + rarFile.Checksum = lfnMetadata[lfn]['Checksum'] + rarFile.GUID = lfnMetadata[lfn]['GUID'] + rarFile.ChecksumType = 'ADLER32' + oOperation.addFile(rarFile) + + oRequest.addOperation(oOperation) + + isValid = RequestValidator().validate(oRequest) + if not isValid['OK']: + print("Request is not valid: ", isValid['Message']) + DIRAC.exit(1) + + result = reqClient.putRequest(oRequest) + if result['OK']: + print('Request %d Submitted' % result['Value']) + else: + print('Failed to submit Request: ', result['Message']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py index 92c3fca7507..dc380de68b3 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py @@ -12,56 +12,64 @@ import DIRAC from DIRAC import gLogger from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -unit = 'GB' -Script.registerSwitch("u:", "Unit=", " Unit to use [default %s] (MB,GB,TB,PB)" % unit) -Script.setUsageMessage(""" -Get the size of the given file or a list of files +@DIRACScript() +def main(): + unit = 'GB' + Script.registerSwitch("u:", "Unit=", " Unit to use [default %s] (MB,GB,TB,PB)" % unit) -Usage: - %s -""" % Script.scriptName) + Script.setUsageMessage(""" + Get the size of the given file or a list of files -Script.parseCommandLine(ignoreErrors=False) -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "u" or switch[0].lower() == "unit": - unit = switch[1] -scaleDict = {'MB': 1000 * 1000.0, - 'GB': 1000 * 1000 * 1000.0, - 'TB': 1000 * 1000 * 1000 * 1000.0, - 'PB': 1000 * 1000 * 1000 * 1000 * 1000.0} -if unit not in scaleDict.keys(): - gLogger.error("Unit must be one of MB,GB,TB,PB") - DIRAC.exit(2) -scaleFactor = scaleDict[unit] + Usage: + %s + """ % Script.scriptName) -args = Script.getPositionalArgs() -lfns = [] -for inputFileName in args: - if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - inputFile.close() - lfns.extend([lfn.strip() for lfn in string.splitlines()]) - else: - lfns.append(inputFileName) + Script.parseCommandLine(ignoreErrors=False) + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "u" or switch[0].lower() == "unit": + unit = switch[1] + scaleDict = {'MB': 1000 * 1000.0, + 'GB': 1000 * 1000 * 1000.0, + 'TB': 1000 * 1000 * 1000 * 1000.0, + 'PB': 1000 * 1000 * 1000 * 1000 * 1000.0} + if unit not in scaleDict.keys(): + gLogger.error("Unit must be one of MB,GB,TB,PB") + DIRAC.exit(2) + scaleFactor = scaleDict[unit] -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -res = FileCatalog().getFileSize(lfns) -if not res['OK']: - gLogger.error("Failed to get size of data", res['Message']) - DIRAC.exit(-2) -for lfn, reason in res['Value']['Failed'].items(): - gLogger.error("Failed to get size for %s" % lfn, reason) -totalSize = 0 -totalFiles = 0 -for lfn, size in res['Value']['Successful'].items(): - totalFiles += 1 - totalSize += size -gLogger.notice('-' * 30) -gLogger.notice('%s|%s' % ('Files'.ljust(15), ('Size (%s)' % unit).rjust(15))) -gLogger.notice('-' * 30) -gLogger.notice('%s|%s' % (str(totalFiles).ljust(15), str('%.1f' % (totalSize / scaleFactor)).rjust(15))) -gLogger.notice('-' * 30) -DIRAC.exit(0) + args = Script.getPositionalArgs() + lfns = [] + for inputFileName in args: + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + inputFile.close() + lfns.extend([lfn.strip() for lfn in string.splitlines()]) + else: + lfns.append(inputFileName) + + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + res = FileCatalog().getFileSize(lfns) + if not res['OK']: + gLogger.error("Failed to get size of data", res['Message']) + DIRAC.exit(-2) + for lfn, reason in res['Value']['Failed'].items(): + gLogger.error("Failed to get size for %s" % lfn, reason) + totalSize = 0 + totalFiles = 0 + for lfn, size in res['Value']['Successful'].items(): + totalFiles += 1 + totalSize += size + gLogger.notice('-' * 30) + gLogger.notice('%s|%s' % ('Files'.ljust(15), ('Size (%s)' % unit).rjust(15))) + gLogger.notice('-' * 30) + gLogger.notice('%s|%s' % (str(totalFiles).ljust(15), str('%.1f' % (totalSize / scaleFactor)).rjust(15))) + gLogger.notice('-' * 30) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py index 6d1e283f1fe..8ac7c47a760 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py @@ -19,557 +19,540 @@ import os import DIRAC from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s Source Destination' % Script.scriptName, - ' ', - ' e.g.: Download', - ' %s LFN Path' % Script.scriptName, - ' or Upload', - ' %s Path LFN SE' % Script.scriptName, - 'Arguments:', - ' LFN: Logical File Name (Path to directory)', - ' Path: Local path to the file (Path to directory)', - ' SE: DIRAC Storage Element'] - ) - ) - -Script.registerSwitch("D", "sync", "Make target directory identical to source") -Script.registerSwitch("j:", "parallel=", "Multithreaded download and upload") -Script.parseCommandLine(ignoreErrors=False) - -args = Script.getPositionalArgs() -if len(args) < 1 or len(args) > 3: - Script.showHelp() - -sync = False -parallel = 1 -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "s" or switch[0].lower() == "sync": - sync = True - if switch[0].lower() == "j" or switch[0].lower() == "parallel": - parallel = int(switch[1]) - - -from DIRAC import S_OK, S_ERROR -from DIRAC import gConfig, gLogger -from DIRAC.Core.Utilities.List import breakListIntoChunks -from DIRAC.Core.Utilities.ReturnValues import returnSingleResult -from DIRAC.Core.Utilities.File import mkDir -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -from DIRAC.DataManagementSystem.Client.DataManager import DataManager -from DIRAC.Resources.Storage.StorageElement import StorageElement - -__RCSID__ = "$Id$" - - -def getSetOfLocalDirectoriesAndFiles(path): - """Return a set of all directories and subdirectories and a set of - files contained therein for a given local path - """ - - fullPath = os.path.abspath(path) - - if not os.path.isdir(fullPath): - return S_ERROR("The path: " + fullPath + " does not exist!") - - directories = set() - files = set() - - for dirname, dirnames, filenames in os.walk(path): - # add path to all subdirectories first. - for subdirname in dirnames: - fullSubdirname = os.path.join(dirname, subdirname) - fullSubdirname = os.path.abspath(fullSubdirname) - fullSubdirname = fullSubdirname.replace(fullPath, '').lstrip('/') - directories.add(fullSubdirname) - # add path to all filenames. - for filename in filenames: - fullFilename = os.path.join(dirname, filename) - fullFilename = os.path.abspath(fullFilename) - fullFilename = fullFilename.replace(fullPath, '').lstrip('/') - fileSize = os.path.getsize(fullPath + "/" + fullFilename) - if fileSize > 0: - files.add((fullFilename, long(fileSize))) - - tree = {} - tree["Directories"] = directories - tree["Files"] = files - - return S_OK(tree) - - -def getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories, files): - """ - Recursively traverses all the subdirectories of a directory and returns a set of directories and files - """ - result = fc.listDirectory(path) - if result['OK']: - if result['Value']['Successful']: - for entry in result['Value']['Successful'][path]['Files']: - size = result['Value']['Successful'][path]['Files'][entry]['MetaData']['Size'] - files.add((entry, size)) - for entry in result['Value']['Successful'][path]['SubDirs']: - directories.add(entry) - res = getSetOfRemoteSubDirectoriesAndFiles(entry, fc, directories, files) - if not res['OK']: - return S_ERROR('Error: ' + res['Message']) - return S_OK() +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s Source Destination' % Script.scriptName, + ' ', + ' e.g.: Download', + ' %s LFN Path' % Script.scriptName, + ' or Upload', + ' %s Path LFN SE' % Script.scriptName, + 'Arguments:', + ' LFN: Logical File Name (Path to directory)', + ' Path: Local path to the file (Path to directory)', + ' SE: DIRAC Storage Element' + ])) + + Script.registerSwitch("D", "sync", "Make target directory identical to source") + Script.registerSwitch("j:", "parallel=", "Multithreaded download and upload") + Script.parseCommandLine(ignoreErrors=False) + + args = Script.getPositionalArgs() + if len(args) < 1 or len(args) > 3: + Script.showHelp() + + sync = False + parallel = 1 + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "s" or switch[0].lower() == "sync": + sync = True + if switch[0].lower() == "j" or switch[0].lower() == "parallel": + parallel = int(switch[1]) + + from DIRAC import S_OK, S_ERROR + from DIRAC import gConfig, gLogger + from DIRAC.Core.Utilities.List import breakListIntoChunks + from DIRAC.Core.Utilities.ReturnValues import returnSingleResult + from DIRAC.Core.Utilities.File import mkDir + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + from DIRAC.DataManagementSystem.Client.DataManager import DataManager + from DIRAC.Resources.Storage.StorageElement import StorageElement + + __RCSID__ = "$Id$" + + def getSetOfLocalDirectoriesAndFiles(path): + """Return a set of all directories and subdirectories and a set of + files contained therein for a given local path + """ + + fullPath = os.path.abspath(path) + + if not os.path.isdir(fullPath): + return S_ERROR("The path: " + fullPath + " does not exist!") + + directories = set() + files = set() + + for dirname, dirnames, filenames in os.walk(path): + # add path to all subdirectories first. + for subdirname in dirnames: + fullSubdirname = os.path.join(dirname, subdirname) + fullSubdirname = os.path.abspath(fullSubdirname) + fullSubdirname = fullSubdirname.replace(fullPath, '').lstrip('/') + directories.add(fullSubdirname) + # add path to all filenames. + for filename in filenames: + fullFilename = os.path.join(dirname, filename) + fullFilename = os.path.abspath(fullFilename) + fullFilename = fullFilename.replace(fullPath, '').lstrip('/') + fileSize = os.path.getsize(fullPath + "/" + fullFilename) + if fileSize > 0: + files.add((fullFilename, long(fileSize))) + + tree = {} + tree["Directories"] = directories + tree["Files"] = files + + return S_OK(tree) + + def getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories, files): + """ + Recursively traverses all the subdirectories of a directory and returns a set of directories and files + """ + result = fc.listDirectory(path) + if result['OK']: + if result['Value']['Successful']: + for entry in result['Value']['Successful'][path]['Files']: + size = result['Value']['Successful'][path]['Files'][entry]['MetaData']['Size'] + files.add((entry, size)) + for entry in result['Value']['Successful'][path]['SubDirs']: + directories.add(entry) + res = getSetOfRemoteSubDirectoriesAndFiles(entry, fc, directories, files) + if not res['OK']: + return S_ERROR('Error: ' + res['Message']) + return S_OK() + else: + return S_ERROR("Error: %s" % result['Value']) else: - return S_ERROR("Error: %s" % result['Value']) - else: - return S_ERROR("Error:" + result['Message']) - - -def getSetOfRemoteDirectoriesAndFiles(fc, path): - """ - Return a set of all directories and subdirectories and the therein contained files for a given LFN - """ - directories = set() - files = set() + return S_ERROR("Error:" + result['Message']) - res = getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories, files) - if not res['OK']: - return S_ERROR('Could not list remote directory: ' + res['Message']) + def getSetOfRemoteDirectoriesAndFiles(fc, path): + """ + Return a set of all directories and subdirectories and the therein contained files for a given LFN + """ + directories = set() + files = set() - return_directories = set() - return_files = set() + res = getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories, files) + if not res['OK']: + return S_ERROR('Could not list remote directory: ' + res['Message']) - for myfile in files: - return_files.add((myfile[0].replace(path, '').lstrip('/'), myfile[1])) + return_directories = set() + return_files = set() - for mydirectory in directories: - return_directories.add(mydirectory.replace(path, '').lstrip('/')) + for myfile in files: + return_files.add((myfile[0].replace(path, '').lstrip('/'), myfile[1])) - tree = {} - tree["Directories"] = return_directories - tree["Files"] = return_files + for mydirectory in directories: + return_directories.add(mydirectory.replace(path, '').lstrip('/')) - return S_OK(tree) + tree = {} + tree["Directories"] = return_directories + tree["Files"] = return_files + return S_OK(tree) -def isInFileCatalog(fc, path): - """ - Check if the file is in the File Catalog - """ + def isInFileCatalog(fc, path): + """ + Check if the file is in the File Catalog + """ - result = fc.listDirectory(path) - if result['OK']: - if result['Value']['Successful']: - return S_OK() + result = fc.listDirectory(path) + if result['OK']: + if result['Value']['Successful']: + return S_OK() + else: + return S_ERROR() else: return S_ERROR() - else: - return S_ERROR() + def getContentToSync(upload, fc, source_dir, dest_dir): + """ + Return list of files and directories to be create and deleted + """ -def getContentToSync(upload, fc, source_dir, dest_dir): - """ - Return list of files and directories to be create and deleted - """ + if upload: + res = getSetOfRemoteDirectoriesAndFiles(fc, dest_dir) + if not res['OK']: + return S_ERROR(res['Message']) + to_dirs = res['Value']['Directories'] + to_files = res['Value']['Files'] - if upload: - res = getSetOfRemoteDirectoriesAndFiles(fc, dest_dir) - if not res['OK']: - return S_ERROR(res['Message']) - to_dirs = res['Value']['Directories'] - to_files = res['Value']['Files'] + res = getSetOfLocalDirectoriesAndFiles(source_dir) + if not res['OK']: + return S_ERROR(res['Message']) + from_dirs = res['Value']['Directories'] + from_files = res['Value']['Files'] - res = getSetOfLocalDirectoriesAndFiles(source_dir) - if not res['OK']: - return S_ERROR(res['Message']) - from_dirs = res['Value']['Directories'] - from_files = res['Value']['Files'] + else: + res = getSetOfLocalDirectoriesAndFiles(dest_dir) + if not res['OK']: + return S_ERROR(res['Message']) + to_dirs = res['Value']['Directories'] + to_files = res['Value']['Files'] - else: - res = getSetOfLocalDirectoriesAndFiles(dest_dir) - if not res['OK']: - return S_ERROR(res['Message']) - to_dirs = res['Value']['Directories'] - to_files = res['Value']['Files'] + res = getSetOfRemoteDirectoriesAndFiles(fc, source_dir) + if not res['OK']: + return S_ERROR(res['Message']) + from_dirs = res['Value']['Directories'] + from_files = res['Value']['Files'] + + # Create list of directories to delete + dirs_delete = list(to_dirs - from_dirs) + # Sort the list by depth of directory tree + dirs_delete.sort(key=lambda s: -s.count('/')) + # Create list of directories to create + dirs_create = list(from_dirs - to_dirs) + # Sort the list by depth of directory tree + dirs_create.sort(key=lambda s: s.count('/')) + + # Flatten the list of pairs (filename, size) to list of filename + files_delete = [pair[0] for pair in list(to_files - from_files)] + files_create = [pair[0] for pair in list(from_files - to_files)] + + create = {} + create["Directories"] = dirs_create + create["Files"] = files_create + + delete = {} + delete["Directories"] = dirs_delete + delete["Files"] = files_delete + + tree = {} + tree["Create"] = create + tree["Delete"] = delete + + return S_OK(tree) + + def removeRemoteFiles(dm, lfns): + """ + Remove file from the catalog + """ + for lfnList in breakListIntoChunks(lfns, 100): + res = dm.removeFile(lfnList) + if not res['OK']: + return S_ERROR("Failed to remove files:" + lfnList + res['Message']) + else: + return S_OK() - res = getSetOfRemoteDirectoriesAndFiles(fc, source_dir) + def uploadLocalFile(dm, lfn, localfile, storage): + """ + Upload a local file to a storage element + """ + res = dm.putAndRegister(lfn, localfile, storage, None) if not res['OK']: - return S_ERROR(res['Message']) - from_dirs = res['Value']['Directories'] - from_files = res['Value']['Files'] - - # Create list of directories to delete - dirs_delete = list(to_dirs - from_dirs) - # Sort the list by depth of directory tree - dirs_delete.sort(key=lambda s: -s.count('/')) - # Create list of directories to create - dirs_create = list(from_dirs - to_dirs) - # Sort the list by depth of directory tree - dirs_create.sort(key=lambda s: s.count('/')) - - # Flatten the list of pairs (filename, size) to list of filename - files_delete = [pair[0] for pair in list(to_files - from_files)] - files_create = [pair[0] for pair in list(from_files - to_files)] - - create = {} - create["Directories"] = dirs_create - create["Files"] = files_create - - delete = {} - delete["Directories"] = dirs_delete - delete["Files"] = files_delete - - tree = {} - tree["Create"] = create - tree["Delete"] = delete - - return S_OK(tree) - - -def removeRemoteFiles(dm, lfns): - """ - Remove file from the catalog - """ - for lfnList in breakListIntoChunks(lfns, 100): - res = dm.removeFile(lfnList) - if not res['OK']: - return S_ERROR("Failed to remove files:" + lfnList + res['Message']) + return S_ERROR('Error: failed to upload %s to %s' % (lfn, storage)) else: - return S_OK() - - -def uploadLocalFile(dm, lfn, localfile, storage): - """ - Upload a local file to a storage element - """ - res = dm.putAndRegister(lfn, localfile, storage, None) - if not res['OK']: - return S_ERROR('Error: failed to upload %s to %s' % (lfn, storage)) - else: - return S_OK('Successfully uploaded file to %s' % storage) - - -def downloadRemoteFile(dm, lfn, destination): - """ - Download a file from the system - """ - res = dm.getFile(lfn, destination) - if not res['OK']: - return S_ERROR('Error: failed to download %s ' % lfn) - else: - return S_OK('Successfully uploaded file %s' % lfn) - + return S_OK('Successfully uploaded file to %s' % storage) -def removeStorageDirectoryFromSE(directory, storageElement): - """ - Delete directory on selected storage element - """ + def downloadRemoteFile(dm, lfn, destination): + """ + Download a file from the system + """ + res = dm.getFile(lfn, destination) + if not res['OK']: + return S_ERROR('Error: failed to download %s ' % lfn) + else: + return S_OK('Successfully uploaded file %s' % lfn) - se = StorageElement(storageElement, False) - res = returnSingleResult(se.exists(directory)) + def removeStorageDirectoryFromSE(directory, storageElement): + """ + Delete directory on selected storage element + """ - if not res['OK']: - return S_ERROR("Failed to obtain existence of directory" + res['Message']) + se = StorageElement(storageElement, False) + res = returnSingleResult(se.exists(directory)) - exists = res['Value'] - if not exists: - return S_OK("The directory %s does not exist at %s " % (directory, storageElement)) + if not res['OK']: + return S_ERROR("Failed to obtain existence of directory" + res['Message']) - res = returnSingleResult(se.removeDirectory(directory, recursive=True)) - if not res['OK']: - return S_ERROR("Failed to remove storage directory" + res['Message']) + exists = res['Value'] + if not exists: + return S_OK("The directory %s does not exist at %s " % (directory, storageElement)) - return S_OK() + res = returnSingleResult(se.removeDirectory(directory, recursive=True)) + if not res['OK']: + return S_ERROR("Failed to remove storage directory" + res['Message']) + return S_OK() -def removeRemoteDirectory(fc, lfn): - """ - Remove file from the catalog - """ - storageElements = gConfig.getValue('Resources/StorageElementGroups/SE_Cleaning_List', []) + def removeRemoteDirectory(fc, lfn): + """ + Remove file from the catalog + """ + storageElements = gConfig.getValue('Resources/StorageElementGroups/SE_Cleaning_List', []) - for storageElement in sorted(storageElements): - res = removeStorageDirectoryFromSE(lfn, storageElement) + for storageElement in sorted(storageElements): + res = removeStorageDirectoryFromSE(lfn, storageElement) + if not res['OK']: + return S_ERROR("Failed to clean storage directory at all SE:" + res['Message']) + res = returnSingleResult(fc.removeDirectory(lfn, recursive=True)) if not res['OK']: return S_ERROR("Failed to clean storage directory at all SE:" + res['Message']) - res = returnSingleResult(fc.removeDirectory(lfn, recursive=True)) - if not res['OK']: - return S_ERROR("Failed to clean storage directory at all SE:" + res['Message']) - - return S_OK("Successfully removed directory") - - -def createRemoteDirectory(fc, newdir): - """ - Create directory in file catalog - """ - result = fc.createDirectory(newdir) - if result['OK']: - if result['Value']['Successful'] and newdir in result['Value']['Successful']: - return S_OK("Successfully created directory:" + newdir) - elif result['Value']['Failed'] and newdir in result['Value']['Failed']: - return S_ERROR('Failed to create directory: ' + result['Value']['Failed'][newdir]) - else: - return S_ERROR('Failed to create directory:' + result['Message']) - - -def createLocalDirectory(directory): - """ - Create local directory - """ - mkDir(directory) - if not os.path.exists(directory): - return S_ERROR('Directory creation failed') - return S_OK('Created directory successfully') - - -def removeLocalFile(path): - """ - Remove local file - """ - try: - os.remove(path) - except OSError as e: - return S_ERROR('Directory creation failed:' + e.strerror) - - if os.path.isfile(path): - return S_ERROR('File deleting failed') - return S_OK('Removed file successfully') - - -def removeLocaDirectory(path): - """ - Remove local directory - """ - try: - os.rmdir(path) - except OSError as e: - return S_ERROR('Deleting directory failed: ' + e.strerror) - - if os.path.isdir(path): - return S_ERROR('Directory deleting failed') - return S_OK('Removed directory successfully') - - -def doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads): - """ - Wrapper for uploading files - """ - if delete: - lfns = [dest_dir + "/" + filename for filename in result['Value']['Delete']['Files']] - if len(lfns) > 0: - res = removeRemoteFiles(dm, lfns) - if not res['OK']: - gLogger.fatal('Deleting of files: ' + lfns + " -X- [FAILED]" + res['Message']) - DIRAC.exit(1) - else: - gLogger.notice("Deleting " + ', '.join(lfns) + " -> [DONE]") - for directoryname in result['Value']['Delete']['Directories']: - res = removeRemoteDirectory(fc, dest_dir + "/" + directoryname) + return S_OK("Successfully removed directory") + + def createRemoteDirectory(fc, newdir): + """ + Create directory in file catalog + """ + result = fc.createDirectory(newdir) + if result['OK']: + if result['Value']['Successful'] and newdir in result['Value']['Successful']: + return S_OK("Successfully created directory:" + newdir) + elif result['Value']['Failed'] and newdir in result['Value']['Failed']: + return S_ERROR('Failed to create directory: ' + result['Value']['Failed'][newdir]) + else: + return S_ERROR('Failed to create directory:' + result['Message']) + + def createLocalDirectory(directory): + """ + Create local directory + """ + mkDir(directory) + if not os.path.exists(directory): + return S_ERROR('Directory creation failed') + return S_OK('Created directory successfully') + + def removeLocalFile(path): + """ + Remove local file + """ + try: + os.remove(path) + except OSError as e: + return S_ERROR('Directory creation failed:' + e.strerror) + + if os.path.isfile(path): + return S_ERROR('File deleting failed') + return S_OK('Removed file successfully') + + def removeLocaDirectory(path): + """ + Remove local directory + """ + try: + os.rmdir(path) + except OSError as e: + return S_ERROR('Deleting directory failed: ' + e.strerror) + + if os.path.isdir(path): + return S_ERROR('Directory deleting failed') + return S_OK('Removed directory successfully') + + def doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads): + """ + Wrapper for uploading files + """ + if delete: + lfns = [dest_dir + "/" + filename for filename in result['Value']['Delete']['Files']] + if len(lfns) > 0: + res = removeRemoteFiles(dm, lfns) + if not res['OK']: + gLogger.fatal('Deleting of files: ' + lfns + " -X- [FAILED]" + res['Message']) + DIRAC.exit(1) + else: + gLogger.notice("Deleting " + ', '.join(lfns) + " -> [DONE]") + + for directoryname in result['Value']['Delete']['Directories']: + res = removeRemoteDirectory(fc, dest_dir + "/" + directoryname) + if not res['OK']: + gLogger.fatal('Deleting of directory: ' + directoryname + " -X- [FAILED] " + res['Message']) + DIRAC.exit(1) + else: + gLogger.notice("Deleting " + directoryname + " -> [DONE]") + + for directoryname in result['Value']['Create']['Directories']: + res = createRemoteDirectory(fc, dest_dir + "/" + directoryname) if not res['OK']: - gLogger.fatal('Deleting of directory: ' + directoryname + " -X- [FAILED] " + res['Message']) + gLogger.fatal('Creation of directory: ' + directoryname + " -X- [FAILED] " + res['Message']) DIRAC.exit(1) else: - gLogger.notice("Deleting " + directoryname + " -> [DONE]") - - for directoryname in result['Value']['Create']['Directories']: - res = createRemoteDirectory(fc, dest_dir + "/" + directoryname) + gLogger.notice("Creating " + directoryname + " -> [DONE]") + + listOfFiles = result['Value']['Create']['Files'] + # Chech that we do not have to many threads + if nthreads > len(listOfFiles): + nthreads = len(listOfFiles) + + if nthreads == 0: + return S_OK('Upload finished successfully') + + listOfListOfFiles = chunkList(listOfFiles, nthreads) + res = runInParallel( + arguments=[ + dm, + source_dir, + dest_dir, + storage], + listOfLists=listOfListOfFiles, + function=uploadListOfFiles) if not res['OK']: - gLogger.fatal('Creation of directory: ' + directoryname + " -X- [FAILED] " + res['Message']) - DIRAC.exit(1) - else: - gLogger.notice("Creating " + directoryname + " -> [DONE]") - - listOfFiles = result['Value']['Create']['Files'] - # Chech that we do not have to many threads - if nthreads > len(listOfFiles): - nthreads = len(listOfFiles) + return S_ERROR("Upload of files failed") - if nthreads == 0: return S_OK('Upload finished successfully') - listOfListOfFiles = chunkList(listOfFiles, nthreads) - res = runInParallel( - arguments=[ - dm, - source_dir, - dest_dir, - storage], - listOfLists=listOfListOfFiles, - function=uploadListOfFiles) - if not res['OK']: - return S_ERROR("Upload of files failed") - - return S_OK('Upload finished successfully') - - -def uploadListOfFiles(dm, source_dir, dest_dir, storage, listOfFiles, tID): - """ - Wrapper for multithreaded uploading of a list of files - """ - log = gLogger.getSubLogger("[Thread %s] " % tID) - threadLine = "[Thread %s]" % tID - for filename in listOfFiles: - res = uploadLocalFile(dm, dest_dir + "/" + filename, source_dir + "/" + filename, storage) - if not res['OK']: - log.fatal(threadLine + ' Uploading ' + filename + ' -X- [FAILED] ' + res['Message']) - DIRAC.exit(1) - else: - log.notice(threadLine + " Uploading " + filename + " -> [DONE]") - - -def doDownload(dm, result, source_dir, dest_dir, delete, nthreads): - """ - Wrapper for downloading files - """ - if delete: - for filename in result['Value']['Delete']['Files']: - res = removeLocalFile(dest_dir + "/" + filename) + def uploadListOfFiles(dm, source_dir, dest_dir, storage, listOfFiles, tID): + """ + Wrapper for multithreaded uploading of a list of files + """ + log = gLogger.getSubLogger("[Thread %s] " % tID) + threadLine = "[Thread %s]" % tID + for filename in listOfFiles: + res = uploadLocalFile(dm, dest_dir + "/" + filename, source_dir + "/" + filename, storage) if not res['OK']: - gLogger.fatal('Deleting of file: ' + filename + ' -X- [FAILED] ' + res['Message']) + log.fatal(threadLine + ' Uploading ' + filename + ' -X- [FAILED] ' + res['Message']) DIRAC.exit(1) else: - gLogger.notice("Deleting " + filename + " -> [DONE]") + log.notice(threadLine + " Uploading " + filename + " -> [DONE]") + + def doDownload(dm, result, source_dir, dest_dir, delete, nthreads): + """ + Wrapper for downloading files + """ + if delete: + for filename in result['Value']['Delete']['Files']: + res = removeLocalFile(dest_dir + "/" + filename) + if not res['OK']: + gLogger.fatal('Deleting of file: ' + filename + ' -X- [FAILED] ' + res['Message']) + DIRAC.exit(1) + else: + gLogger.notice("Deleting " + filename + " -> [DONE]") + + for directoryname in result['Value']['Delete']['Directories']: + res = removeLocaDirectory(dest_dir + "/" + directoryname) + if not res['OK']: + gLogger.fatal('Deleting of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message']) + DIRAC.exit(1) + else: + gLogger.notice("Deleting " + directoryname + " -> [DONE]") - for directoryname in result['Value']['Delete']['Directories']: - res = removeLocaDirectory(dest_dir + "/" + directoryname) + for directoryname in result['Value']['Create']['Directories']: + res = createLocalDirectory(dest_dir + "/" + directoryname) if not res['OK']: - gLogger.fatal('Deleting of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message']) + gLogger.fatal('Creation of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message']) DIRAC.exit(1) else: - gLogger.notice("Deleting " + directoryname + " -> [DONE]") - - for directoryname in result['Value']['Create']['Directories']: - res = createLocalDirectory(dest_dir + "/" + directoryname) - if not res['OK']: - gLogger.fatal('Creation of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message']) - DIRAC.exit(1) - else: - gLogger.notice("Creating " + directoryname + " -> [DONE]") + gLogger.notice("Creating " + directoryname + " -> [DONE]") - listOfFiles = result['Value']['Create']['Files'] - # Chech that we do not have to many threads - if nthreads > len(listOfFiles): - nthreads = len(listOfFiles) + listOfFiles = result['Value']['Create']['Files'] + # Chech that we do not have to many threads + if nthreads > len(listOfFiles): + nthreads = len(listOfFiles) - if nthreads == 0: - return S_OK('Upload finished successfully') - - listOfListOfFiles = chunkList(listOfFiles, nthreads) - res = runInParallel(arguments=[dm, source_dir, dest_dir], listOfLists=listOfListOfFiles, function=downloadListOfFiles) - - if not res['OK']: - return S_ERROR("Download of files failed") - - return S_OK('Upload finished successfully') - - -def chunkList(alist, nchunks): - """ - Split a list into a list of equaliy sized lists - """ - avg = len(alist) / float(nchunks) - out = [] - last = 0.0 + if nthreads == 0: + return S_OK('Upload finished successfully') - while last < len(alist): - out.append(alist[int(last):int(last + avg)]) - last += avg + listOfListOfFiles = chunkList(listOfFiles, nthreads) + res = runInParallel( + arguments=[dm, source_dir, dest_dir], + listOfLists=listOfListOfFiles, + function=downloadListOfFiles, + ) - return out - - -def downloadListOfFiles(dm, source_dir, dest_dir, listOfFiles, tID): - """ - Wrapper for multithreaded downloading of a list of files - """ - log = gLogger.getSubLogger("[Thread %s] " % tID) - threadLine = "[Thread %s]" % tID - for filename in listOfFiles: - res = downloadRemoteFile(dm, source_dir + "/" + filename, dest_dir + ("/" + filename).rsplit("/", 1)[0]) if not res['OK']: - log.fatal(threadLine + ' Downloading ' + filename + ' -X- [FAILED] ' + res['Message']) - DIRAC.exit(1) - else: - log.notice(threadLine + " Downloading " + filename + " -> [DONE]") - - -def runInParallel(arguments, listOfLists, function): - """ - Helper for execution of uploads and downloads in parallel - """ - from multiprocessing import Process - processes = [] - for tID, alist in enumerate(listOfLists): - argums = arguments + [alist] + [tID] - pro = Process(target=function, args=argums) - pro.start() - processes.append(pro) - for process in processes: - process.join() - - for process in processes: - if process.exitcode == 1: - return S_ERROR() - return S_OK() - + return S_ERROR("Download of files failed") -def syncDestinations(upload, source_dir, dest_dir, storage, delete, nthreads): - """ - Top level wrapper to execute functions - """ + return S_OK('Upload finished successfully') - fc = FileCatalog() - dm = DataManager() + def chunkList(alist, nchunks): + """ + Split a list into a list of equaliy sized lists + """ + avg = len(alist) / float(nchunks) + out = [] + last = 0.0 + + while last < len(alist): + out.append(alist[int(last):int(last + avg)]) + last += avg + + return out + + def downloadListOfFiles(dm, source_dir, dest_dir, listOfFiles, tID): + """ + Wrapper for multithreaded downloading of a list of files + """ + log = gLogger.getSubLogger("[Thread %s] " % tID) + threadLine = "[Thread %s]" % tID + for filename in listOfFiles: + res = downloadRemoteFile(dm, source_dir + "/" + filename, dest_dir + ("/" + filename).rsplit("/", 1)[0]) + if not res['OK']: + log.fatal(threadLine + ' Downloading ' + filename + ' -X- [FAILED] ' + res['Message']) + DIRAC.exit(1) + else: + log.notice(threadLine + " Downloading " + filename + " -> [DONE]") + + def runInParallel(arguments, listOfLists, function): + """ + Helper for execution of uploads and downloads in parallel + """ + from multiprocessing import Process + processes = [] + for tID, alist in enumerate(listOfLists): + argums = arguments + [alist] + [tID] + pro = Process(target=function, args=argums) + pro.start() + processes.append(pro) + for process in processes: + process.join() + + for process in processes: + if process.exitcode == 1: + return S_ERROR() + return S_OK() + + def syncDestinations(upload, source_dir, dest_dir, storage, delete, nthreads): + """ + Top level wrapper to execute functions + """ + + fc = FileCatalog() + dm = DataManager() + + result = getContentToSync(upload, fc, source_dir, dest_dir) + if not result['OK']: + return S_ERROR(result['Message']) + + if upload: + res = doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads) + if not res['OK']: + return S_ERROR('Upload failed: ' + res['Message']) + else: + res = doDownload(dm, result, source_dir, dest_dir, delete, nthreads) + if not res['OK']: + return S_ERROR('Download failed: ' + res['Message']) + + return S_OK('Mirroring successfully finished') + + def run(parameters, delete, nthreads): + """ + The main user interface + """ + + source_dir = parameters[0] + dest_dir = parameters[1] + upload = False + storage = None + + if len(parameters) == 3: + storage = parameters[2] + source_dir = os.path.abspath(source_dir) + dest_dir = dest_dir.rstrip('/') + upload = True + if not os.path.isdir(source_dir): + gLogger.fatal("Source directory does not exist") + DIRAC.exit(1) - result = getContentToSync(upload, fc, source_dir, dest_dir) - if not result['OK']: - return S_ERROR(result['Message']) + if len(parameters) == 2: + dest_dir = os.path.abspath(dest_dir) + source_dir = source_dir.rstrip('/') + if not os.path.isdir(dest_dir): + gLogger.fatal("Destination directory does not exist") + DIRAC.exit(1) - if upload: - res = doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads) - if not res['OK']: - return S_ERROR('Upload failed: ' + res['Message']) - else: - res = doDownload(dm, result, source_dir, dest_dir, delete, nthreads) + res = syncDestinations(upload, source_dir, dest_dir, storage, delete, nthreads) if not res['OK']: - return S_ERROR('Download failed: ' + res['Message']) - - return S_OK('Mirroring successfully finished') - - -def run(parameters, delete, nthreads): - """ - The main user interface - """ - - source_dir = parameters[0] - dest_dir = parameters[1] - upload = False - storage = None - - if len(parameters) == 3: - storage = parameters[2] - source_dir = os.path.abspath(source_dir) - dest_dir = dest_dir.rstrip('/') - upload = True - if not os.path.isdir(source_dir): - gLogger.fatal("Source directory does not exist") - DIRAC.exit(1) - - if len(parameters) == 2: - dest_dir = os.path.abspath(dest_dir) - source_dir = source_dir.rstrip('/') - if not os.path.isdir(dest_dir): - gLogger.fatal("Destination directory does not exist") - DIRAC.exit(1) - - res = syncDestinations(upload, source_dir, dest_dir, storage, delete, nthreads) - if not res['OK']: - return S_ERROR(res['Message']) - - return S_OK("Successfully mirrored " + source_dir + " into " + dest_dir) + return S_ERROR(res['Message']) + return S_OK("Successfully mirrored " + source_dir + " into " + dest_dir) -if __name__ == "__main__": returnValue = run(args, sync, parallel) if not returnValue['OK']: gLogger.fatal(returnValue['Message']) @@ -577,3 +560,7 @@ def run(parameters, delete, nthreads): else: gLogger.notice(returnValue['Value']) DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py index cd5bd7368b3..6152a3d1357 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py @@ -6,51 +6,60 @@ import sys from DIRAC.Core.Base import Script -Script.setUsageMessage(""" -Launch the File Catalog shell - -Usage: - %s [option] -""" % Script.scriptName) - -fcType = 'FileCatalog' -Script.registerSwitch("f:", "file-catalog=", " Catalog client type to use (default %s)" % fcType) -Script.parseCommandLine(ignoreErrors=False) - -from DIRAC import gConfig, exit as dexit -from DIRAC.Resources.Catalog.FileCatalogFactory import FileCatalogFactory - -__RCSID__ = "$Id$" - -fcType = gConfig.getValue("/LocalSite/FileCatalog", "") - -res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True) -if not res['OK']: - dexit(1) -fcList = res['Value'] -if not fcType: - if res['OK']: - fcType = res['Value'][0] - -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "f" or switch[0].lower() == "file-catalog": - fcType = switch[1] - -if not fcType: - print("No file catalog given and defaults could not be obtained") - sys.exit(1) - -from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI - -result = FileCatalogFactory().createCatalog(fcType) -if not result['OK']: - print(result['Message']) - if fcList: - print("Possible choices are:") - for fc in fcList: - print(' ' * 5, fc) - sys.exit(1) -print("Starting %s client" % fcType) -catalog = result['Value'] -cli = FileCatalogClientCLI(catalog) -cli.cmdloop() +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Launch the File Catalog shell + + Usage: + %s [option] + """ % Script.scriptName) + + fcType = 'FileCatalog' + Script.registerSwitch("f:", "file-catalog=", " Catalog client type to use (default %s)" % fcType) + Script.parseCommandLine(ignoreErrors=False) + + from DIRAC import gConfig, exit as dexit + from DIRAC.Resources.Catalog.FileCatalogFactory import FileCatalogFactory + + __RCSID__ = "$Id$" + + fcType = gConfig.getValue("/LocalSite/FileCatalog", "") + + res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True) + if not res['OK']: + dexit(1) + fcList = res['Value'] + if not fcType: + if res['OK']: + fcType = res['Value'][0] + + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "f" or switch[0].lower() == "file-catalog": + fcType = switch[1] + + if not fcType: + print("No file catalog given and defaults could not be obtained") + sys.exit(1) + + from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI + + result = FileCatalogFactory().createCatalog(fcType) + if not result['OK']: + print(result['Message']) + if fcList: + print("Possible choices are:") + for fc in fcList: + print(' ' * 5, fc) + sys.exit(1) + print("Starting %s client" % fcType) + catalog = result['Value'] + cli = FileCatalogClientCLI(catalog) + cli.cmdloop() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py index 2ae25a1169c..61cb8f5421e 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py @@ -8,9 +8,11 @@ from __future__ import absolute_import from __future__ import division import DIRAC +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if __name__ == "__main__": +@DIRACScript() +def main(): from DIRAC.Core.Base import Script Script.registerSwitch('', 'Path=', ' Path to search for') @@ -78,3 +80,7 @@ lfnList = sorted(result['Value']) gLogger.notice('\n'.join(lfn for lfn in lfnList)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py index 9fa7de43fe2..2268aba5375 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py @@ -11,14 +11,7 @@ from hashlib import md5 import time from DIRAC.Core.Base import Script -Script.setUsageMessage('\n'.join([__doc__.split('\n')[0], - __doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... sourceSE LFN targetSE1 [targetSE2...]' % Script.scriptName, - 'Arguments:', - ' sourceSE: source SE', - ' targetSE: target SE', - ' LFN: LFN or file containing a List of LFNs'])) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript def getLFNList(arg): @@ -31,8 +24,18 @@ def getLFNList(arg): return list(set(lfnList)) -# # execution -if __name__ == "__main__": +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[0], + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... sourceSE LFN targetSE1 [targetSE2...]' % Script.scriptName, + 'Arguments:', + ' sourceSE: source SE', + ' targetSE: target SE', + ' LFN: LFN or file containing a List of LFNs' + ])) from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() @@ -126,3 +129,7 @@ def getLFNList(arg): if multiRequests: gLogger.always("%d requests have been submitted" % (count)) DIRAC.exit(error) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py index 46337e08a73..4b87c7d73e7 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py @@ -46,16 +46,19 @@ from collections import defaultdict from DIRAC.Core.Base import Script -Script.registerSwitch('', 'FromSE=', 'SE1[,SE2,...]') -Script.registerSwitch('', 'TargetSE=', 'SE1[,SE2,...]') -Script.registerSwitch('', 'OutputFile=', 'CSV output file (default /tmp/protocol-matrix.csv)') -Script.registerSwitch('', 'Bidirection', 'If FromSE or TargetSE are specified, make a square matrix ') -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s [option|cfgfile] % Script.scriptName'])) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if __name__ == '__main__': +@DIRACScript() +def main(): + Script.registerSwitch('', 'FromSE=', 'SE1[,SE2,...]') + Script.registerSwitch('', 'TargetSE=', 'SE1[,SE2,...]') + Script.registerSwitch('', 'OutputFile=', 'CSV output file (default /tmp/protocol-matrix.csv)') + Script.registerSwitch('', 'Bidirection', 'If FromSE or TargetSE are specified, make a square matrix ') + Script.setUsageMessage('\n'.join([__doc__, + 'Usage:', + ' %s [option|cfgfile] % Script.scriptName'])) + from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() from DIRAC import gConfig, gLogger @@ -177,3 +180,7 @@ srcRow.append(tpMatrix[src].get(dst, 'NA')) csvWriter.writerow(srcRow) gLogger.notice('Wrote Matrix to', outputFile) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py index 163343a7810..a3f5d8a8e50 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py @@ -12,17 +12,21 @@ import os from DIRAC.Core.Base import Script -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s [option|cfgfile] requestName LFN localFile targetSE' % Script.scriptName, - 'Arguments:', - ' requestName: a request name', - ' LFN: logical file name' - ' localFile: local file you want to put', - ' targetSE: target SE'])) - -# # execution -if __name__ == "__main__": +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__, + 'Usage:', + ' %s [option|cfgfile] requestName LFN localFile targetSE' % Script.scriptName, + 'Arguments:', + ' requestName: a request name', + ' LFN: logical file name' + ' localFile: local file you want to put', + ' targetSE: target SE' + ])) from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() @@ -91,3 +95,7 @@ gLogger.always("Request '%s' has been put to ReqDB for execution." % requestName) gLogger.always("You can monitor its status using command: 'dirac-rms-request %s'" % requestName) DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py index 97454be243b..b0f6c99358d 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py @@ -9,56 +9,65 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import exit as dexit from DIRAC import gLogger -Script.setUsageMessage(""" -Remove the given file or a list of files from the File Catalog - -Usage: - %s -""" % Script.scriptName) - -Script.parseCommandLine() - -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -allowUsers = Operations().getValue("DataManagement/AllowUserReplicaManagement", False) - -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -res = getProxyInfo() -if not res['OK']: - gLogger.fatal("Can't get proxy info", res['Message']) - dexit(1) -properties = res['Value'].get('groupProperties', []) - -if not allowUsers: - if 'FileCatalogManagement' not in properties: - gLogger.error("You need to use a proxy from a group with FileCatalogManagement") - dexit(5) - -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -fc = FileCatalog() -import os - -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) -else: - inputFileName = args[0] - -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - lfns = [lfn.strip() for lfn in string.splitlines()] - inputFile.close() -else: - lfns = [inputFileName] - -res = fc.removeFile(lfns) -if not res['OK']: - print("Error:", res['Message']) - dexit(1) -for lfn in sorted(res['Value']['Failed'].keys()): - message = res['Value']['Failed'][lfn] - print('Error: failed to remove %s: %s' % (lfn, message)) -print('Successfully removed %d catalog files.' % (len(res['Value']['Successful']))) + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Remove the given file or a list of files from the File Catalog + + Usage: + %s + """ % Script.scriptName) + + Script.parseCommandLine() + + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + allowUsers = Operations().getValue("DataManagement/AllowUserReplicaManagement", False) + + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + res = getProxyInfo() + if not res['OK']: + gLogger.fatal("Can't get proxy info", res['Message']) + dexit(1) + properties = res['Value'].get('groupProperties', []) + + if not allowUsers: + if 'FileCatalogManagement' not in properties: + gLogger.error("You need to use a proxy from a group with FileCatalogManagement") + dexit(5) + + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + fc = FileCatalog() + import os + + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + else: + inputFileName = args[0] + + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + lfns = [lfn.strip() for lfn in string.splitlines()] + inputFile.close() + else: + lfns = [inputFileName] + + res = fc.removeFile(lfns) + if not res['OK']: + print("Error:", res['Message']) + dexit(1) + for lfn in sorted(res['Value']['Failed'].keys()): + message = res['Value']['Failed'][lfn] + print('Error: failed to remove %s: %s' % (lfn, message)) + print('Successfully removed %d catalog files.' % (len(res['Value']['Successful']))) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py index 19b250f9241..89584c11ad7 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py @@ -10,56 +10,64 @@ from DIRAC import exit as dexit from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import gLogger -Script.setUsageMessage(""" -Remove the given file replica or a list of file replicas from the File Catalog -This script should be used with great care as it may leave dark data in the storage! -Use dirac-dms-remove-replicas instead - -Usage: - %s -""" % Script.scriptName) - -Script.parseCommandLine() - -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -allowUsers = Operations().getValue("DataManagement/AllowUserReplicaManagement", False) - -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -res = getProxyInfo() -if not res['OK']: - gLogger.fatal("Can't get proxy info", res['Message']) - dexit(1) -properties = res['Value'].get('groupProperties', []) - -if not allowUsers: - if 'FileCatalogManagement' not in properties: - gLogger.error("You need to use a proxy from a group with FileCatalogManagement") - dexit(5) - -from DIRAC.DataManagementSystem.Client.DataManager import DataManager -dm = DataManager() -args = Script.getPositionalArgs() -if len(args) < 2: - Script.showHelp(exitCode=1) -else: - inputFileName = args[0] - storageElementName = args[1] - -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - lfns = [lfn.strip() for lfn in string.splitlines()] - inputFile.close() -else: - lfns = [inputFileName] - -res = dm.removeReplicaFromCatalog(storageElementName, lfns) -if not res['OK']: - print(res['Message']) - dexit(0) -for lfn in sorted(res['Value']['Failed']): - message = res['Value']['Failed'][lfn] - print('Failed to remove %s replica of %s: %s' % (storageElementName, lfn, message)) -print('Successfully remove %d catalog replicas at %s' % (len(res['Value']['Successful']), storageElementName)) + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Remove the given file replica or a list of file replicas from the File Catalog + This script should be used with great care as it may leave dark data in the storage! + Use dirac-dms-remove-replicas instead + + Usage: + %s + """ % Script.scriptName) + + Script.parseCommandLine() + + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + allowUsers = Operations().getValue("DataManagement/AllowUserReplicaManagement", False) + + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + res = getProxyInfo() + if not res['OK']: + gLogger.fatal("Can't get proxy info", res['Message']) + dexit(1) + properties = res['Value'].get('groupProperties', []) + + if not allowUsers: + if 'FileCatalogManagement' not in properties: + gLogger.error("You need to use a proxy from a group with FileCatalogManagement") + dexit(5) + + from DIRAC.DataManagementSystem.Client.DataManager import DataManager + dm = DataManager() + args = Script.getPositionalArgs() + if len(args) < 2: + Script.showHelp(exitCode=1) + else: + inputFileName = args[0] + storageElementName = args[1] + + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + lfns = [lfn.strip() for lfn in string.splitlines()] + inputFile.close() + else: + lfns = [inputFileName] + + res = dm.removeReplicaFromCatalog(storageElementName, lfns) + if not res['OK']: + print(res['Message']) + dexit(0) + for lfn in sorted(res['Value']['Failed']): + message = res['Value']['Failed'][lfn] + print('Failed to remove %s replica of %s: %s' % (storageElementName, lfn, message)) + print('Successfully remove %d catalog replicas at %s' % (len(res['Value']['Successful']), storageElementName)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py index c0edd1a9aa6..450215a6373 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py @@ -8,52 +8,60 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage(""" -Remove the given file or a list of files from the File Catalog and from the storage - -Usage: - %s -""" % Script.scriptName) - -Script.parseCommandLine() - -import sys -import os -import DIRAC -from DIRAC import gLogger - -args = Script.getPositionalArgs() -lfns = [] -for inputFileName in args: - if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - inputFile.close() - lfns.extend([lfn.strip() for lfn in string.splitlines()]) - else: - lfns.append(inputFileName) - -from DIRAC.Core.Utilities.List import breakListIntoChunks -from DIRAC.DataManagementSystem.Client.DataManager import DataManager -dm = DataManager() - -errorReasons = {} -successfullyRemoved = 0 -for lfnList in breakListIntoChunks(lfns, 100): - res = dm.removeFile(lfnList) - if not res['OK']: - gLogger.error("Failed to remove data", res['Message']) - DIRAC.exit(-2) - for lfn, r in res['Value']['Failed'].items(): - reason = str(r) - if reason not in errorReasons: - errorReasons[reason] = [] - errorReasons[reason].append(lfn) - successfullyRemoved += len(res['Value']['Successful']) - -for reason, lfns in errorReasons.items(): - gLogger.notice("Failed to remove %d files with error: %s" % (len(lfns), reason)) -if successfullyRemoved > 0: - gLogger.notice("Successfully removed %d files" % successfullyRemoved) -DIRAC.exit(0) + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Remove the given file or a list of files from the File Catalog and from the storage + + Usage: + %s + """ % Script.scriptName) + + Script.parseCommandLine() + + import sys + import os + import DIRAC + from DIRAC import gLogger + + args = Script.getPositionalArgs() + lfns = [] + for inputFileName in args: + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + inputFile.close() + lfns.extend([lfn.strip() for lfn in string.splitlines()]) + else: + lfns.append(inputFileName) + + from DIRAC.Core.Utilities.List import breakListIntoChunks + from DIRAC.DataManagementSystem.Client.DataManager import DataManager + dm = DataManager() + + errorReasons = {} + successfullyRemoved = 0 + for lfnList in breakListIntoChunks(lfns, 100): + res = dm.removeFile(lfnList) + if not res['OK']: + gLogger.error("Failed to remove data", res['Message']) + DIRAC.exit(-2) + for lfn, r in res['Value']['Failed'].items(): + reason = str(r) + if reason not in errorReasons: + errorReasons[reason] = [] + errorReasons[reason].append(lfn) + successfullyRemoved += len(res['Value']['Successful']) + + for reason, lfns in errorReasons.items(): + gLogger.notice("Failed to remove %d files with error: %s" % (len(lfns), reason)) + if successfullyRemoved > 0: + gLogger.notice("Successfully removed %d files" % successfullyRemoved) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_replicas.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_replicas.py index 9c9ab589333..028c65409fb 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_replicas.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_replicas.py @@ -7,9 +7,11 @@ from DIRAC import exit as DIRACExit from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if __name__ == "__main__": +@DIRACScript() +def main(): Script.setUsageMessage(""" Remove the given file replica or a list of file replicas from the File Catalog @@ -53,3 +55,7 @@ for lfn in sorted(res['Value']['Failed']): message = res['Value']['Failed'][lfn] print('Error: failed to remove %s replica of %s: %s' % (storageElementName, lfn, message)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py index b8d7db387ea..47744c0b87b 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py @@ -10,47 +10,55 @@ from DIRAC import exit as DIRACExit from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage(""" -Get the given file replica metadata from the File Catalog - -Usage: - %s SE -""" % Script.scriptName) - -Script.parseCommandLine() - -from DIRAC import gLogger -from DIRAC.DataManagementSystem.Client.DataManager import DataManager - -args = Script.getPositionalArgs() -if not len(args) == 2: - Script.showHelp(exitCode=1) -else: - inputFileName = args[0] - storageElement = args[1] - -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - lfns = [lfn.strip() for lfn in string.splitlines()] - inputFile.close() -else: - lfns = [inputFileName] - -res = DataManager().getReplicaMetadata(lfns, storageElement) -if not res['OK']: - print('Error:', res['Message']) - DIRACExit(1) - -print('%s %s %s %s' % ('File'.ljust(100), 'Migrated'.ljust(8), 'Cached'.ljust(8), 'Size (bytes)'.ljust(10))) -for lfn, metadata in res['Value']['Successful'].items(): - print( - '%s %s %s %s' % - (lfn.ljust(100), str( - metadata['Migrated']).ljust(8), str( - metadata.get( - 'Cached', metadata['Accessible'])).ljust(8), str( - metadata['Size']).ljust(10))) -for lfn, reason in res['Value']['Failed'].items(): - print('%s %s' % (lfn.ljust(100), reason.ljust(8))) + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Get the given file replica metadata from the File Catalog + + Usage: + %s SE + """ % Script.scriptName) + + Script.parseCommandLine() + + from DIRAC import gLogger + from DIRAC.DataManagementSystem.Client.DataManager import DataManager + + args = Script.getPositionalArgs() + if not len(args) == 2: + Script.showHelp(exitCode=1) + else: + inputFileName = args[0] + storageElement = args[1] + + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + lfns = [lfn.strip() for lfn in string.splitlines()] + inputFile.close() + else: + lfns = [inputFileName] + + res = DataManager().getReplicaMetadata(lfns, storageElement) + if not res['OK']: + print('Error:', res['Message']) + DIRACExit(1) + + print('%s %s %s %s' % ('File'.ljust(100), 'Migrated'.ljust(8), 'Cached'.ljust(8), 'Size (bytes)'.ljust(10))) + for lfn, metadata in res['Value']['Successful'].items(): + print( + '%s %s %s %s' % + (lfn.ljust(100), str( + metadata['Migrated']).ljust(8), str( + metadata.get( + 'Cached', metadata['Accessible'])).ljust(8), str( + metadata['Size']).ljust(10))) + for lfn, reason in res['Value']['Failed'].items(): + print('%s %s' % (lfn.ljust(100), reason.ljust(8))) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py index 6edd6ca797c..73c15036314 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py @@ -6,22 +6,9 @@ __RCSID__ = "$Id$" import os from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import gLogger import DIRAC -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s [option|cfgfile] requestName LFNs targetSE1 [targetSE2 ...]' % Script.scriptName, - 'Arguments:', - ' requestName: a request name', - ' LFNs: single LFN or file with LFNs', - ' targetSE: target SE'])) - -catalog = None -Script.registerSwitch("C:", "Catalog=", "Catalog to use") -Script.parseCommandLine() -for switch in Script.getUnprocessedSwitches(): - if switch[0] == "C" or switch[0].lower() == "catalog": - catalog = switch[1] def getLFNList(arg): @@ -34,8 +21,24 @@ def getLFNList(arg): return list(set(lfnList)) -# # execution -if __name__ == "__main__": +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__, + 'Usage:', + ' %s [option|cfgfile] requestName LFNs targetSE1 [targetSE2 ...]' % Script.scriptName, + 'Arguments:', + ' requestName: a request name', + ' LFNs: single LFN or file with LFNs', + ' targetSE: target SE', + ])) + + catalog = None + Script.registerSwitch("C:", "Catalog=", "Catalog to use") + Script.parseCommandLine() + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "C" or switch[0].lower() == "catalog": + catalog = switch[1] args = Script.getPositionalArgs() @@ -126,3 +129,7 @@ def getLFNList(arg): gLogger.always("RequestID(s): %s" % " ".join(requestIDs)) gLogger.always("You can monitor requests' status using command: 'dirac-rms-request '") DIRAC.exit(error) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_resolve_guid.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_resolve_guid.py index 9254f4fbb79..8b5efb3693f 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_resolve_guid.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_resolve_guid.py @@ -6,44 +6,52 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage(""" -Returns the LFN matching given GUIDs -Usage: - %s -""" % Script.scriptName) -Script.parseCommandLine() +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Returns the LFN matching given GUIDs + Usage: + %s + """ % Script.scriptName) -import DIRAC -from DIRAC import gLogger + Script.parseCommandLine() -args = Script.getPositionalArgs() -if len(args) != 1: - Script.showHelp() -guids = args[0] + import DIRAC + from DIRAC import gLogger -try: - guids = guids.split(',') -except BaseException: - pass + args = Script.getPositionalArgs() + if len(args) != 1: + Script.showHelp() + guids = args[0] -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + try: + guids = guids.split(',') + except BaseException: + pass -fc = FileCatalog() -res = fc.getLFNForGUID(guids) -if not res['OK']: - gLogger.error("Failed to get the LFNs", res['Message']) - DIRAC.exit(-2) + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -errorGuid = {} -for guid, reason in res['Value']['Failed'].items(): - errorGuid.setdefault(reason, []).append(guid) + fc = FileCatalog() + res = fc.getLFNForGUID(guids) + if not res['OK']: + gLogger.error("Failed to get the LFNs", res['Message']) + DIRAC.exit(-2) -for error, guidList in errorGuid.items(): - gLogger.notice("Error '%s' for guids %s" % (error, guidList)) + errorGuid = {} + for guid, reason in res['Value']['Failed'].items(): + errorGuid.setdefault(reason, []).append(guid) -for guid, lfn in res['Value']['Successful'].items(): - gLogger.notice("%s -> %s" % (guid, lfn)) + for error, guidList in errorGuid.items(): + gLogger.notice("Error '%s' for guids %s" % (error, guidList)) -DIRAC.exit(0) + for guid, lfn in res['Value']['Successful'].items(): + gLogger.notice("%s -> %s" % (guid, lfn)) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py index 9bc69abe39c..fd31066cab4 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py @@ -11,64 +11,72 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - '\nUsage:', - ' %s [option|cfgfile] ... SE Status' % Script.scriptName, - 'Arguments:', - ' LFN: LFN', - ' File: File name containing a list of affected LFNs', - ' SE: Name of Storage Element', - ' Status: New Status for the replica'])) -Script.parseCommandLine(ignoreErrors=False) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + '\nUsage:', + ' %s [option|cfgfile] ... SE Status' % Script.scriptName, + 'Arguments:', + ' LFN: LFN', + ' File: File name containing a list of affected LFNs', + ' SE: Name of Storage Element', + ' Status: New Status for the replica'])) -import DIRAC -from DIRAC import gConfig, gLogger -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -import os + Script.parseCommandLine(ignoreErrors=False) -args = Script.getPositionalArgs() -if not len(args) == 3: - Script.showHelp() + import DIRAC + from DIRAC import gConfig, gLogger + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + import os -inputFileName = args[0] -storageElement = args[1] -status = args[2] + args = Script.getPositionalArgs() + if not len(args) == 3: + Script.showHelp() -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - inputFile.close() - lfns = sorted(string.splitlines()) -else: - lfns = [inputFileName] + inputFileName = args[0] + storageElement = args[1] + status = args[2] -fc = FileCatalog() - -replicaDict = {} -res = fc.getReplicas(lfns, allStatus=True) -if not res['OK']: - gLogger.error("Failed to get catalog replicas.", res['Message']) - DIRAC.exit(-1) -lfnDict = {} -for lfn, error in res['Value']['Failed'].items(): - gLogger.error("Failed to get replicas for file.", "%s:%s" % (lfn, error)) -for lfn, replicas in res['Value']['Successful'].items(): - if storageElement not in replicas.keys(): - gLogger.error("LFN not registered at provided storage element.", "%s %s" % (lfn, storageElement)) + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + inputFile.close() + lfns = sorted(string.splitlines()) else: - lfnDict[lfn] = {'SE': storageElement, 'PFN': replicas[storageElement], 'Status': status} -if not lfnDict: - gLogger.error("No files found at the supplied storage element.") - DIRAC.exit(2) + lfns = [inputFileName] + + fc = FileCatalog() + + replicaDict = {} + res = fc.getReplicas(lfns, allStatus=True) + if not res['OK']: + gLogger.error("Failed to get catalog replicas.", res['Message']) + DIRAC.exit(-1) + lfnDict = {} + for lfn, error in res['Value']['Failed'].items(): + gLogger.error("Failed to get replicas for file.", "%s:%s" % (lfn, error)) + for lfn, replicas in res['Value']['Successful'].items(): + if storageElement not in replicas.keys(): + gLogger.error("LFN not registered at provided storage element.", "%s %s" % (lfn, storageElement)) + else: + lfnDict[lfn] = {'SE': storageElement, 'PFN': replicas[storageElement], 'Status': status} + if not lfnDict: + gLogger.error("No files found at the supplied storage element.") + DIRAC.exit(2) + + res = fc.setReplicaStatus(lfnDict) + if not res['OK']: + gLogger.error("Failed to set catalog replica status.", res['Message']) + DIRAC.exit(-1) + for lfn, error in res['Value']['Failed'].items(): + gLogger.error("Failed to set replica status for file.", "%s:%s" % (lfn, error)) + gLogger.notice("Successfully updated the status of %d files at %s." % + (len(res['Value']['Successful'].keys()), storageElement)) + DIRAC.exit(0) + -res = fc.setReplicaStatus(lfnDict) -if not res['OK']: - gLogger.error("Failed to set catalog replica status.", res['Message']) - DIRAC.exit(-1) -for lfn, error in res['Value']['Failed'].items(): - gLogger.error("Failed to set replica status for file.", "%s:%s" % (lfn, error)) -gLogger.notice("Successfully updated the status of %d files at %s." % - (len(res['Value']['Successful'].keys()), storageElement)) -DIRAC.exit(0) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py index cb91c953f33..5ad4d83debb 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py @@ -6,16 +6,10 @@ from DIRAC import S_OK, exit as DIRACexit from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript __RCSID__ = "$Id$" -Script.setUsageMessage(""" -Get status of the available Storage Elements - -Usage: - %s [] -""" % Script.scriptName) - vo = None @@ -44,62 +38,79 @@ def setNoVO(arg): return S_OK() -Script.registerSwitch("V:", "vo=", "Virtual Organization", setVO) -Script.registerSwitch("a", "all", "All Virtual Organizations flag", setAllVO) -Script.registerSwitch("n", "noVO", "No Virtual Organizations assigned flag", setNoVO) +@DIRACScript() +def main(): + global vo + global noVOFlag + global allVOsFlag + + Script.setUsageMessage(""" +Get status of the available Storage Elements -Script.parseCommandLine() +Usage: + %s [] +""" % Script.scriptName) -from DIRAC import gConfig, gLogger -from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus -from DIRAC.Core.Utilities.PrettyPrint import printTable -from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup + Script.registerSwitch("V:", "vo=", "Virtual Organization", setVO) + Script.registerSwitch("a", "all", "All Virtual Organizations flag", setAllVO) + Script.registerSwitch("n", "noVO", "No Virtual Organizations assigned flag", setNoVO) -storageCFGBase = "/Resources/StorageElements" + Script.parseCommandLine() -res = gConfig.getSections(storageCFGBase, True) -if not res['OK']: - gLogger.error('Failed to get storage element info') - gLogger.error(res['Message']) - DIRACexit(1) + from DIRAC import gConfig, gLogger + from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus + from DIRAC.Core.Utilities.PrettyPrint import printTable + from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup -gLogger.info("%s %s %s" % ('Storage Element'.ljust(25), 'Read Status'.rjust(15), 'Write Status'.rjust(15))) + storageCFGBase = "/Resources/StorageElements" -seList = sorted(res['Value']) + res = gConfig.getSections(storageCFGBase, True) + if not res['OK']: + gLogger.error('Failed to get storage element info') + gLogger.error(res['Message']) + DIRACexit(1) -resourceStatus = ResourceStatus() + gLogger.info("%s %s %s" % ('Storage Element'.ljust(25), 'Read Status'.rjust(15), 'Write Status'.rjust(15))) -res = resourceStatus.getElementStatus(seList, "StorageElement") -if not res['OK']: - gLogger.error("Failed to get StorageElement status for %s" % str(seList)) - DIRACexit(1) + seList = sorted(res['Value']) -fields = ['SE', 'ReadAccess', 'WriteAccess', 'RemoveAccess', 'CheckAccess'] -records = [] + resourceStatus = ResourceStatus() -if vo is None and not allVOsFlag: - result = getVOfromProxyGroup() - if not result['OK']: - gLogger.error('Failed to determine the user VO') + res = resourceStatus.getElementStatus(seList, "StorageElement") + if not res['OK']: + gLogger.error("Failed to get StorageElement status for %s" % str(seList)) DIRACexit(1) - vo = result['Value'] -for se, statusDict in res['Value'].items(): + fields = ['SE', 'ReadAccess', 'WriteAccess', 'RemoveAccess', 'CheckAccess'] + records = [] + + if vo is None and not allVOsFlag: + result = getVOfromProxyGroup() + if not result['OK']: + gLogger.error('Failed to determine the user VO') + DIRACexit(1) + vo = result['Value'] + + for se, statusDict in res['Value'].items(): + + # Check if the SE is allowed for the user VO + if not allVOsFlag: + voList = gConfig.getValue('/Resources/StorageElements/%s/VO' % se, []) + if noVOFlag and voList: + continue + if voList and vo not in voList: + continue + + record = [se] + for status in fields[1:]: + value = statusDict.get(status, 'Unknown') + record.append(value) + records.append(record) - # Check if the SE is allowed for the user VO - if not allVOsFlag: - voList = gConfig.getValue('/Resources/StorageElements/%s/VO' % se, []) - if noVOFlag and voList: - continue - if voList and vo not in voList: - continue + printTable(fields, records, numbering=False, sortField='SE') - record = [se] - for status in fields[1:]: - value = statusDict.get(status, 'Unknown') - record.append(value) - records.append(record) + DIRACexit(0) -printTable(fields, records, numbering=False, sortField='SE') -DIRACexit(0) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py index 0aaf28a5a10..23ce9e99821 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py @@ -11,141 +11,148 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script -days = 0 -months = 0 -years = 0 -wildcard = None -baseDir = '' -emptyDirsFlag = False -Script.registerSwitch("D:", "Days=", "Match files older than number of days [%s]" % days) -Script.registerSwitch("M:", "Months=", "Match files older than number of months [%s]" % months) -Script.registerSwitch("Y:", "Years=", "Match files older than number of years [%s]" % years) -Script.registerSwitch("w:", "Wildcard=", "Wildcard for matching filenames [All]") -Script.registerSwitch("b:", "BaseDir=", "Base directory to begin search (default /[vo]/user/[initial]/[username])") -Script.registerSwitch("e", "EmptyDirs", "Create a list of empty directories") - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % Script.scriptName, ])) - -Script.parseCommandLine(ignoreErrors=False) - -for switch in Script.getUnprocessedSwitches(): - if switch[0] == "D" or switch[0].lower() == "days": - days = int(switch[1]) - if switch[0] == "M" or switch[0].lower() == "months": - months = int(switch[1]) - if switch[0] == "Y" or switch[0].lower() == "years": - years = int(switch[1]) - if switch[0].lower() == "w" or switch[0].lower() == "wildcard": - wildcard = '*' + switch[1] - if switch[0].lower() == "b" or switch[0].lower() == "basedir": - baseDir = switch[1] - if switch[0].lower() == "e" or switch[0].lower() == "emptydirs": - emptyDirsFlag = True - -import DIRAC -from DIRAC import gLogger -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -from datetime import datetime, timedelta -import sys -import os -import time -import fnmatch -fc = FileCatalog() - - -def isOlderThan(cTimeStruct, days): - timeDelta = timedelta(days=days) - maxCTime = datetime.utcnow() - timeDelta - if cTimeStruct < maxCTime: - return True - return False - - -withMetadata = False -if days or months or years: - withMetadata = True -totalDays = 0 -if years: - totalDays += 365 * years -if months: - totalDays += 30 * months -if days: - totalDays += days - -res = getProxyInfo(False, False) -if not res['OK']: - gLogger.error("Failed to get client proxy information.", res['Message']) - DIRAC.exit(2) -proxyInfo = res['Value'] -if proxyInfo['secondsLeft'] == 0: - gLogger.error("Proxy expired") - DIRAC.exit(2) -username = proxyInfo['username'] -vo = '' -if 'group' in proxyInfo: - vo = getVOForGroup(proxyInfo['group']) -if not baseDir: - if not vo: - gLogger.error('Could not determine VO') - Script.showHelp() - baseDir = '/%s/user/%s/%s' % (vo, username[0], username) - -baseDir = baseDir.rstrip('/') - -gLogger.notice('Will search for files in %s%s' % (baseDir, (' matching %s' % wildcard) if wildcard else '')) -activeDirs = [baseDir] - -allFiles = [] -emptyDirs = [] - -while len(activeDirs) > 0: - currentDir = activeDirs.pop() - res = fc.listDirectory(currentDir, withMetadata, timeout=360) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + days = 0 + months = 0 + years = 0 + wildcard = None + baseDir = '' + emptyDirsFlag = False + Script.registerSwitch("D:", "Days=", "Match files older than number of days [%s]" % days) + Script.registerSwitch("M:", "Months=", "Match files older than number of months [%s]" % months) + Script.registerSwitch("Y:", "Years=", "Match files older than number of years [%s]" % years) + Script.registerSwitch("w:", "Wildcard=", "Wildcard for matching filenames [All]") + Script.registerSwitch("b:", "BaseDir=", "Base directory to begin search (default /[vo]/user/[initial]/[username])") + Script.registerSwitch("e", "EmptyDirs", "Create a list of empty directories") + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % Script.scriptName, ])) + + Script.parseCommandLine(ignoreErrors=False) + + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "D" or switch[0].lower() == "days": + days = int(switch[1]) + if switch[0] == "M" or switch[0].lower() == "months": + months = int(switch[1]) + if switch[0] == "Y" or switch[0].lower() == "years": + years = int(switch[1]) + if switch[0].lower() == "w" or switch[0].lower() == "wildcard": + wildcard = '*' + switch[1] + if switch[0].lower() == "b" or switch[0].lower() == "basedir": + baseDir = switch[1] + if switch[0].lower() == "e" or switch[0].lower() == "emptydirs": + emptyDirsFlag = True + + import DIRAC + from DIRAC import gLogger + from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + from datetime import datetime, timedelta + import sys + import os + import time + import fnmatch + fc = FileCatalog() + + def isOlderThan(cTimeStruct, days): + timeDelta = timedelta(days=days) + maxCTime = datetime.utcnow() - timeDelta + if cTimeStruct < maxCTime: + return True + return False + + withMetadata = False + if days or months or years: + withMetadata = True + totalDays = 0 + if years: + totalDays += 365 * years + if months: + totalDays += 30 * months + if days: + totalDays += days + + res = getProxyInfo(False, False) if not res['OK']: - gLogger.error("Error retrieving directory contents", "%s %s" % (currentDir, res['Message'])) - elif currentDir in res['Value']['Failed']: - gLogger.error("Error retrieving directory contents", "%s %s" % (currentDir, res['Value']['Failed'][currentDir])) - else: - dirContents = res['Value']['Successful'][currentDir] - subdirs = dirContents['SubDirs'] - files = dirContents['Files'] - if not subdirs and not files: - emptyDirs.append(currentDir) - gLogger.notice('%s: empty directory' % currentDir) + gLogger.error("Failed to get client proxy information.", res['Message']) + DIRAC.exit(2) + proxyInfo = res['Value'] + if proxyInfo['secondsLeft'] == 0: + gLogger.error("Proxy expired") + DIRAC.exit(2) + username = proxyInfo['username'] + vo = '' + if 'group' in proxyInfo: + vo = getVOForGroup(proxyInfo['group']) + if not baseDir: + if not vo: + gLogger.error('Could not determine VO') + Script.showHelp() + baseDir = '/%s/user/%s/%s' % (vo, username[0], username) + + baseDir = baseDir.rstrip('/') + + gLogger.notice('Will search for files in %s%s' % (baseDir, (' matching %s' % wildcard) if wildcard else '')) + activeDirs = [baseDir] + + allFiles = [] + emptyDirs = [] + + while len(activeDirs) > 0: + currentDir = activeDirs.pop() + res = fc.listDirectory(currentDir, withMetadata, timeout=360) + if not res['OK']: + gLogger.error("Error retrieving directory contents", "%s %s" % (currentDir, res['Message'])) + elif currentDir in res['Value']['Failed']: + gLogger.error("Error retrieving directory contents", "%s %s" % (currentDir, res['Value']['Failed'][currentDir])) else: - for subdir in sorted(subdirs, reverse=True): - if (not withMetadata) or isOlderThan(subdirs[subdir]['CreationDate'], totalDays): - activeDirs.append(subdir) - for filename in sorted(files): - fileOK = False - if (not withMetadata) or isOlderThan(files[filename]['MetaData']['CreationDate'], totalDays): - if wildcard is None or fnmatch.fnmatch(filename, wildcard): - fileOK = True - if not fileOK: - files.pop(filename) - allFiles += sorted(files) - - if len(files) or len(subdirs): - gLogger.notice("%s: %d files%s, %d sub-directories" % - (currentDir, len(files), ' matching' if withMetadata or wildcard else '', len(subdirs))) - -outputFileName = '%s.lfns' % baseDir.replace('/%s' % vo, '%s' % vo).replace('/', '-') -outputFile = open(outputFileName, 'w') -for lfn in sorted(allFiles): - outputFile.write(lfn + '\n') -outputFile.close() -gLogger.notice('%d matched files have been put in %s' % (len(allFiles), outputFileName)) - -if emptyDirsFlag: - outputFileName = '%s.emptydirs' % baseDir.replace('/%s' % vo, '%s' % vo).replace('/', '-') + dirContents = res['Value']['Successful'][currentDir] + subdirs = dirContents['SubDirs'] + files = dirContents['Files'] + if not subdirs and not files: + emptyDirs.append(currentDir) + gLogger.notice('%s: empty directory' % currentDir) + else: + for subdir in sorted(subdirs, reverse=True): + if (not withMetadata) or isOlderThan(subdirs[subdir]['CreationDate'], totalDays): + activeDirs.append(subdir) + for filename in sorted(files): + fileOK = False + if (not withMetadata) or isOlderThan(files[filename]['MetaData']['CreationDate'], totalDays): + if wildcard is None or fnmatch.fnmatch(filename, wildcard): + fileOK = True + if not fileOK: + files.pop(filename) + allFiles += sorted(files) + + if len(files) or len(subdirs): + gLogger.notice("%s: %d files%s, %d sub-directories" % + (currentDir, len(files), ' matching' if withMetadata or wildcard else '', len(subdirs))) + + outputFileName = '%s.lfns' % baseDir.replace('/%s' % vo, '%s' % vo).replace('/', '-') outputFile = open(outputFileName, 'w') - for dir in sorted(emptyDirs): - outputFile.write(dir + '\n') + for lfn in sorted(allFiles): + outputFile.write(lfn + '\n') outputFile.close() - gLogger.notice('%d empty directories have been put in %s' % (len(emptyDirs), outputFileName)) + gLogger.notice('%d matched files have been put in %s' % (len(allFiles), outputFileName)) -DIRAC.exit(0) + if emptyDirsFlag: + outputFileName = '%s.emptydirs' % baseDir.replace('/%s' % vo, '%s' % vo).replace('/', '-') + outputFile = open(outputFileName, 'w') + for dir in sorted(emptyDirs): + outputFile.write(dir + '\n') + outputFile.close() + gLogger.notice('%d empty directories have been put in %s' % (len(emptyDirs), outputFileName)) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py index f6efb12f4ca..968e8dd7012 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py @@ -6,32 +6,40 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage(""" -Get the currently defined user data volume quotas - -Usage: - %s [options] -""" % Script.scriptName) - -Script.parseCommandLine(ignoreErrors=False) - -import DIRAC -from DIRAC import gLogger, gConfig -from DIRAC.Core.Security.ProxyInfo import getProxyInfo - -res = getProxyInfo(False, False) -if not res['OK']: - gLogger.error("Failed to get client proxy information.", res['Message']) - DIRAC.exit(2) -proxyInfo = res['Value'] -username = proxyInfo['username'] - -try: - quota = gConfig.getValue('/Registry/DefaultStorageQuota', 0.) - quota = gConfig.getValue('/Registry/Users/%s/Quota' % username, quota) - gLogger.notice('Current quota found to be %.1f GB' % quota) - DIRAC.exit(0) -except Exception as x: - gLogger.exception("Failed to convert retrieved quota", '', x) - DIRAC.exit(-1) + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Get the currently defined user data volume quotas + + Usage: + %s [options] + """ % Script.scriptName) + + Script.parseCommandLine(ignoreErrors=False) + + import DIRAC + from DIRAC import gLogger, gConfig + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + + res = getProxyInfo(False, False) + if not res['OK']: + gLogger.error("Failed to get client proxy information.", res['Message']) + DIRAC.exit(2) + proxyInfo = res['Value'] + username = proxyInfo['username'] + + try: + quota = gConfig.getValue('/Registry/DefaultStorageQuota', 0.) + quota = gConfig.getValue('/Registry/Users/%s/Quota' % username, quota) + gLogger.notice('Current quota found to be %.1f GB' % quota) + DIRAC.exit(0) + except Exception as x: + gLogger.exception("Failed to convert retrieved quota", '', x) + DIRAC.exit(-1) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_CAs.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_CAs.py index 167c9680d9d..ff9d72caada 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_CAs.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_CAs.py @@ -18,32 +18,41 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient __RCSID__ = "$Id$" -Script.addDefaultOptionValue('/DIRAC/Security/SkipCAChecks', 'yes') -Script.setUsageMessage(__doc__) - -Script.parseCommandLine(ignoreErrors=True) - -bdc = BundleDeliveryClient() - -result = bdc.syncCAs() -if not result['OK']: - DIRAC.gLogger.error("Error while updating CAs", result['Message']) - DIRAC.exit(1) -elif result['Value']: - DIRAC.gLogger.notice("CAs got updated") -else: - DIRAC.gLogger.notice("CAs are already synchronized") - -result = bdc.syncCRLs() -if not result['OK']: - DIRAC.gLogger.error("Error while updating CRLs", result['Message']) - DIRAC.exit(1) -elif result['Value']: - DIRAC.gLogger.notice("CRLs got updated") -else: - DIRAC.gLogger.notice("CRLs are already synchronized") - -DIRAC.exit(0) + + +@DIRACScript() +def main(): + Script.addDefaultOptionValue('/DIRAC/Security/SkipCAChecks', 'yes') + Script.setUsageMessage(__doc__) + + Script.parseCommandLine(ignoreErrors=True) + + bdc = BundleDeliveryClient() + + result = bdc.syncCAs() + if not result['OK']: + DIRAC.gLogger.error("Error while updating CAs", result['Message']) + DIRAC.exit(1) + elif result['Value']: + DIRAC.gLogger.notice("CAs got updated") + else: + DIRAC.gLogger.notice("CAs are already synchronized") + + result = bdc.syncCRLs() + if not result['OK']: + DIRAC.gLogger.error("Error while updating CRLs", result['Message']) + DIRAC.exit(1) + elif result['Value']: + DIRAC.gLogger.notice("CRLs got updated") + else: + DIRAC.gLogger.notice("CRLs are already synchronized") + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py index 444654d12c2..69830d92b4f 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py @@ -14,6 +14,7 @@ from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.ConfigurationSystem.Client.Helpers import Registry @@ -92,71 +93,77 @@ def registerCLISwitches(self): Script.registerSwitch("m:", "vomsAttr=", "VOMS attribute to require", self.setVOMSAttr) -params = Params() -params.registerCLISwitches() - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... group' % Script.scriptName, - 'Arguments:', - ' DN: DN of the user', - ' user: DIRAC user name (will fail if there is more than 1 DN registered)', - ' group: DIRAC group name'])) - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) != 2: - Script.showHelp() - -userGroup = str(args[1]) -userDN = str(args[0]) -userName = False -if userDN.find("/") != 0: - userName = userDN - retVal = Registry.getDNForUsername(userName) - if not retVal['OK']: - gLogger.notice("Cannot discover DN for username %s\n\t%s" % (userName, retVal['Message'])) - DIRAC.exit(2) - DNList = retVal['Value'] - if len(DNList) > 1: - gLogger.notice("Username %s has more than one DN registered" % userName) - ind = 0 - for dn in DNList: - gLogger.notice("%d %s" % (ind, dn)) - ind += 1 - inp = raw_input("Which DN do you want to download? [default 0] ") - if not inp: - inp = 0 +@DIRACScript() +def main(): + params = Params() + params.registerCLISwitches() + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... group' % Script.scriptName, + 'Arguments:', + ' DN: DN of the user', + ' user: DIRAC user name (will fail if there is more than 1 DN registered)', + ' group: DIRAC group name'])) + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) != 2: + Script.showHelp() + + userGroup = str(args[1]) + userDN = str(args[0]) + userName = False + if userDN.find("/") != 0: + userName = userDN + retVal = Registry.getDNForUsername(userName) + if not retVal['OK']: + gLogger.notice("Cannot discover DN for username %s\n\t%s" % (userName, retVal['Message'])) + DIRAC.exit(2) + DNList = retVal['Value'] + if len(DNList) > 1: + gLogger.notice("Username %s has more than one DN registered" % userName) + ind = 0 + for dn in DNList: + gLogger.notice("%d %s" % (ind, dn)) + ind += 1 + inp = raw_input("Which DN do you want to download? [default 0] ") + if not inp: + inp = 0 + else: + inp = int(inp) + userDN = DNList[inp] else: - inp = int(inp) - userDN = DNList[inp] + userDN = DNList[0] + + if not params.proxyPath: + if not userName: + result = Registry.getUsernameForDN(userDN) + if not result['OK']: + gLogger.notice("DN '%s' is not registered in DIRAC" % userDN) + DIRAC.exit(2) + userName = result['Value'] + params.proxyPath = "%s/proxy.%s.%s" % (os.getcwd(), userName, userGroup) + + if params.enableVOMS: + result = gProxyManager.downloadVOMSProxy(userDN, userGroup, limited=params.limited, + requiredTimeLeft=params.proxyLifeTime, + requiredVOMSAttribute=params.vomsAttr) else: - userDN = DNList[0] + result = gProxyManager.downloadProxy(userDN, userGroup, limited=params.limited, + requiredTimeLeft=params.proxyLifeTime) + if not result['OK']: + gLogger.notice('Proxy file cannot be retrieved: %s' % result['Message']) + DIRAC.exit(2) + chain = result['Value'] + result = chain.dumpAllToFile(params.proxyPath) + if not result['OK']: + gLogger.notice('Proxy file cannot be written to %s: %s' % (params.proxyPath, result['Message'])) + DIRAC.exit(2) + gLogger.notice("Proxy downloaded to %s" % params.proxyPath) + DIRAC.exit(0) -if not params.proxyPath: - if not userName: - result = Registry.getUsernameForDN(userDN) - if not result['OK']: - gLogger.notice("DN '%s' is not registered in DIRAC" % userDN) - DIRAC.exit(2) - userName = result['Value'] - params.proxyPath = "%s/proxy.%s.%s" % (os.getcwd(), userName, userGroup) - -if params.enableVOMS: - result = gProxyManager.downloadVOMSProxy(userDN, userGroup, limited=params.limited, - requiredTimeLeft=params.proxyLifeTime, - requiredVOMSAttribute=params.vomsAttr) -else: - result = gProxyManager.downloadProxy(userDN, userGroup, limited=params.limited, - requiredTimeLeft=params.proxyLifeTime) -if not result['OK']: - gLogger.notice('Proxy file cannot be retrieved: %s' % result['Message']) - DIRAC.exit(2) -chain = result['Value'] -result = chain.dumpAllToFile(params.proxyPath) -if not result['OK']: - gLogger.notice('Proxy file cannot be written to %s: %s' % (params.proxyPath, result['Message'])) - DIRAC.exit(2) -gLogger.notice("Proxy downloaded to %s" % params.proxyPath) -DIRAC.exit(0) + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_proxy_upload.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_proxy_upload.py index 29fe5ab25e1..832cf7cc9f8 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_proxy_upload.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_proxy_upload.py @@ -9,11 +9,14 @@ from __future__ import division import sys from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.FrameworkSystem.Client.ProxyUpload import CLIParams, uploadProxy __RCSID__ = "$Id$" -if __name__ == "__main__": + +@DIRACScript() +def main(): cliParams = CLIParams() cliParams.registerCLISwitches() @@ -24,3 +27,7 @@ print(retVal['Message']) sys.exit(1) sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_sysadmin_cli.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_sysadmin_cli.py index 86394b7a93a..2f6383fe0f4 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_sysadmin_cli.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_sysadmin_cli.py @@ -3,17 +3,25 @@ from __future__ import division from __future__ import print_function from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript __RCSID__ = "$Id$" -host = None -Script.registerSwitch("H:", "host=", " Target host") -Script.parseCommandLine(ignoreErrors=False) -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "h" or switch[0].lower() == "host": - host = switch[1] -from DIRAC.FrameworkSystem.Client.SystemAdministratorClientCLI import SystemAdministratorClientCLI +@DIRACScript() +def main(): + host = None + Script.registerSwitch("H:", "host=", " Target host") + Script.parseCommandLine(ignoreErrors=False) + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "h" or switch[0].lower() == "host": + host = switch[1] -cli = SystemAdministratorClientCLI(host) -cli.cmdloop() + from DIRAC.FrameworkSystem.Client.SystemAdministratorClientCLI import SystemAdministratorClientCLI + + cli = SystemAdministratorClientCLI(host) + cli.cmdloop() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_instance.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_instance.py index 6ffdb2e3c3c..26f139b839e 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_instance.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_instance.py @@ -12,235 +12,235 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s version' % Script.scriptName, + ' ', + 'Arguments:', + ' version: version of DIRAC you want to update to', + ])) + Script.registerSwitch("", "hosts=", "Comma separated list of hosts or file containing row wise list of hosts" + " targeted for update (leave empty for all)") + Script.registerSwitch("", "retry=", "Number of retry attempts on hosts that have failed to update") + + Script.parseCommandLine(ignoreErrors=False) + + args = Script.getPositionalArgs() + if len(args) < 1 or len(args) > 2: + Script.showHelp() + + version = args[0] + retry = 0 + hosts = [] + + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "hosts": + hosts = switch[1] + if switch[0] == "retry": + retry = int(switch[1]) + + try: + with open(hosts, 'r') as f: + hosts = f.read().splitlines() + hosts = [str(host) for host in hosts] + except Exception: + pass + + if not isinstance(hosts, list): + hosts = hosts.split(',') + + from concurrent.futures import ThreadPoolExecutor, as_completed + + from DIRAC import S_OK, S_ERROR + from DIRAC import gLogger + from DIRAC.FrameworkSystem.Client.SystemAdministratorClient import SystemAdministratorClient + from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient + + def getListOfHosts(): + """ + Return the list of all hosts that constitute a DIRAC instance + """ + client = ComponentMonitoringClient() + result = client.getHosts({}, False, False) + if result['OK']: + hosts = [host['HostName'] for host in result['Value']] + return S_OK(hosts) + return S_ERROR('Cannot get list of hosts: %s' % result['Message']) + + def parseHostname(hostName): + """ + Separate the hostname from the port + + :param str hostName: hostname you want to parse + """ + hostList = hostName.split(':') + host = hostList[0] + if len(hostList) == 2: + port = hostList[1] + else: + port = None + return host, port -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s version' % Script.scriptName, - ' ', - 'Arguments:', - ' version: version of DIRAC you want to update to'] - ) - ) -Script.registerSwitch("", "hosts=", "Comma separated list of hosts or file containing row wise list of hosts" - " targeted for update (leave empty for all)") -Script.registerSwitch("", "retry=", "Number of retry attempts on hosts that have failed to update") - -Script.parseCommandLine(ignoreErrors=False) - -args = Script.getPositionalArgs() -if len(args) < 1 or len(args) > 2: - Script.showHelp() - -version = args[0] -retry = 0 -hosts = [] - -for switch in Script.getUnprocessedSwitches(): - if switch[0] == "hosts": - hosts = switch[1] - if switch[0] == "retry": - retry = int(switch[1]) - -try: - with open(hosts, 'r') as f: - hosts = f.read().splitlines() - hosts = [str(host) for host in hosts] -except Exception: - pass - -if not isinstance(hosts, list): - hosts = hosts.split(',') - -from concurrent.futures import ThreadPoolExecutor, as_completed - -from DIRAC import S_OK, S_ERROR -from DIRAC import gLogger -from DIRAC.FrameworkSystem.Client.SystemAdministratorClient import SystemAdministratorClient -from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient - - -def getListOfHosts(): - """ - Return the list of all hosts that constitute a DIRAC instance - """ - client = ComponentMonitoringClient() - result = client.getHosts({}, False, False) - if result['OK']: - hosts = [host['HostName'] for host in result['Value']] - return S_OK(hosts) - return S_ERROR('Cannot get list of hosts: %s' % result['Message']) - - -def parseHostname(hostName): - """ - Separate the hostname from the port - - :param str hostName: hostname you want to parse - """ - hostList = hostName.split(':') - host = hostList[0] - if len(hostList) == 2: - port = hostList[1] - else: - port = None - return host, port - - -def updateHost(hostName, version): - """ - Apply update to specific host - - :param str hostName: name of the host you want to update - :param str version: version vArBpC you want to update to - """ - host, port = parseHostname(hostName) - - client = SystemAdministratorClient(host, port) - result = client.ping() - if not result['OK']: - gLogger.error("Cannot connect to %s" % host) - return result - - gLogger.notice("Initiating software update of %s, this can take a while, please be patient ..." % host) - result = client.updateSoftware(version, '', '', timeout=600) - if not result['OK']: - return result - return S_OK() + def updateHost(hostName, version): + """ + Apply update to specific host + :param str hostName: name of the host you want to update + :param str version: version vArBpC you want to update to + """ + host, port = parseHostname(hostName) -def updateHosts(version, hosts=[]): - """ - Apply update to all hosts + client = SystemAdministratorClient(host, port) + result = client.ping() + if not result['OK']: + gLogger.error("Cannot connect to %s" % host) + return result - :param str version: version vArBpC you want to update to - :param list[str] hosts: list of hosts to be updated, defaults to [] - """ - if not hosts: - result = getListOfHosts() + gLogger.notice("Initiating software update of %s, this can take a while, please be patient ..." % host) + result = client.updateSoftware(version, '', '', timeout=600) if not result['OK']: return result - hosts = result['Value'] + return S_OK() + + def updateHosts(version, hosts=[]): + """ + Apply update to all hosts + + :param str version: version vArBpC you want to update to + :param list[str] hosts: list of hosts to be updated, defaults to [] + """ + if not hosts: + result = getListOfHosts() + if not result['OK']: + return result + hosts = result['Value'] + + updateSuccess = [] + updateFail = [] + + executor = ThreadPoolExecutor(max_workers=len(hosts)) + futureUpdate = {executor.submit(updateHost, host, version): host for host in hosts} + for future in as_completed(futureUpdate): + host = futureUpdate[future] + result = future.result() + if result['OK']: + updateSuccess.append(host) + else: + updateFail.append(host) + + if not updateFail: + gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + gLogger.notice("!!! Successfully updated all hosts !!!") + gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + return S_OK([updateSuccess, updateFail]) + if not updateSuccess: + gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") + gLogger.notice("XXXXX Failed to update all hosts XXXXX") + gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") + return S_OK([updateSuccess, updateFail]) + gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!") + gLogger.notice("X!X!X Partially updated hosts X!X!X!") + gLogger.notice("Succeeded to update:") + for host in updateSuccess: + gLogger.notice(" + %s" % host) + gLogger.notice("Failed to update:") + for host in updateFail: + gLogger.notice(" - %s" % host) + gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!") + return S_OK([updateSuccess, updateFail]) - updateSuccess = [] - updateFail = [] + def restartHost(hostName): + """ + Restart all systems and components of a host - executor = ThreadPoolExecutor(max_workers=len(hosts)) - futureUpdate = {executor.submit(updateHost, host, version): host for host in hosts} - for future in as_completed(futureUpdate): - host = futureUpdate[future] - result = future.result() - if result['OK']: - updateSuccess.append(host) - else: - updateFail.append(host) + :param str hostName: name of the host you want to restart + """ + host, port = parseHostname(hostName) - if not updateFail: - gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - gLogger.notice("!!! Successfully updated all hosts !!!") - gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - return S_OK([updateSuccess, updateFail]) - if not updateSuccess: - gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") - gLogger.notice("XXXXX Failed to update all hosts XXXXX") - gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") - return S_OK([updateSuccess, updateFail]) - gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!") - gLogger.notice("X!X!X Partially updated hosts X!X!X!") - gLogger.notice("Succeeded to update:") - for host in updateSuccess: - gLogger.notice(" + %s" % host) - gLogger.notice("Failed to update:") - for host in updateFail: - gLogger.notice(" - %s" % host) - gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!") - return S_OK([updateSuccess, updateFail]) - - -def restartHost(hostName): - """ - Restart all systems and components of a host - - :param str hostName: name of the host you want to restart - """ - host, port = parseHostname(hostName) - - gLogger.notice("Pinging %s ..." % host) - - client = SystemAdministratorClient(host, port) - result = client.ping() - if not result['OK']: - gLogger.error("Could not connect to %s: %s" % (host, result['Message'])) - return result - gLogger.notice("Host %s is active" % host) - - gLogger.notice("Initiating restart of all systems and components") - # This restart call will always return S_ERROR because of SystemAdministrator restart - # Connection will be lost to the host - result = client.restartComponent('*', '*') - if result['Message'] == "Peer closed connection": - gLogger.notice("Restarted all systems on %s : connection to SystemAdministrator lost" % host) - return S_OK(result['Message']) - gLogger.error("Received unxpected message: %s" % result['Message']) - return result - - -def updateInstance(version, hosts, retry): - """ - Update each server of an instance and restart them - - :param str version: version vArBpC you want to update to - :param list[str] hosts: list of hosts to be updated - :param int retry: number of retry attempts on hosts that have failed to update - """ - result = updateHosts(version, hosts) - if not result['OK']: + gLogger.notice("Pinging %s ..." % host) + + client = SystemAdministratorClient(host, port) + result = client.ping() + if not result['OK']: + gLogger.error("Could not connect to %s: %s" % (host, result['Message'])) + return result + gLogger.notice("Host %s is active" % host) + + gLogger.notice("Initiating restart of all systems and components") + # This restart call will always return S_ERROR because of SystemAdministrator restart + # Connection will be lost to the host + result = client.restartComponent('*', '*') + if result['Message'] == "Peer closed connection": + gLogger.notice("Restarted all systems on %s : connection to SystemAdministrator lost" % host) + return S_OK(result['Message']) + gLogger.error("Received unxpected message: %s" % result['Message']) return result - updateSuccess = result['Value'][0] - updateFail = result['Value'][1] - restartSuccess = [] - restartFail = [] - for host in updateSuccess: - result = restartHost(host) - if result['OK']: - restartSuccess.append(host) - else: - restartFail.append(host) - - if not restartFail and not updateFail: - return S_OK("Successfully updated and restarted all hosts") - - gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") - gLogger.notice("XXXXX There were problems in the update process XXXXX") - gLogger.notice("Succeeded to update:") - for host in updateSuccess: - gLogger.notice(" + %s" % host) - gLogger.notice("Succeeded to restart:") - for host in restartSuccess: - gLogger.notice(" + %s" % host) - gLogger.notice("Failed to update:") - for host in updateFail: - gLogger.notice(" - %s" % host) - gLogger.notice("Failed to restart:") - for host in restartFail: - gLogger.notice(" - %s" % host) - gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") - - if retry > 0: - retryHosts = list(set(updateFail + restartFail)) - gLogger.notice("Retrying update on (%s attempts remaining):" % retry) - for host in retryHosts: + def updateInstance(version, hosts, retry): + """ + Update each server of an instance and restart them + + :param str version: version vArBpC you want to update to + :param list[str] hosts: list of hosts to be updated + :param int retry: number of retry attempts on hosts that have failed to update + """ + result = updateHosts(version, hosts) + if not result['OK']: + return result + + updateSuccess = result['Value'][0] + updateFail = result['Value'][1] + restartSuccess = [] + restartFail = [] + for host in updateSuccess: + result = restartHost(host) + if result['OK']: + restartSuccess.append(host) + else: + restartFail.append(host) + + if not restartFail and not updateFail: + return S_OK("Successfully updated and restarted all hosts") + + gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") + gLogger.notice("XXXXX There were problems in the update process XXXXX") + gLogger.notice("Succeeded to update:") + for host in updateSuccess: + gLogger.notice(" + %s" % host) + gLogger.notice("Succeeded to restart:") + for host in restartSuccess: + gLogger.notice(" + %s" % host) + gLogger.notice("Failed to update:") + for host in updateFail: + gLogger.notice(" - %s" % host) + gLogger.notice("Failed to restart:") + for host in restartFail: gLogger.notice(" - %s" % host) gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") - return updateInstance(version, retryHosts, retry - 1) - return S_ERROR("Update failed!") + if retry > 0: + retryHosts = list(set(updateFail + restartFail)) + gLogger.notice("Retrying update on (%s attempts remaining):" % retry) + for host in retryHosts: + gLogger.notice(" - %s" % host) + gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") + return updateInstance(version, retryHosts, retry - 1) + return S_ERROR("Update failed!") -if __name__ == "__main__": result = updateInstance(version, hosts, retry) if not result['OK']: gLogger.fatal(result['Message']) DIRAC.exit(1) gLogger.notice(result['Value']) DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py index cf07745ffb6..54badad3bf3 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py @@ -10,72 +10,80 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s version' % Script.scriptName, - ' ', - 'Arguments:', - ' version: pilot version you want to update to'] - ) - ) -Script.registerSwitch( - "v:", "vo=", "Location of pilot version in CS /Operations//Pilot/Version" - " (default value specified in CS under /DIRAC/DefaultSetup)") - -Script.parseCommandLine(ignoreErrors=False) - -args = Script.getPositionalArgs() -if len(args) < 1 or len(args) > 2: - Script.showHelp() - -version = args[0] -vo = None -for switch in Script.getUnprocessedSwitches(): - if switch[0] == "v" or switch[0] == "vo": - vo = switch[1] - -from DIRAC import S_OK, S_ERROR -from DIRAC import gConfig, gLogger -from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI - - -def updatePilot(version, vo): - """ - Update in the CS the pilot version used, - If only one version present in CS it's overwritten. - If two versions present, the new one is added and the last removed - - :param version: version vArBpC of pilot you want to use - :param vo: Location of pilot version in CS /Operations//Pilot/Version - """ - setup = vo - if not vo: - setup = gConfig.getValue('/DIRAC/DefaultSetup') - if not setup: - return S_ERROR("No value set for /DIRAC/DefaultSetup in CS") - - pilotVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup, []) - if not pilotVersion: - return S_ERROR("No pilot version set under Operations/%s/Pilot/Version in CS" % setup) - - pilotVersion.pop() - pilotVersion.insert(0, version) - api = CSAPI() - api.setOption('Operations/%s/Pilot/Version' % setup, ", ".join(pilotVersion)) - result = api.commit() - if not result['OK']: - gLogger.fatal('Could not commit new version of pilot!') - return result - newVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup) - return S_OK("New version of pilot set to %s" % newVersion) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s version' % Script.scriptName, + ' ', + 'Arguments:', + ' version: pilot version you want to update to' + ])) + Script.registerSwitch( + "v:", + "vo=", + "Location of pilot version in CS /Operations//Pilot/Version" + " (default value specified in CS under /DIRAC/DefaultSetup)" + ) + Script.parseCommandLine(ignoreErrors=False) + + args = Script.getPositionalArgs() + if len(args) < 1 or len(args) > 2: + Script.showHelp() + + version = args[0] + vo = None + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "v" or switch[0] == "vo": + vo = switch[1] + + from DIRAC import S_OK, S_ERROR + from DIRAC import gConfig, gLogger + from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI + + def updatePilot(version, vo): + """ + Update in the CS the pilot version used, + If only one version present in CS it's overwritten. + If two versions present, the new one is added and the last removed + + :param version: version vArBpC of pilot you want to use + :param vo: Location of pilot version in CS /Operations//Pilot/Version + """ + setup = vo + if not vo: + setup = gConfig.getValue('/DIRAC/DefaultSetup') + if not setup: + return S_ERROR("No value set for /DIRAC/DefaultSetup in CS") + + pilotVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup, []) + if not pilotVersion: + return S_ERROR("No pilot version set under Operations/%s/Pilot/Version in CS" % setup) + + pilotVersion.pop() + pilotVersion.insert(0, version) + api = CSAPI() + api.setOption('Operations/%s/Pilot/Version' % setup, ", ".join(pilotVersion)) + result = api.commit() + if not result['OK']: + gLogger.fatal('Could not commit new version of pilot!') + return result + + newVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup) + return S_OK("New version of pilot set to %s" % newVersion) -if __name__ == "__main__": result = updatePilot(version, vo) if not result['OK']: gLogger.fatal(result['Message']) DIRAC.exit(1) gLogger.notice(result['Value']) DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py index fe1511412d6..0aea5d1f32f 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py @@ -5,6 +5,7 @@ from __future__ import division import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.Core.Utilities import Time @@ -30,41 +31,45 @@ def registerCLISwitches(self): Script.registerSwitch("v:", "valid=", "Required HH:MM for the users", self.setProxyLifeTime) -params = Params() -params.registerCLISwitches() +@DIRACScript() +def main(): + params = Params() + params.registerCLISwitches() -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -result = gProxyManager.getDBContents() -if not result['OK']: - print("Can't retrieve list of users: %s" % result['Message']) - DIRAC.exit(1) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + result = gProxyManager.getDBContents() + if not result['OK']: + print("Can't retrieve list of users: %s" % result['Message']) + DIRAC.exit(1) -keys = result['Value']['ParameterNames'] -records = result['Value']['Records'] -dataDict = {} -now = Time.dateTime() -for record in records: - expirationDate = record[3] - dt = expirationDate - now - secsLeft = dt.days * 86400 + dt.seconds - if secsLeft > params.proxyLifeTime: - userName, userDN, userGroup, _, persistent = record - if userName not in dataDict: - dataDict[userName] = [] - dataDict[userName].append((userDN, userGroup, expirationDate, persistent)) + keys = result['Value']['ParameterNames'] + records = result['Value']['Records'] + dataDict = {} + now = Time.dateTime() + for record in records: + expirationDate = record[3] + dt = expirationDate - now + secsLeft = dt.days * 86400 + dt.seconds + if secsLeft > params.proxyLifeTime: + userName, userDN, userGroup, _, persistent = record + if userName not in dataDict: + dataDict[userName] = [] + dataDict[userName].append((userDN, userGroup, expirationDate, persistent)) + for userName in dataDict: + print("* %s" % userName) + for iP in range(len(dataDict[userName])): + data = dataDict[userName][iP] + print(" DN : %s" % data[0]) + print(" group : %s" % data[1]) + print(" not after : %s" % Time.toString(data[2])) + print(" persistent : %s" % data[3]) + if iP < len(dataDict[userName]) - 1: + print(" -") -for userName in dataDict: - print("* %s" % userName) - for iP in range(len(dataDict[userName])): - data = dataDict[userName][iP] - print(" DN : %s" % data[0]) - print(" group : %s" % data[1]) - print(" not after : %s" % Time.toString(data[2])) - print(" persistent : %s" % data[3]) - if iP < len(dataDict[userName]) - 1: - print(" -") + DIRAC.exit(0) -DIRAC.exit(0) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py index de62a892638..79f0b4b27da 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py @@ -11,6 +11,7 @@ from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import exit as DIRACexit from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller @@ -45,84 +46,94 @@ def setSpecialOption(optVal): return S_OK() -Script.registerSwitch("w", "overwrite", "Overwrite the configuration in the global CS", setOverwrite) -Script.registerSwitch("m:", "module=", "Python module name for the component code", setModule) -Script.registerSwitch("p:", "parameter=", "Special component option ", setSpecialOption) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, - 'Arguments:', - ' System: Name of the DIRAC system (ie: WorkloadManagement)', - ' Service: Name of the DIRAC component (ie: Matcher)'])) - -Script.parseCommandLine() -args = Script.getPositionalArgs() - -if len(args) == 1: - args = args[0].split('/') - -if len(args) != 2: - Script.showHelp(exitCode=1) - -cType = None -system = args[0] -component = args[1] -compOrMod = module if module else component - - -result = gComponentInstaller.getSoftwareComponents(getCSExtensions()) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) -else: - availableComponents = result['Value'] - -for compType in availableComponents: - if system in availableComponents[compType] and compOrMod in availableComponents[compType][system]: - cType = compType[:-1].lower() - break - -if not cType: - gLogger.error('Component %s/%s is not available for installation' % (system, component)) - DIRACexit(1) - -if module: - result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, module, - getCSExtensions(), - overwrite=overwrite) - result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, component, - getCSExtensions(), - specialOptions=specialOptions, - overwrite=overwrite, - addDefaultOptions=False) -else: - result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, component, - getCSExtensions(), - specialOptions=specialOptions, - overwrite=overwrite) - -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) -else: - result = gComponentInstaller.installComponent(cType, system, component, getCSExtensions(), module) +@DIRACScript() +def main(): + global overwrite + global specialOptions + global module + global specialOptions + + Script.registerSwitch("w", "overwrite", "Overwrite the configuration in the global CS", setOverwrite) + Script.registerSwitch("m:", "module=", "Python module name for the component code", setModule) + Script.registerSwitch("p:", "parameter=", "Special component option ", setSpecialOption) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, + 'Arguments:', + ' System: Name of the DIRAC system (ie: WorkloadManagement)', + ' Service: Name of the DIRAC component (ie: Matcher)'])) + + Script.parseCommandLine() + args = Script.getPositionalArgs() + + if len(args) == 1: + args = args[0].split('/') + + if len(args) != 2: + Script.showHelp(exitCode=1) + + cType = None + system = args[0] + component = args[1] + compOrMod = module if module else component + + result = gComponentInstaller.getSoftwareComponents(getCSExtensions()) if not result['OK']: gLogger.error(result['Message']) DIRACexit(1) else: - gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) - result = gComponentInstaller.setupComponent(cType, system, component, getCSExtensions(), module) + availableComponents = result['Value'] + + for compType in availableComponents: + if system in availableComponents[compType] and compOrMod in availableComponents[compType][system]: + cType = compType[:-1].lower() + break + + if not cType: + gLogger.error('Component %s/%s is not available for installation' % (system, component)) + DIRACexit(1) + + if module: + result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, module, + getCSExtensions(), + overwrite=overwrite) + result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, component, + getCSExtensions(), + specialOptions=specialOptions, + overwrite=overwrite, + addDefaultOptions=False) + else: + result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, component, + getCSExtensions(), + specialOptions=specialOptions, + overwrite=overwrite) + + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + else: + result = gComponentInstaller.installComponent(cType, system, component, getCSExtensions(), module) if not result['OK']: gLogger.error(result['Message']) DIRACexit(1) - if component == 'ComponentMonitoring': - result = MonitoringUtilities.monitorInstallation('DB', system, 'InstalledComponentsDB') + else: + gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) + result = gComponentInstaller.setupComponent(cType, system, component, getCSExtensions(), module) if not result['OK']: gLogger.error(result['Message']) DIRACexit(1) - result = MonitoringUtilities.monitorInstallation(cType, system, component, module) - if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - gLogger.notice('Successfully completed the installation of %s/%s' % (system, component)) - DIRACexit() + if component == 'ComponentMonitoring': + result = MonitoringUtilities.monitorInstallation('DB', system, 'InstalledComponentsDB') + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + result = MonitoringUtilities.monitorInstallation(cType, system, component, module) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + gLogger.notice('Successfully completed the installation of %s/%s' % (system, component)) + DIRACexit() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py b/src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py index f857fb196b4..1d88dbe3b24 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py @@ -13,6 +13,7 @@ from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import exit as DIRACexit from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller @@ -47,61 +48,68 @@ def setSpecialOption(optVal): return S_OK() -Script.registerSwitch("w", "overwrite", "Overwrite the configuration in the global CS", setOverwrite) -Script.registerSwitch("m:", "module=", "Python module name for the component code", setModule) -Script.registerSwitch("p:", "parameter=", "Special component option ", setSpecialOption) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, - 'Arguments:', - ' System: Name of the DIRAC system (ie: WorkloadManagement)', - ' Service: Name of the DIRAC component (ie: Matcher)'])) - -Script.parseCommandLine() -args = Script.getPositionalArgs() - -if len(args) == 1: - args = args[0].split('/') - -if len(args) != 2: - Script.showHelp() - DIRACexit(1) - -system = args[0] -component = args[1] -compOrMod = module if module else component - - -result = gComponentInstaller.addDefaultOptionsToCS(gConfig, 'service', system, component, - getCSExtensions(), - specialOptions=specialOptions, - overwrite=overwrite) - -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - - -result = gComponentInstaller.addTornadoOptionsToCS(gConfig) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - -result = gComponentInstaller.installTornado() -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - - -gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) -result = gComponentInstaller.setupTornadoService(system, component, getCSExtensions(), module) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - -result = MonitoringUtilities.monitorInstallation('service', system, component, module) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) -gLogger.notice('Successfully completed the installation of %s/%s' % (system, component)) -DIRACexit() +@DIRACScript() +def main(): + global overwrite + global specialOptions + global module + global specialOptions + Script.registerSwitch("w", "overwrite", "Overwrite the configuration in the global CS", setOverwrite) + Script.registerSwitch("m:", "module=", "Python module name for the component code", setModule) + Script.registerSwitch("p:", "parameter=", "Special component option ", setSpecialOption) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, + 'Arguments:', + ' System: Name of the DIRAC system (ie: WorkloadManagement)', + ' Service: Name of the DIRAC component (ie: Matcher)'])) + + Script.parseCommandLine() + args = Script.getPositionalArgs() + + if len(args) == 1: + args = args[0].split('/') + + if len(args) != 2: + Script.showHelp() + DIRACexit(1) + + system = args[0] + component = args[1] + compOrMod = module if module else component + + result = gComponentInstaller.addDefaultOptionsToCS(gConfig, 'service', system, component, + getCSExtensions(), + specialOptions=specialOptions, + overwrite=overwrite) + + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + result = gComponentInstaller.addTornadoOptionsToCS(gConfig) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + result = gComponentInstaller.installTornado() + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) + result = gComponentInstaller.setupTornadoService(system, component, getCSExtensions(), module) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + result = MonitoringUtilities.monitorInstallation('service', system, component, module) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + gLogger.notice('Successfully completed the installation of %s/%s' % (system, component)) + DIRACexit() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_monitoring_get_components_status.py b/src/DIRAC/FrameworkSystem/scripts/dirac_monitoring_get_components_status.py index 234c627763e..01ef7db6ea1 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_monitoring_get_components_status.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_monitoring_get_components_status.py @@ -5,47 +5,54 @@ import sys from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript __RCSID__ = "$Id$" -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -fieldsToShow = ('ComponentName', 'Type', 'Host', 'Port', 'Status', 'Message') - -from DIRAC.FrameworkSystem.Client.MonitoringClient import gMonitor - -result = gMonitor.getComponentsStatusWebFormatted(sortingList=[['ComponentName', 'ASC']]) -if not result['OK']: - print("ERROR: %s" % result['Message']) - sys.exit(1) -paramNames = result['Value']['ParameterNames'] -records = result['Value']['Records'] -fieldLengths = [] -for param in paramNames: - fieldLengths.append(len(param)) - -for record in records: - for i in range(len(record)): - if paramNames[i] in fieldsToShow: - fieldLengths[i] = max(fieldLengths[i], len(str(record[i]))) -# Print time! -line = [] -sepLine = [] -for i in range(len(paramNames)): - param = paramNames[i] - if param in fieldsToShow: - line.append("%s%s" % (param, " " * (fieldLengths[i] - len(param)))) - sepLine.append("-" * fieldLengths[i]) -print("|".join(line)) -sepLine = "+".join(sepLine) -print(sepLine) -for record in records: +@DIRACScript() +def main(): + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + fieldsToShow = ('ComponentName', 'Type', 'Host', 'Port', 'Status', 'Message') + + from DIRAC.FrameworkSystem.Client.MonitoringClient import gMonitor + + result = gMonitor.getComponentsStatusWebFormatted(sortingList=[['ComponentName', 'ASC']]) + if not result['OK']: + print("ERROR: %s" % result['Message']) + sys.exit(1) + paramNames = result['Value']['ParameterNames'] + records = result['Value']['Records'] + fieldLengths = [] + for param in paramNames: + fieldLengths.append(len(param)) + + for record in records: + for i in range(len(record)): + if paramNames[i] in fieldsToShow: + fieldLengths[i] = max(fieldLengths[i], len(str(record[i]))) + # Print time! line = [] - for i in range(len(record)): - if paramNames[i] in fieldsToShow: - val = str(record[i]) - line.append("%s%s" % (val, " " * (fieldLengths[i] - len(val)))) + sepLine = [] + for i in range(len(paramNames)): + param = paramNames[i] + if param in fieldsToShow: + line.append("%s%s" % (param, " " * (fieldLengths[i] - len(param)))) + sepLine.append("-" * fieldLengths[i]) print("|".join(line)) - # print sepLine + sepLine = "+".join(sepLine) + print(sepLine) + for record in records: + line = [] + for i in range(len(record)): + if paramNames[i] in fieldsToShow: + val = str(record[i]) + line.append("%s%s" % (val, " " * (fieldLengths[i] - len(val)))) + print("|".join(line)) + # print sepLine + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_myproxy_upload.py b/src/DIRAC/FrameworkSystem/scripts/dirac_myproxy_upload.py index 2f5313e2f09..0e7882a69ea 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_myproxy_upload.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_myproxy_upload.py @@ -12,10 +12,10 @@ import sys import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript class Params: - proxyLoc = False dnAsUsername = False @@ -34,31 +34,37 @@ def showVersion(self, arg): return DIRAC.S_OK() -params = Params() +@DIRACScript() +def main(): + params = Params() + + Script.registerSwitch("f:", "file=", "File to use as proxy", params.setProxyLocation) + Script.registerSwitch("D", "DN", "Use DN as myproxy username", params.setDNAsUsername) + Script.registerSwitch("i", "version", "Print version", params.showVersion) + + Script.addDefaultOptionValue("LogLevel", "always") + Script.parseCommandLine() -Script.registerSwitch("f:", "file=", "File to use as proxy", params.setProxyLocation) -Script.registerSwitch("D", "DN", "Use DN as myproxy username", params.setDNAsUsername) -Script.registerSwitch("i", "version", "Print version", params.showVersion) + from DIRAC.Core.Security.MyProxy import MyProxy + from DIRAC.Core.Security import Locations -Script.addDefaultOptionValue("LogLevel", "always") -Script.parseCommandLine() + if not params.proxyLoc: + params.proxyLoc = Locations.getProxyLocation() -from DIRAC.Core.Security.MyProxy import MyProxy -from DIRAC.Core.Security import Locations + if not params.proxyLoc: + print("Can't find any valid proxy") + sys.exit(1) + print("Uploading proxy file %s" % params.proxyLoc) -if not params.proxyLoc: - params.proxyLoc = Locations.getProxyLocation() + mp = MyProxy() + retVal = mp.uploadProxy(params.proxyLoc, params.dnAsUsername) + if not retVal['OK']: + print("Can't upload proxy:") + print(" ", retVal['Message']) + sys.exit(1) + print("Proxy uploaded") + sys.exit(0) -if not params.proxyLoc: - print("Can't find any valid proxy") - sys.exit(1) -print("Uploading proxy file %s" % params.proxyLoc) -mp = MyProxy() -retVal = mp.uploadProxy(params.proxyLoc, params.dnAsUsername) -if not retVal['OK']: - print("Can't upload proxy:") - print(" ", retVal['Message']) - sys.exit(1) -print("Proxy uploaded") -sys.exit(0) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_populate_component_db.py b/src/DIRAC/FrameworkSystem/scripts/dirac_populate_component_db.py index 672dbb441c3..799ab52449f 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_populate_component_db.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_populate_component_db.py @@ -15,6 +15,7 @@ from DIRAC import S_OK, gLogger, gConfig from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient from DIRAC.FrameworkSystem.Client.SystemAdministratorIntegrator \ import SystemAdministratorIntegrator @@ -37,175 +38,181 @@ def setExcludedHosts(value): return S_OK() -Script.registerSwitch( - "e:", - "exclude=", - "Comma separated list of hosts to be excluded from the scanning process", - setExcludedHosts) - - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... [debug]' % Script.scriptName])) +@DIRACScript() +def main(): + global excludedHosts + Script.registerSwitch( + "e:", + "exclude=", + "Comma separated list of hosts to be excluded from the scanning process", + setExcludedHosts) -Script.parseCommandLine(ignoreErrors=False) -args = Script.getPositionalArgs() + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... [debug]' % Script.scriptName])) -componentType = '' + Script.parseCommandLine(ignoreErrors=False) + args = Script.getPositionalArgs() -# Get my setup -mySetup = gConfig.getValue('DIRAC/Setup') + componentType = '' -# Retrieve information from all the hosts -client = SystemAdministratorIntegrator(exclude=excludedHosts) -resultAll = client.getOverallStatus() + # Get my setup + mySetup = gConfig.getValue('DIRAC/Setup') -# Retrieve user installing the component -result = getProxyInfo() -if result['OK']: - user = result['Value']['username'] -else: - DIRACexit(-1) -if not user: - user = 'unknown' + # Retrieve information from all the hosts + client = SystemAdministratorIntegrator(exclude=excludedHosts) + resultAll = client.getOverallStatus() -notificationClient = NotificationClient() -for host in resultAll['Value']: - if not resultAll['Value'][host]['OK']: - # If the host cannot be contacted, exclude it and send message - excludedHosts.append(host) + # Retrieve user installing the component + result = getProxyInfo() + if result['OK']: + user = result['Value']['username'] + else: + DIRACexit(-1) + if not user: + user = 'unknown' + + notificationClient = NotificationClient() + for host in resultAll['Value']: + if not resultAll['Value'][host]['OK']: + # If the host cannot be contacted, exclude it and send message + excludedHosts.append(host) + + result = notificationClient.sendMail( + Operations().getValue( + 'EMail/Production', + []), + 'Unreachable host', + '\ndirac-populate-component-db: Could not fill the database with the components from unreachable host %s\n' % + host) + if not result['OK']: + gLogger.error('Can not send unreachable host notification mail: %s' % result['Message']) + + if not resultAll['OK']: + gLogger.error(resultAll['Message']) + DIRACexit(-1) + resultHosts = client.getHostInfo() + if not resultHosts['OK']: + gLogger.error(resultHosts['Message']) + DIRACexit(-1) + resultInfo = client.getInfo() + if not resultInfo['OK']: + gLogger.error(resultInfo['Message']) + DIRACexit(-1) + resultMySQL = client.getMySQLStatus() + if not resultMySQL['OK']: + gLogger.error(resultMySQL['Message']) + DIRACexit(-1) + resultAllDB = client.getDatabases() + if not resultAllDB['OK']: + gLogger.error(resultAllDB['Message']) + DIRACexit(-1) + resultAvailableDB = client.getAvailableDatabases() + if not resultAvailableDB['OK']: + gLogger.error(resultAvailableDB['Message']) + DIRACexit(-1) + + records = [] + finalSet = list(set(resultAll['Value']) - set(excludedHosts)) + for host in finalSet: + hasMySQL = True + result = resultAll['Value'][host] + hostResult = resultHosts['Value'][host] + infoResult = resultInfo['Value'][host] + mySQLResult = resultMySQL['Value'][host] + allDBResult = resultAllDB['Value'][host] + availableDBResult = resultAvailableDB['Value'][host] - result = notificationClient.sendMail( - Operations().getValue( - 'EMail/Production', - []), - 'Unreachable host', - '\ndirac-populate-component-db: Could not fill the database with the components from unreachable host %s\n' % - host) if not result['OK']: - gLogger.error('Can not send unreachable host notification mail: %s' % result['Message']) - -if not resultAll['OK']: - gLogger.error(resultAll['Message']) - DIRACexit(-1) -resultHosts = client.getHostInfo() -if not resultHosts['OK']: - gLogger.error(resultHosts['Message']) - DIRACexit(-1) -resultInfo = client.getInfo() -if not resultInfo['OK']: - gLogger.error(resultInfo['Message']) - DIRACexit(-1) -resultMySQL = client.getMySQLStatus() -if not resultMySQL['OK']: - gLogger.error(resultMySQL['Message']) - DIRACexit(-1) -resultAllDB = client.getDatabases() -if not resultAllDB['OK']: - gLogger.error(resultAllDB['Message']) - DIRACexit(-1) -resultAvailableDB = client.getAvailableDatabases() -if not resultAvailableDB['OK']: - gLogger.error(resultAvailableDB['Message']) - DIRACexit(-1) - -records = [] -finalSet = list(set(resultAll['Value']) - set(excludedHosts)) -for host in finalSet: - hasMySQL = True - result = resultAll['Value'][host] - hostResult = resultHosts['Value'][host] - infoResult = resultInfo['Value'][host] - mySQLResult = resultMySQL['Value'][host] - allDBResult = resultAllDB['Value'][host] - availableDBResult = resultAvailableDB['Value'][host] - - if not result['OK']: - gLogger.error('Host %s: %s' % (host, result['Message'])) - continue - if not hostResult['OK']: - gLogger.error('Host %s: %s' % (host, hostResult['Message'])) - continue - if not infoResult['OK']: - gLogger.error('Host %s: %s' % (host, infoResult['Message'])) - continue - if mySQLResult['OK']: - if not allDBResult['OK']: - gLogger.error('Host %s: %s' % (host, allDBResult['Message'])) + gLogger.error('Host %s: %s' % (host, result['Message'])) continue - if not availableDBResult['OK']: - gLogger.error('Host %s: %s' % (host, availableDBResult['Message'])) + if not hostResult['OK']: + gLogger.error('Host %s: %s' % (host, hostResult['Message'])) continue - else: - hasMySQL = False - - setup = infoResult['Value']['Setup'] - if setup != mySetup: - continue - - cpu = hostResult['Value']['CPUModel'].strip() - rDict = result['Value'] - # Components other than databases - for compType in rDict: - if componentType and componentType != compType: + if not infoResult['OK']: + gLogger.error('Host %s: %s' % (host, infoResult['Message'])) + continue + if mySQLResult['OK']: + if not allDBResult['OK']: + gLogger.error('Host %s: %s' % (host, allDBResult['Message'])) + continue + if not availableDBResult['OK']: + gLogger.error('Host %s: %s' % (host, availableDBResult['Message'])) + continue + else: + hasMySQL = False + + setup = infoResult['Value']['Setup'] + if setup != mySetup: continue - for system in rDict[compType]: - components = sorted(rDict[compType][system]) - for component in components: - record = {'Installation': {}, 'Component': {}, 'Host': {}} - if rDict[compType][system][component]['Installed'] and \ - component != 'ComponentMonitoring': - runitStatus = \ - str(rDict[compType][system][component]['RunitStatus']) - if runitStatus != 'Unknown': - module = \ - str(rDict[compType][system][component]['Module']) - record['Component']['System'] = system - record['Component']['Module'] = module - # Transform 'Services' into 'service', 'Agents' into 'agent' ... - record['Component']['Type'] = compType.lower()[:-1] + + cpu = hostResult['Value']['CPUModel'].strip() + rDict = result['Value'] + # Components other than databases + for compType in rDict: + if componentType and componentType != compType: + continue + for system in rDict[compType]: + components = sorted(rDict[compType][system]) + for component in components: + record = {'Installation': {}, 'Component': {}, 'Host': {}} + if rDict[compType][system][component]['Installed'] and \ + component != 'ComponentMonitoring': + runitStatus = \ + str(rDict[compType][system][component]['RunitStatus']) + if runitStatus != 'Unknown': + module = \ + str(rDict[compType][system][component]['Module']) + record['Component']['System'] = system + record['Component']['Module'] = module + # Transform 'Services' into 'service', 'Agents' into 'agent' ... + record['Component']['Type'] = compType.lower()[:-1] + record['Host']['HostName'] = host + record['Host']['CPU'] = cpu + record['Installation']['Instance'] = component + record['Installation']['InstallationTime'] = datetime.utcnow() + record['Installation']['InstalledBy'] = user + records.append(record) + + # Databases + csClient = CSAPI() + cfg = csClient.getCurrentCFG()['Value'] + + if hasMySQL: + allDB = allDBResult['Value'] + availableDB = availableDBResult['Value'] + + for db in allDB: + # Check for DIRAC only databases + if db in availableDB and db != 'InstalledComponentsDB': + # Check for 'installed' databases + isSection = cfg.isSection('Systems/' + availableDB[db]['System'] + '/' + + cfg.getOption('DIRAC/Setups/' + setup + '/' + + availableDB[db]['System']) + '/Databases/' + db + + '/') + if isSection: + record = {'Installation': {}, 'Component': {}, 'Host': {}} + record['Component']['System'] = availableDB[db]['System'] + record['Component']['Module'] = db + record['Component']['Type'] = 'DB' record['Host']['HostName'] = host record['Host']['CPU'] = cpu - record['Installation']['Instance'] = component + record['Installation']['Instance'] = db record['Installation']['InstallationTime'] = datetime.utcnow() record['Installation']['InstalledBy'] = user records.append(record) - # Databases - csClient = CSAPI() - cfg = csClient.getCurrentCFG()['Value'] - - if hasMySQL: - allDB = allDBResult['Value'] - availableDB = availableDBResult['Value'] - - for db in allDB: - # Check for DIRAC only databases - if db in availableDB and db != 'InstalledComponentsDB': - # Check for 'installed' databases - isSection = cfg.isSection('Systems/' + availableDB[db]['System'] + '/' + - cfg.getOption('DIRAC/Setups/' + setup + '/' + - availableDB[db]['System']) + '/Databases/' + db + - '/') - if isSection: - record = {'Installation': {}, 'Component': {}, 'Host': {}} - record['Component']['System'] = availableDB[db]['System'] - record['Component']['Module'] = db - record['Component']['Type'] = 'DB' - record['Host']['HostName'] = host - record['Host']['CPU'] = cpu - record['Installation']['Instance'] = db - record['Installation']['InstallationTime'] = datetime.utcnow() - record['Installation']['InstalledBy'] = user - records.append(record) - -monitoringClient = ComponentMonitoringClient() - -# Add the installations to the database -for record in records: - result = MonitoringUtilities.monitorInstallation( - record['Component']['Type'], record['Component']['System'], - record['Installation']['Instance'], record['Component']['Module'], - record['Host']['CPU'], record['Host']['HostName']) - if not result['OK']: - gLogger.error(result['Message']) + monitoringClient = ComponentMonitoringClient() + + # Add the installations to the database + for record in records: + result = MonitoringUtilities.monitorInstallation( + record['Component']['Type'], record['Component']['System'], + record['Installation']['Instance'], record['Component']['Module'], + record['Host']['CPU'], record['Host']['HostName']) + if not result['OK']: + gLogger.error(result['Message']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_destroy.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_destroy.py index 50239d8b114..8b109a63cbb 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_destroy.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_destroy.py @@ -13,6 +13,7 @@ from DIRAC import gLogger, S_OK from DIRAC.Core.Security import Locations from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager @@ -126,7 +127,7 @@ def deleteLocalProxy(proxyLoc): gLogger.notice('Local proxy deleted.') -def main(): +def run(): """ main program entry point """ @@ -180,10 +181,15 @@ def main(): return 0 -if __name__ == "__main__": +@DIRACScript() +def main(): try: - retval = main() + retval = run() DIRAC.exit(retval) except RuntimeError as rtError: gLogger.error('Operation failed: %s' % str(rtError)) DIRAC.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_get_uploaded_info.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_get_uploaded_info.py index 68401aacd5f..3a68a55b366 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_get_uploaded_info.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_get_uploaded_info.py @@ -11,6 +11,7 @@ from DIRAC import gLogger, S_OK from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient from DIRAC.Core.Security import Properties from DIRAC.Core.Security.ProxyInfo import getProxyInfo @@ -33,68 +34,74 @@ def setUser(arg): return S_OK() -Script.registerSwitch("u:", "user=", "User to query (by default oneself)", setUser) +@DIRACScript() +def main(): + global userName + Script.registerSwitch("u:", "user=", "User to query (by default oneself)", setUser) -Script.parseCommandLine() + Script.parseCommandLine() -result = getProxyInfo() -if not result['OK']: - gLogger.notice("Do you have a valid proxy?") - gLogger.notice(result['Message']) - sys.exit(1) -proxyProps = result['Value'] + result = getProxyInfo() + if not result['OK']: + gLogger.notice("Do you have a valid proxy?") + gLogger.notice(result['Message']) + sys.exit(1) + proxyProps = result['Value'] -userName = userName or proxyProps.get('username') -if not userName: - gLogger.notice("Your proxy don`t have username extension") - sys.exit(1) + userName = userName or proxyProps.get('username') + if not userName: + gLogger.notice("Your proxy don`t have username extension") + sys.exit(1) -if userName in Registry.getAllUsers(): - if Properties.PROXY_MANAGEMENT not in proxyProps['groupProperties']: - if userName != proxyProps['username'] and userName != proxyProps['issuer']: - gLogger.notice("You can only query info about yourself!") + if userName in Registry.getAllUsers(): + if Properties.PROXY_MANAGEMENT not in proxyProps['groupProperties']: + if userName != proxyProps['username'] and userName != proxyProps['issuer']: + gLogger.notice("You can only query info about yourself!") + sys.exit(1) + result = Registry.getDNForUsername(userName) + if not result['OK']: + gLogger.notice("Oops %s" % result['Message']) + dnList = result['Value'] + if not dnList: + gLogger.notice("User %s has no DN defined!" % userName) sys.exit(1) - result = Registry.getDNForUsername(userName) + userDNs = dnList + else: + userDNs = [userName] + + gLogger.notice("Checking for DNs %s" % " | ".join(userDNs)) + pmc = ProxyManagerClient() + result = pmc.getDBContents({'UserDN': userDNs}) if not result['OK']: - gLogger.notice("Oops %s" % result['Message']) - dnList = result['Value'] - if not dnList: - gLogger.notice("User %s has no DN defined!" % userName) + gLogger.notice("Could not retrieve the proxy list: %s" % result['Value']) sys.exit(1) - userDNs = dnList -else: - userDNs = [userName] - - -gLogger.notice("Checking for DNs %s" % " | ".join(userDNs)) -pmc = ProxyManagerClient() -result = pmc.getDBContents({'UserDN': userDNs}) -if not result['OK']: - gLogger.notice("Could not retrieve the proxy list: %s" % result['Value']) - sys.exit(1) - -data = result['Value'] -colLengths = [] -for pN in data['ParameterNames']: - colLengths.append(len(pN)) -for row in data['Records']: - for i in range(len(row)): - colLengths[i] = max(colLengths[i], len(str(row[i]))) - -lines = [""] -for i in range(len(data['ParameterNames'])): - pN = data['ParameterNames'][i] - lines[0] += "| %s " % pN.ljust(colLengths[i]) -lines[0] += "|" -tL = len(lines[0]) -lines.insert(0, "-" * tL) -lines.append("-" * tL) -for row in data['Records']: - nL = "" - for i in range(len(row)): - nL += "| %s " % str(row[i]).ljust(colLengths[i]) - nL += "|" - lines.append(nL) + + data = result['Value'] + colLengths = [] + for pN in data['ParameterNames']: + colLengths.append(len(pN)) + for row in data['Records']: + for i in range(len(row)): + colLengths[i] = max(colLengths[i], len(str(row[i]))) + + lines = [""] + for i in range(len(data['ParameterNames'])): + pN = data['ParameterNames'][i] + lines[0] += "| %s " % pN.ljust(colLengths[i]) + lines[0] += "|" + tL = len(lines[0]) + lines.insert(0, "-" * tL) lines.append("-" * tL) + for row in data['Records']: + nL = "" + for i in range(len(row)): + nL += "| %s " % str(row[i]).ljust(colLengths[i]) + nL += "|" + lines.append(nL) + lines.append("-" * tL) + + gLogger.notice("\n".join(lines)) + -gLogger.notice("\n".join(lines)) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_restart_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_restart_component.py index 20ab5a0bad4..ba4e2fa16d7 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_restart_component.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_restart_component.py @@ -8,33 +8,42 @@ __RCSID__ = "$Id$" # from DIRAC.Core.Base import Script -Script.disableCS() -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... [System [Service|Agent]]' % Script.scriptName, - 'Arguments:', - ' System: Name of the system for the component (default *: all)', - ' Service|Agent: Name of the particular component (default *: all)'])) -Script.parseCommandLine() -args = Script.getPositionalArgs() -if len(args) > 2: - Script.showHelp(exitCode=1) - -system = '*' -component = '*' -if args: - system = args[0] -if system != '*': - if len(args) > 1: - component = args[1] -# -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller -# -gComponentInstaller.exitOnError = True -# -result = gComponentInstaller.runsvctrlComponent(system, component, 't') -if not result['OK']: - print('ERROR:', result['Message']) - exit(-1) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.disableCS() + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... [System [Service|Agent]]' % Script.scriptName, + 'Arguments:', + ' System: Name of the system for the component (default *: all)', + ' Service|Agent: Name of the particular component (default *: all)'])) + Script.parseCommandLine() + args = Script.getPositionalArgs() + if len(args) > 2: + Script.showHelp(exitCode=1) + + system = '*' + component = '*' + if args: + system = args[0] + if system != '*': + if len(args) > 1: + component = args[1] + + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + + gComponentInstaller.exitOnError = True + + result = gComponentInstaller.runsvctrlComponent(system, component, 't') + if not result['OK']: + print('ERROR:', result['Message']) + exit(-1) + + gComponentInstaller.printStartupStatus(result['Value']) + -gComponentInstaller.printStartupStatus(result['Value']) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_start_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_start_component.py index 10276533821..5a3e03ecafe 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_start_component.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_start_component.py @@ -9,33 +9,42 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script -Script.disableCS() -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... [system [service|agent]]' % Script.scriptName, - 'Arguments:', - ' system: Name of the system for the component (default *: all)', - ' service|agent: Name of the particular component (default *: all)'])) -Script.parseCommandLine() -args = Script.getPositionalArgs() -if len(args) > 2: - Script.showHelp(exitCode=1) - -system = '*' -component = '*' -if len(args) > 0: - system = args[0] -if system != '*': - if len(args) > 1: - component = args[1] -# -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller -# -gComponentInstaller.exitOnError = True -# -result = gComponentInstaller.runsvctrlComponent(system, component, 'u') -if not result['OK']: - print('ERROR:', result['Message']) - exit(-1) - -gComponentInstaller.printStartupStatus(result['Value']) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.disableCS() + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... [system [service|agent]]' % Script.scriptName, + 'Arguments:', + ' system: Name of the system for the component (default *: all)', + ' service|agent: Name of the particular component (default *: all)'])) + Script.parseCommandLine() + args = Script.getPositionalArgs() + if len(args) > 2: + Script.showHelp(exitCode=1) + + system = '*' + component = '*' + if len(args) > 0: + system = args[0] + if system != '*': + if len(args) > 1: + component = args[1] + # + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + # + gComponentInstaller.exitOnError = True + # + result = gComponentInstaller.runsvctrlComponent(system, component, 'u') + if not result['OK']: + print('ERROR:', result['Message']) + exit(-1) + + gComponentInstaller.printStartupStatus(result['Value']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_status_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_status_component.py index 969a4ee4edf..d202db7329e 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_status_component.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_status_component.py @@ -8,38 +8,46 @@ from __future__ import division from DIRAC.Core.Base import Script -Script.disableCS() +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... [system [service|agent]]' % Script.scriptName, - 'Arguments:', - ' system: Name of the system for the component (default *: all)', - ' service|agent: Name of the particular component (default *: all)'])) -Script.parseCommandLine() -args = Script.getPositionalArgs() -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller +@DIRACScript() +def main(): + Script.disableCS() + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... [system [service|agent]]' % Script.scriptName, + 'Arguments:', + ' system: Name of the system for the component (default *: all)', + ' service|agent: Name of the particular component (default *: all)'])) + Script.parseCommandLine() + args = Script.getPositionalArgs() -__RCSID__ = "$Id$" + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller -if len(args) > 2: - Script.showHelp(exitCode=1) + __RCSID__ = "$Id$" + + if len(args) > 2: + Script.showHelp(exitCode=1) + + system = '*' + component = '*' + if len(args) > 0: + system = args[0] + if system != '*': + if len(args) > 1: + component = args[1] + # + gComponentInstaller.exitOnError = True + # + result = gComponentInstaller.getStartupComponentStatus([system, component]) + if not result['OK']: + print('ERROR:', result['Message']) + exit(-1) + + gComponentInstaller.printStartupStatus(result['Value']) -system = '*' -component = '*' -if len(args) > 0: - system = args[0] -if system != '*': - if len(args) > 1: - component = args[1] -# -gComponentInstaller.exitOnError = True -# -result = gComponentInstaller.getStartupComponentStatus([system, component]) -if not result['OK']: - print('ERROR:', result['Message']) - exit(-1) -gComponentInstaller.printStartupStatus(result['Value']) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_stop_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_stop_component.py index 55f2f0a27ec..a4992d27614 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_stop_component.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_stop_component.py @@ -6,37 +6,46 @@ from __future__ import absolute_import from __future__ import division from DIRAC.Core.Base import Script -Script.disableCS() -Script.setUsageMessage('\n'.join(['Stop DIRAC component using runsvctrl utility', - 'Usage:', - ' %s [option|cfgfile] ... [system [service|agent]]' % Script.scriptName, - 'Arguments:', - ' system: Name of the system for the component (default *: all)', - ' service|agent: Name of the particular component (default *: all)'])) -Script.parseCommandLine() -args = Script.getPositionalArgs() - -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller - -__RCSID__ = "$Id$" - -if len(args) > 2: - Script.showHelp(exitCode=1) - -system = '*' -component = '*' -if len(args) > 0: - system = args[0] -if system != '*': - if len(args) > 1: - component = args[1] -# -# -gComponentInstaller.exitOnError = True -# -result = gComponentInstaller.runsvctrlComponent(system, component, 'd') -if not result['OK']: - print('ERROR:', result['Message']) - exit(-1) - -gComponentInstaller.printStartupStatus(result['Value']) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.disableCS() + Script.setUsageMessage('\n'.join(['Stop DIRAC component using runsvctrl utility', + 'Usage:', + ' %s [option|cfgfile] ... [system [service|agent]]' % Script.scriptName, + 'Arguments:', + ' system: Name of the system for the component (default *: all)', + ' service|agent: Name of the particular component (default *: all)'])) + Script.parseCommandLine() + args = Script.getPositionalArgs() + + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + + __RCSID__ = "$Id$" + + if len(args) > 2: + Script.showHelp(exitCode=1) + + system = '*' + component = '*' + if len(args) > 0: + system = args[0] + if system != '*': + if len(args) > 1: + component = args[1] + # + # + gComponentInstaller.exitOnError = True + # + result = gComponentInstaller.runsvctrlComponent(system, component, 'd') + if not result['OK']: + print('ERROR:', result['Message']) + exit(-1) + + gComponentInstaller.printStartupStatus(result['Value']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_sys_sendmail.py b/src/DIRAC/FrameworkSystem/scripts/dirac_sys_sendmail.py index dd08dfab72c..f0f99c2b84c 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_sys_sendmail.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_sys_sendmail.py @@ -33,58 +33,66 @@ from DIRAC import gLogger, exit as DIRACexit from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient -Script.setUsageMessage(''.join(__doc__)) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() +@DIRACScript() +def main(): + Script.setUsageMessage(''.join(__doc__)) -arg = "".join(args) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -if not len(arg) > 0: - gLogger.error("Missing argument") - DIRACexit(2) + arg = "".join(args) -try: - head, body = arg.split("\\n\\n") -except Exception as x: - head = "To: %s" % arg - body = sys.stdin.read() + if not len(arg) > 0: + gLogger.error("Missing argument") + DIRACexit(2) -try: - tmp, body = body.split("\\n\\n") - head = tmp + "\\n" + head -except Exception as x: - pass + try: + head, body = arg.split("\\n\\n") + except Exception as x: + head = "To: %s" % arg + body = sys.stdin.read() -body = "".join(body.strip()) + try: + tmp, body = body.split("\\n\\n") + head = tmp + "\\n" + head + except Exception as x: + pass -try: - headers = dict((i.strip(), j.strip()) for i, j in - (item.split(':') for item in head.split('\\n'))) -except BaseException: - gLogger.error("Failed to convert string: %s to email headers" % head) - DIRACexit(4) + body = "".join(body.strip()) -if "To" not in headers: - gLogger.error("Failed to get 'To:' field from headers %s" % head) - DIRACexit(5) -to = headers["To"] + try: + headers = dict((i.strip(), j.strip()) for i, j in + (item.split(':') for item in head.split('\\n'))) + except BaseException: + gLogger.error("Failed to convert string: %s to email headers" % head) + DIRACexit(4) -origin = "%s@%s" % (os.getenv("LOGNAME", "dirac"), socket.getfqdn()) -if "From" in headers: - origin = headers["From"] + if "To" not in headers: + gLogger.error("Failed to get 'To:' field from headers %s" % head) + DIRACexit(5) + to = headers["To"] -subject = "Sent from %s" % socket.getfqdn() -if "Subject" in headers: - subject = headers["Subject"] + origin = "%s@%s" % (os.getenv("LOGNAME", "dirac"), socket.getfqdn()) + if "From" in headers: + origin = headers["From"] -ntc = NotificationClient() -print("sendMail(%s,%s,%s,%s,%s)" % (to, subject, body, origin, False)) -result = ntc.sendMail(to, subject, body, origin, localAttempt=False) -if not result["OK"]: - gLogger.error(result["Message"]) - DIRACexit(6) + subject = "Sent from %s" % socket.getfqdn() + if "Subject" in headers: + subject = headers["Subject"] -DIRACexit(0) + ntc = NotificationClient() + print("sendMail(%s,%s,%s,%s,%s)" % (to, subject, body, origin, False)) + result = ntc.sendMail(to, subject, body, origin, localAttempt=False) + if not result["OK"]: + gLogger.error(result["Message"]) + DIRACexit(6) + + DIRACexit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_uninstall_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_uninstall_component.py index 144667a7069..c4d2b4ceb81 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_uninstall_component.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_uninstall_component.py @@ -12,6 +12,7 @@ from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities from DIRAC import gLogger, S_OK from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Utilities.PromptUser import promptUser from DIRAC import exit as DIRACexit from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller @@ -29,60 +30,67 @@ def setForce(opVal): return S_OK() -Script.registerSwitch("f", "force", "Forces the removal of the logs", setForce) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, - 'Arguments:', - ' System: Name of the DIRAC system (ie: WorkloadManagement)', - ' Component: Name of the DIRAC component (ie: Matcher)'])) - -Script.parseCommandLine() -args = Script.getPositionalArgs() - -if len(args) == 1: - args = args[0].split('/') - -if len(args) < 2: - Script.showHelp(exitCode=1) - -system = args[0] -component = args[1] - -monitoringClient = ComponentMonitoringClient() -result = monitoringClient.getInstallations({'Instance': component, 'UnInstallationTime': None}, - {'System': system}, - {'HostName': socket.getfqdn()}, True) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) -if len(result['Value']) < 1: - gLogger.warn('Given component does not exist') - DIRACexit(1) -if len(result['Value']) > 1: - gLogger.error('Too many components match') - DIRACexit(1) - -removeLogs = False -if force: - removeLogs = True -else: - if result['Value'][0]['Component']['Type'] in gComponentInstaller.componentTypes: - result = promptUser('Remove logs?', ['y', 'n'], 'n') - if result['OK']: - removeLogs = result['Value'] == 'y' - else: - gLogger.error(result['Message']) - DIRACexit(1) - -result = gComponentInstaller.uninstallComponent(system, component, removeLogs) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - -result = MonitoringUtilities.monitorUninstallation(system, component) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) -gLogger.notice('Successfully uninstalled component %s/%s' % (system, component)) -DIRACexit() +@DIRACScript() +def main(): + global force + Script.registerSwitch("f", "force", "Forces the removal of the logs", setForce) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, + 'Arguments:', + ' System: Name of the DIRAC system (ie: WorkloadManagement)', + ' Component: Name of the DIRAC component (ie: Matcher)'])) + + Script.parseCommandLine() + args = Script.getPositionalArgs() + + if len(args) == 1: + args = args[0].split('/') + + if len(args) < 2: + Script.showHelp(exitCode=1) + + system = args[0] + component = args[1] + + monitoringClient = ComponentMonitoringClient() + result = monitoringClient.getInstallations({'Instance': component, 'UnInstallationTime': None}, + {'System': system}, + {'HostName': socket.getfqdn()}, True) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + if len(result['Value']) < 1: + gLogger.warn('Given component does not exist') + DIRACexit(1) + if len(result['Value']) > 1: + gLogger.error('Too many components match') + DIRACexit(1) + + removeLogs = False + if force: + removeLogs = True + else: + if result['Value'][0]['Component']['Type'] in gComponentInstaller.componentTypes: + result = promptUser('Remove logs?', ['y', 'n'], 'n') + if result['OK']: + removeLogs = result['Value'] == 'y' + else: + gLogger.error(result['Message']) + DIRACexit(1) + + result = gComponentInstaller.uninstallComponent(system, component, removeLogs) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + result = MonitoringUtilities.monitorUninstallation(system, component) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + gLogger.notice('Successfully uninstalled component %s/%s' % (system, component)) + DIRACexit() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_add_group.py b/src/DIRAC/Interfaces/scripts/dirac_admin_add_group.py index a97a8bce590..7fde384cb34 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_add_group.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_add_group.py @@ -9,6 +9,7 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript groupName = None groupProperties = [] @@ -38,64 +39,73 @@ def addProperty(arg): groupProperties.append(arg) -Script.setUsageMessage('\n'.join(['Add or Modify a Group info in DIRAC', - '\nUsage:\n', - ' %s [option|cfgfile] ... Property= ...' % Script.scriptName, - '\nArguments:\n', - ' Property=: Other properties to be added to the User like (VOMSRole=XXXX)', - ])) - -Script.registerSwitch('G:', 'GroupName:', 'Name of the Group (Mandatory)', setGroupName) -Script.registerSwitch( - 'U:', - 'UserName:', - 'Short Name of user to be added to the Group (Allow Multiple instances or None)', - addUserName) -Script.registerSwitch( - 'P:', - 'Property:', - 'Property to be added to the Group (Allow Multiple instances or None)', - addProperty) - -Script.parseCommandLine(ignoreErrors=True) - -if groupName is None: - Script.showHelp(exitCode=1) - -args = Script.getPositionalArgs() - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -groupProps = {} -if userNames: - groupProps['Users'] = ', '.join(userNames) -if groupProperties: - groupProps['Properties'] = ', '.join(groupProperties) - -for prop in args: - pl = prop.split("=") - if len(pl) < 2: - errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) +@DIRACScript() +def main(): + global groupName + global groupProperties + global userNames + Script.setUsageMessage('\n'.join(['Add or Modify a Group info in DIRAC', + '\nUsage:\n', + ' %s [option|cfgfile] ... Property= ...' % Script.scriptName, + '\nArguments:\n', + ' Property=: Other properties to be added to the User like (VOMSRole=XXXX)', + ])) + + Script.registerSwitch('G:', 'GroupName:', 'Name of the Group (Mandatory)', setGroupName) + Script.registerSwitch( + 'U:', + 'UserName:', + 'Short Name of user to be added to the Group (Allow Multiple instances or None)', + addUserName) + Script.registerSwitch( + 'P:', + 'Property:', + 'Property to be added to the Group (Allow Multiple instances or None)', + addProperty) + + Script.parseCommandLine(ignoreErrors=True) + + if groupName is None: + Script.showHelp(exitCode=1) + + args = Script.getPositionalArgs() + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + groupProps = {} + if userNames: + groupProps['Users'] = ', '.join(userNames) + if groupProperties: + groupProps['Properties'] = ', '.join(groupProperties) + + for prop in args: + pl = prop.split("=") + if len(pl) < 2: + errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) + exitCode = 255 + else: + pName = pl[0] + pValue = "=".join(pl[1:]) + Script.gLogger.info("Setting property %s to %s" % (pName, pValue)) + groupProps[pName] = pValue + + if not diracAdmin.csModifyGroup(groupName, groupProps, createIfNonExistant=True)['OK']: + errorList.append(("add group", "Cannot register group %s" % groupName)) exitCode = 255 else: - pName = pl[0] - pValue = "=".join(pl[1:]) - Script.gLogger.info("Setting property %s to %s" % (pName, pValue)) - groupProps[pName] = pValue - -if not diracAdmin.csModifyGroup(groupName, groupProps, createIfNonExistant=True)['OK']: - errorList.append(("add group", "Cannot register group %s" % groupName)) - exitCode = 255 -else: - result = diracAdmin.csCommitChanges() - if not result['OK']: - errorList.append(("commit", result['Message'])) - exitCode = 255 + result = diracAdmin.csCommitChanges() + if not result['OK']: + errorList.append(("commit", result['Message'])) + exitCode = 255 + + for error in errorList: + Script.gLogger.error("%s: %s" % error) + + DIRAC.exit(exitCode) -for error in errorList: - Script.gLogger.error("%s: %s" % error) -DIRAC.exit(exitCode) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_add_host.py b/src/DIRAC/Interfaces/scripts/dirac_admin_add_host.py index 611913b4ebe..8b989ab1a0d 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_add_host.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_add_host.py @@ -5,6 +5,7 @@ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript hostName = None hostDN = None @@ -33,71 +34,81 @@ def addProperty(arg): hostProperties.append(arg) -Script.setUsageMessage('\n'.join(['Add or Modify a Host info in DIRAC', - '\nUsage:\n', - ' %s [option|cfgfile] ... Property= ...' % Script.scriptName, - '\nArguments:\n', - ' Property=: Other properties to be added to the User like (Responsible=XXX)', - ])) - -Script.registerSwitch('H:', 'HostName:', 'Name of the Host (Mandatory)', setHostName) -Script.registerSwitch('D:', 'HostDN:', 'DN of the Host Certificate (Mandatory)', setHostDN) -Script.registerSwitch( - 'P:', - 'Property:', - 'Property to be added to the Host (Allow Multiple instances or None)', - addProperty) - -Script.parseCommandLine(ignoreErrors=True) - -if hostName is None or hostDN is None: - Script.showHelp(exitCode=1) - -args = Script.getPositionalArgs() - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -hostProps = {'DN': hostDN} -if hostProperties: - hostProps['Properties'] = ', '.join(hostProperties) +@DIRACScript() +def main(): + global hostName + global hostDN + global hostProperties + Script.setUsageMessage('\n'.join([ + 'Add or Modify a Host info in DIRAC', + '\nUsage:\n', + ' %s [option|cfgfile] ... Property= ...' % Script.scriptName, + '\nArguments:\n', + ' Property=: Other properties to be added to the User like (Responsible=XXX)', + ])) + + Script.registerSwitch('H:', 'HostName:', 'Name of the Host (Mandatory)', setHostName) + Script.registerSwitch('D:', 'HostDN:', 'DN of the Host Certificate (Mandatory)', setHostDN) + Script.registerSwitch( + 'P:', + 'Property:', + 'Property to be added to the Host (Allow Multiple instances or None)', + addProperty) + + Script.parseCommandLine(ignoreErrors=True) + + if hostName is None or hostDN is None: + Script.showHelp(exitCode=1) -for prop in args: - pl = prop.split("=") - if len(pl) < 2: - errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) + args = Script.getPositionalArgs() + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + hostProps = {'DN': hostDN} + if hostProperties: + hostProps['Properties'] = ', '.join(hostProperties) + + for prop in args: + pl = prop.split("=") + if len(pl) < 2: + errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) + exitCode = 255 + else: + pName = pl[0] + pValue = "=".join(pl[1:]) + Script.gLogger.info("Setting property %s to %s" % (pName, pValue)) + hostProps[pName] = pValue + + if not diracAdmin.csModifyHost(hostName, hostProps, createIfNonExistant=True)['OK']: + errorList.append(("add host", "Cannot register host %s" % hostName)) exitCode = 255 else: - pName = pl[0] - pValue = "=".join(pl[1:]) - Script.gLogger.info("Setting property %s to %s" % (pName, pValue)) - hostProps[pName] = pValue - -if not diracAdmin.csModifyHost(hostName, hostProps, createIfNonExistant=True)['OK']: - errorList.append(("add host", "Cannot register host %s" % hostName)) - exitCode = 255 -else: - result = diracAdmin.csCommitChanges() - if not result['OK']: - errorList.append(("commit", result['Message'])) - exitCode = 255 - -if exitCode == 0: - from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient - cmc = ComponentMonitoringClient() - ret = cmc.hostExists(dict(HostName=hostName)) - if not ret['OK']: - Script.gLogger.error('Cannot check if host is registered in ComponentMonitoring', ret['Message']) - elif ret['Value']: - Script.gLogger.info('Host already registered in ComponentMonitoring') - else: - ret = cmc.addHost(dict(HostName=hostName, CPU='TO_COME')) + result = diracAdmin.csCommitChanges() + if not result['OK']: + errorList.append(("commit", result['Message'])) + exitCode = 255 + + if exitCode == 0: + from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient + cmc = ComponentMonitoringClient() + ret = cmc.hostExists(dict(HostName=hostName)) if not ret['OK']: - Script.gLogger.error('Failed to add Host to ComponentMonitoring', ret['Message']) + Script.gLogger.error('Cannot check if host is registered in ComponentMonitoring', ret['Message']) + elif ret['Value']: + Script.gLogger.info('Host already registered in ComponentMonitoring') + else: + ret = cmc.addHost(dict(HostName=hostName, CPU='TO_COME')) + if not ret['OK']: + Script.gLogger.error('Failed to add Host to ComponentMonitoring', ret['Message']) + + for error in errorList: + Script.gLogger.error("%s: %s" % error) + + DIRAC.exit(exitCode) -for error in errorList: - Script.gLogger.error("%s: %s" % error) -DIRAC.exit(exitCode) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_add_user.py b/src/DIRAC/Interfaces/scripts/dirac_admin_add_user.py index 83a3931e121..8c0ef418a22 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_add_user.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_add_user.py @@ -8,6 +8,7 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript userName = None userDN = None @@ -47,57 +48,67 @@ def addUserGroup(arg): userGroups.append(arg) -Script.setUsageMessage('\n'.join(['Add or Modify a User info in DIRAC', - 'Usage:', - ' %s [option|cfgfile] ... Property= ...' % Script.scriptName, - 'Arguments:', - ' Property=: Properties to be added to the User like (Phone=XXXX)', ])) - -Script.registerSwitch('N:', 'UserName:', 'Short Name of the User (Mandatory)', setUserName) -Script.registerSwitch('D:', 'UserDN:', 'DN of the User Certificate (Mandatory)', setUserDN) -Script.registerSwitch('M:', 'UserMail:', 'eMail of the user (Mandatory)', setUserMail) -Script.registerSwitch( - 'G:', - 'UserGroup:', - 'Name of the Group for the User (Allow Multiple instances or None)', - addUserGroup) - -Script.parseCommandLine(ignoreErrors=True) - -if userName is None or userDN is None or userMail is None: - Script.showHelp(exitCode=1) - -args = Script.getPositionalArgs() - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -userProps = {'DN': userDN, 'Email': userMail} -if userGroups: - userProps['Groups'] = userGroups -for prop in args: - pl = prop.split("=") - if len(pl) < 2: - errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) +@DIRACScript() +def main(): + global userName + global userDN + global userMail + global userGroups + Script.setUsageMessage('\n'.join(['Add or Modify a User info in DIRAC', + 'Usage:', + ' %s [option|cfgfile] ... Property= ...' % Script.scriptName, + 'Arguments:', + ' Property=: Properties to be added to the User like (Phone=XXXX)', ])) + + Script.registerSwitch('N:', 'UserName:', 'Short Name of the User (Mandatory)', setUserName) + Script.registerSwitch('D:', 'UserDN:', 'DN of the User Certificate (Mandatory)', setUserDN) + Script.registerSwitch('M:', 'UserMail:', 'eMail of the user (Mandatory)', setUserMail) + Script.registerSwitch( + 'G:', + 'UserGroup:', + 'Name of the Group for the User (Allow Multiple instances or None)', + addUserGroup) + + Script.parseCommandLine(ignoreErrors=True) + + if userName is None or userDN is None or userMail is None: + Script.showHelp(exitCode=1) + + args = Script.getPositionalArgs() + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + userProps = {'DN': userDN, 'Email': userMail} + if userGroups: + userProps['Groups'] = userGroups + for prop in args: + pl = prop.split("=") + if len(pl) < 2: + errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) + exitCode = 255 + else: + pName = pl[0] + pValue = "=".join(pl[1:]) + Script.gLogger.info("Setting property %s to %s" % (pName, pValue)) + userProps[pName] = pValue + + if not diracAdmin.csModifyUser(userName, userProps, createIfNonExistant=True)['OK']: + errorList.append(("add user", "Cannot register user %s" % userName)) exitCode = 255 else: - pName = pl[0] - pValue = "=".join(pl[1:]) - Script.gLogger.info("Setting property %s to %s" % (pName, pValue)) - userProps[pName] = pValue - -if not diracAdmin.csModifyUser(userName, userProps, createIfNonExistant=True)['OK']: - errorList.append(("add user", "Cannot register user %s" % userName)) - exitCode = 255 -else: - result = diracAdmin.csCommitChanges() - if not result['OK']: - errorList.append(("commit", result['Message'])) - exitCode = 255 + result = diracAdmin.csCommitChanges() + if not result['OK']: + errorList.append(("commit", result['Message'])) + exitCode = 255 + + for error in errorList: + Script.gLogger.error("%s: %s" % error) + + DIRAC.exit(exitCode) -for error in errorList: - Script.gLogger.error("%s: %s" % error) -DIRAC.exit(exitCode) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_allow_site.py b/src/DIRAC/Interfaces/scripts/dirac_admin_allow_site.py index c249925598d..1be789f6094 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_allow_site.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_allow_site.py @@ -15,81 +15,90 @@ import time from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch("E:", "email=", "Boolean True/False (True by default)") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Site Comment' % Script.scriptName, + 'Arguments:', + ' Site: Name of the Site', + ' Comment: Reason of the action'])) + Script.parseCommandLine(ignoreErrors=True) + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + from DIRAC import exit as DIRACExit, gConfig, gLogger + + def getBoolean(value): + if value.lower() == 'true': + return True + elif value.lower() == 'false': + return False + else: + Script.showHelp() -Script.registerSwitch("E:", "email=", "Boolean True/False (True by default)") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Site Comment' % Script.scriptName, - 'Arguments:', - ' Site: Name of the Site', - ' Comment: Reason of the action'])) -Script.parseCommandLine(ignoreErrors=True) - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC import exit as DIRACExit, gConfig, gLogger - - -def getBoolean(value): - if value.lower() == 'true': - return True - elif value.lower() == 'false': - return False - else: - Script.showHelp() + email = True + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "email": + email = getBoolean(switch[1]) + args = Script.getPositionalArgs() -email = True -for switch in Script.getUnprocessedSwitches(): - if switch[0] == "email": - email = getBoolean(switch[1]) + if len(args) < 2: + Script.showHelp() -args = Script.getPositionalArgs() + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + setup = gConfig.getValue('/DIRAC/Setup', '') + if not setup: + print('ERROR: Could not contact Configuration Service') + exitCode = 2 + DIRACExit(exitCode) + + # result = promptUser( + # 'All the elements that are associated with this site will be active, ' + # 'are you sure about this action?' + # ) + # if not result['OK'] or result['Value'] is 'n': + # print 'Script stopped' + # DIRACExit( 0 ) + + site = args[0] + comment = args[1] + result = diracAdmin.allowSite(site, comment, printOutput=True) + if not result['OK']: + errorList.append((site, result['Message'])) + exitCode = 2 + else: + if email: + userName = diracAdmin._getCurrentUser() + if not userName['OK']: + print('ERROR: Could not obtain current username from proxy') + exitCode = 2 + DIRACExit(exitCode) + userName = userName['Value'] + subject = '%s is added in site mask for %s setup' % (site, setup) + body = 'Site %s is added to the site mask for %s setup by %s on %s.\n\n' % (site, setup, userName, time.asctime()) + body += 'Comment:\n%s' % comment + addressPath = 'EMail/Production' + address = Operations().getValue(addressPath, '') + if not address: + gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) + else: + result = diracAdmin.sendMail(address, subject, body) + else: + print('Automatic email disabled by flag.') -if len(args) < 2: - Script.showHelp() + for error in errorList: + print("ERROR %s: %s" % error) -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] -setup = gConfig.getValue('/DIRAC/Setup', '') -if not setup: - print('ERROR: Could not contact Configuration Service') - exitCode = 2 DIRACExit(exitCode) -#result = promptUser( 'All the elements that are associated with this site will be active, are you sure about this action?' ) -# if not result['OK'] or result['Value'] is 'n': -# print 'Script stopped' -# DIRACExit( 0 ) - -site = args[0] -comment = args[1] -result = diracAdmin.allowSite(site, comment, printOutput=True) -if not result['OK']: - errorList.append((site, result['Message'])) - exitCode = 2 -else: - if email: - userName = diracAdmin._getCurrentUser() - if not userName['OK']: - print('ERROR: Could not obtain current username from proxy') - exitCode = 2 - DIRACExit(exitCode) - userName = userName['Value'] - subject = '%s is added in site mask for %s setup' % (site, setup) - body = 'Site %s is added to the site mask for %s setup by %s on %s.\n\n' % (site, setup, userName, time.asctime()) - body += 'Comment:\n%s' % comment - addressPath = 'EMail/Production' - address = Operations().getValue(addressPath, '') - if not address: - gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) - else: - result = diracAdmin.sendMail(address, subject, body) - else: - print('Automatic email disabled by flag.') - -for error in errorList: - print("ERROR %s: %s" % error) -DIRACExit(exitCode) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_ban_site.py b/src/DIRAC/Interfaces/scripts/dirac_admin_ban_site.py index abaf453bd0f..55217e3a78d 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_ban_site.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_ban_site.py @@ -16,82 +16,91 @@ from DIRAC import exit as DIRACExit, gConfig, gLogger from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch("E:", "email=", "Boolean True/False (True by default)") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Site Comment' % Script.scriptName, + 'Arguments:', + ' Site: Name of the Site', + ' Comment: Reason of the action'])) + Script.parseCommandLine(ignoreErrors=True) + + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + from DIRAC.Core.Utilities.PromptUser import promptUser + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + + def getBoolean(value): + if value.lower() == 'true': + return True + elif value.lower() == 'false': + return False + else: + Script.showHelp() -Script.registerSwitch("E:", "email=", "Boolean True/False (True by default)") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Site Comment' % Script.scriptName, - 'Arguments:', - ' Site: Name of the Site', - ' Comment: Reason of the action'])) -Script.parseCommandLine(ignoreErrors=True) - -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.Core.Utilities.PromptUser import promptUser -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin - - -def getBoolean(value): - if value.lower() == 'true': - return True - elif value.lower() == 'false': - return False - else: - Script.showHelp() + email = True + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "email": + email = getBoolean(switch[1]) + args = Script.getPositionalArgs() -email = True -for switch in Script.getUnprocessedSwitches(): - if switch[0] == "email": - email = getBoolean(switch[1]) + if len(args) < 2: + Script.showHelp() -args = Script.getPositionalArgs() + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + setup = gConfig.getValue('/DIRAC/Setup', '') + if not setup: + print('ERROR: Could not contact Configuration Service') + exitCode = 2 + DIRACExit(exitCode) + + # result = promptUser( + # 'All the elements that are associated with this site will be banned,' + # 'are you sure about this action?' + # ) + # if not result['OK'] or result['Value'] is 'n': + # print 'Script stopped' + # DIRACExit( 0 ) + + site = args[0] + comment = args[1] + result = diracAdmin.banSite(site, comment, printOutput=True) + if not result['OK']: + errorList.append((site, result['Message'])) + exitCode = 2 + else: + if email: + userName = diracAdmin._getCurrentUser() + if not userName['OK']: + print('ERROR: Could not obtain current username from proxy') + exitCode = 2 + DIRACExit(exitCode) + userName = userName['Value'] + subject = '%s is banned for %s setup' % (site, setup) + body = 'Site %s is removed from site mask for %s setup by %s on %s.\n\n' % (site, setup, userName, time.asctime()) + body += 'Comment:\n%s' % comment + + addressPath = 'EMail/Production' + address = Operations().getValue(addressPath, '') + if not address: + gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) + else: + result = diracAdmin.sendMail(address, subject, body) + else: + print('Automatic email disabled by flag.') -if len(args) < 2: - Script.showHelp() + for error in errorList: + print("ERROR %s: %s" % error) -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] -setup = gConfig.getValue('/DIRAC/Setup', '') -if not setup: - print('ERROR: Could not contact Configuration Service') - exitCode = 2 DIRACExit(exitCode) -#result = promptUser( 'All the elements that are associated with this site will be banned, are you sure about this action?' ) -# if not result['OK'] or result['Value'] is 'n': -# print 'Script stopped' -# DIRACExit( 0 ) - -site = args[0] -comment = args[1] -result = diracAdmin.banSite(site, comment, printOutput=True) -if not result['OK']: - errorList.append((site, result['Message'])) - exitCode = 2 -else: - if email: - userName = diracAdmin._getCurrentUser() - if not userName['OK']: - print('ERROR: Could not obtain current username from proxy') - exitCode = 2 - DIRACExit(exitCode) - userName = userName['Value'] - subject = '%s is banned for %s setup' % (site, setup) - body = 'Site %s is removed from site mask for %s setup by %s on %s.\n\n' % (site, setup, userName, time.asctime()) - body += 'Comment:\n%s' % comment - - addressPath = 'EMail/Production' - address = Operations().getValue(addressPath, '') - if not address: - gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) - else: - result = diracAdmin.sendMail(address, subject, body) - else: - print('Automatic email disabled by flag.') - -for error in errorList: - print("ERROR %s: %s" % error) -DIRACExit(exitCode) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_ce_info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_ce_info.py index c2138950e6a..3b094358a09 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_ce_info.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_ce_info.py @@ -13,30 +13,37 @@ from DIRAC import gConfig, gLogger, exit as Dexit from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... CE ...' % Script.scriptName, - 'Arguments:', - ' CE: Name of the CE (mandatory)'])) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... CE ...' % Script.scriptName, + 'Arguments:', + ' CE: Name of the CE (mandatory)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) + if len(args) < 1: + Script.showHelp(exitCode=1) -res = getCESiteMapping(args[0]) -if not res['OK']: - gLogger.error(res['Message']) - Dexit(1) -site = res['Value'][args[0]] + res = getCESiteMapping(args[0]) + if not res['OK']: + gLogger.error(res['Message']) + Dexit(1) + site = res['Value'][args[0]] -res = gConfig.getOptionsDict(cfgPath('Resources', 'Sites', site.split('.')[0], site, 'CEs', args[0])) -if not res['OK']: - gLogger.error(res['Message']) - Dexit(1) -gLogger.notice(res['Value']) + res = gConfig.getOptionsDict(cfgPath('Resources', 'Sites', site.split('.')[0], site, 'CEs', args[0])) + if not res['OK']: + gLogger.error(res['Message']) + Dexit(1) + gLogger.notice(res['Value']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_delete_user.py b/src/DIRAC/Interfaces/scripts/dirac_admin_delete_user.py index a778cb41bae..4818d9a1a47 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_delete_user.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_delete_user.py @@ -13,42 +13,50 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... User ...' % Script.scriptName, - 'Arguments:', - ' User: User name'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -from DIRAC import exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -if len(args) < 1: - Script.showHelp() - -choice = raw_input("Are you sure you want to delete user/s %s? yes/no [no]: " % ", ".join(args)) -choice = choice.lower() -if choice not in ("yes", "y"): - print("Delete aborted") - DIRACExit(0) - -for user in args: - if not diracAdmin.csDeleteUser(user): - errorList.append(("delete user", "Cannot delete user %s" % user)) - exitCode = 255 - -if not exitCode: - result = diracAdmin.csCommitChanges() - if not result['OK']: - errorList.append(("commit", result['Message'])) - exitCode = 255 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... User ...' % Script.scriptName, + 'Arguments:', + ' User: User name'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + from DIRAC import exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + if len(args) < 1: + Script.showHelp() + + choice = raw_input("Are you sure you want to delete user/s %s? yes/no [no]: " % ", ".join(args)) + choice = choice.lower() + if choice not in ("yes", "y"): + print("Delete aborted") + DIRACExit(0) + + for user in args: + if not diracAdmin.csDeleteUser(user): + errorList.append(("delete user", "Cannot delete user %s" % user)) + exitCode = 255 + + if not exitCode: + result = diracAdmin.csCommitChanges() + if not result['OK']: + errorList.append(("commit", result['Message'])) + exitCode = 255 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_banned_sites.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_banned_sites.py index 36b505b94a5..d3b5ab5e1f1 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_banned_sites.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_banned_sites.py @@ -11,24 +11,33 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script -Script.parseCommandLine(ignoreErrors=True) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -from DIRAC import gLogger, exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() +@DIRACScript() +def main(): + Script.parseCommandLine(ignoreErrors=True) -result = diracAdmin.getBannedSites() -if result['OK']: - bannedSites = result['Value'] -else: - gLogger.error(result['Message']) - DIRACExit(2) + from DIRAC import gLogger, exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -for site in bannedSites: - result = diracAdmin.getSiteMaskLogging(site, printOutput=True) - if not result['OK']: + diracAdmin = DiracAdmin() + + result = diracAdmin.getBannedSites() + if result['OK']: + bannedSites = result['Value'] + else: gLogger.error(result['Message']) DIRACExit(2) -DIRACExit(0) + for site in bannedSites: + result = diracAdmin.getSiteMaskLogging(site, printOutput=True) + if not result['OK']: + gLogger.error(result['Message']) + DIRACExit(2) + + DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py index 9bfaa1fa193..6c0883599cf 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py @@ -12,39 +12,47 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC ID of the Job'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC import exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -for job in args: - - try: - job = int(job) - except Exception as x: - errorList.append(('Expected integer for JobID', job)) - exitCode = 2 - continue - - result = diracAdmin.getJobPilotOutput(job) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC ID of the Job'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC import exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + for job in args: + + try: + job = int(job) + except Exception as x: + errorList.append(('Expected integer for JobID', job)) + exitCode = 2 + continue + + result = diracAdmin.getJobPilotOutput(job) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py index 063ca125a26..1148123c886 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py @@ -14,39 +14,47 @@ # pylint: disable=wrong-import-position from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC ID of the Job'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC import exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -for job in args: - - try: - job = int(job) - except BaseException as x: - errorList.append((job, 'Expected integer for jobID')) - exitCode = 2 - continue - - result = diracAdmin.getJobPilots(job) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC ID of the Job'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC import exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + for job in args: + + try: + job = int(job) + except BaseException as x: + errorList.append((job, 'Expected integer for jobID')) + exitCode = 2 + continue + + result = diracAdmin.getJobPilots(job) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py index 3e4444eb909..b757719da57 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py @@ -15,6 +15,7 @@ from DIRAC import exit as DIRACExit from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript extendedPrint = False @@ -24,71 +25,75 @@ def setExtendedPrint(_arg): extendedPrint = True -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, - 'Arguments:', - ' PilotID: Grid ID of the pilot'])) -Script.registerSwitch('e', 'extended', 'Get extended printout', setExtendedPrint) -Script.parseCommandLine(ignoreErrors=True) - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -from DIRAC.Interfaces.API.Dirac import Dirac - -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - - -diracAdmin = DiracAdmin() -dirac = Dirac() -exitCode = 0 -errorList = [] - -for gridID in args: - - result = diracAdmin.getPilotInfo(gridID) - if not result['OK']: - errorList.append((gridID, result['Message'])) - exitCode = 2 - else: - res = result['Value'][gridID] - if extendedPrint: - tab = '' - for key in [ - 'PilotJobReference', - 'Status', - 'OwnerDN', - 'OwnerGroup', - 'SubmissionTime', - 'DestinationSite', - 'GridSite', - ]: - if key in res: - diracAdmin.log.notice('%s%s: %s' % (tab, key, res[key])) - if not tab: - tab = ' ' - diracAdmin.log.notice('') - for jobID in res['Jobs']: - tab = ' ' - result = dirac.getJobAttributes(int(jobID)) - if not result['OK']: - errorList.append((gridID, result['Message'])) - exitCode = 2 - else: - job = result['Value'] - diracAdmin.log.notice('%sJob ID: %s' % (tab, jobID)) - tab += ' ' - for key in ['OwnerDN', 'OwnerGroup', 'JobName', 'Status', 'StartExecTime', 'LastUpdateTime', 'EndExecTime']: - if key in job: - diracAdmin.log.notice('%s%s:' % (tab, key), job[key]) - diracAdmin.log.notice('') +@DIRACScript() +def main(): + global extendedPrint + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, + 'Arguments:', + ' PilotID: Grid ID of the pilot'])) + Script.registerSwitch('e', 'extended', 'Get extended printout', setExtendedPrint) + Script.parseCommandLine(ignoreErrors=True) + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + from DIRAC.Interfaces.API.Dirac import Dirac + + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + diracAdmin = DiracAdmin() + dirac = Dirac() + exitCode = 0 + errorList = [] + + for gridID in args: + result = diracAdmin.getPilotInfo(gridID) + if not result['OK']: + errorList.append((gridID, result['Message'])) + exitCode = 2 else: - print(diracAdmin.pPrint.pformat({gridID: res})) - - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) + res = result['Value'][gridID] + if extendedPrint: + tab = '' + for key in [ + 'PilotJobReference', + 'Status', + 'OwnerDN', + 'OwnerGroup', + 'SubmissionTime', + 'DestinationSite', + 'GridSite', + ]: + if key in res: + diracAdmin.log.notice('%s%s: %s' % (tab, key, res[key])) + if not tab: + tab = ' ' + diracAdmin.log.notice('') + for jobID in res['Jobs']: + tab = ' ' + result = dirac.getJobAttributes(int(jobID)) + if not result['OK']: + errorList.append((gridID, result['Message'])) + exitCode = 2 + else: + job = result['Value'] + diracAdmin.log.notice('%sJob ID: %s' % (tab, jobID)) + tab += ' ' + for key in ['OwnerDN', 'OwnerGroup', 'JobName', 'Status', 'StartExecTime', 'LastUpdateTime', 'EndExecTime']: + if key in job: + diracAdmin.log.notice('%s%s:' % (tab, key), job[key]) + diracAdmin.log.notice('') + else: + print(diracAdmin.pPrint.pformat({gridID: res})) + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py index 8b6dfa40c43..8fe5ae43da8 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py @@ -15,36 +15,44 @@ # pylint: disable=wrong-import-position from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, - 'Arguments:', - ' PilotID: Grid ID of the pilot'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC import exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -for gridID in args: - - result = diracAdmin.getPilotLoggingInfo(gridID) - if not result['OK']: - errorList.append((gridID, result['Message'])) - exitCode = 2 - else: - print('Pilot Reference: %s', gridID) - print(result['Value']) - print() - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, + 'Arguments:', + ' PilotID: Grid ID of the pilot'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC import exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + for gridID in args: + + result = diracAdmin.getPilotLoggingInfo(gridID) + if not result['OK']: + errorList.append((gridID, result['Message'])) + exitCode = 2 + else: + print('Pilot Reference: %s', gridID) + print(result['Value']) + print() + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py index bb64a6c531b..13b8009aca0 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py @@ -13,33 +13,40 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, - 'Arguments:', - ' PilotID: Grid ID of the pilot'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, + 'Arguments:', + ' PilotID: Grid ID of the pilot'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp() + if len(args) < 1: + Script.showHelp() -from DIRAC import exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] + from DIRAC import exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] -for gridID in args: + for gridID in args: - result = diracAdmin.getPilotOutput(gridID) - if not result['OK']: - errorList.append((gridID, result['Message'])) - exitCode = 2 + result = diracAdmin.getPilotOutput(gridID) + if not result['OK']: + errorList.append((gridID, result['Message'])) + exitCode = 2 -for error in errorList: - print("ERROR %s: %s" % error) + for error in errorList: + print("ERROR %s: %s" % error) -DIRACExit(exitCode) + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py index 0dc5089a0f4..4713e4e053c 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py @@ -11,26 +11,34 @@ from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage(""" -Get the list of sites enabled in the mask for job submission -Usage: - %s [options] -""" % Script.scriptName) +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Get the list of sites enabled in the mask for job submission -Script.parseCommandLine(ignoreErrors=True) + Usage: + %s [options] + """ % Script.scriptName) -from DIRAC import exit as DIRACExit, gLogger -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + Script.parseCommandLine(ignoreErrors=True) -diracAdmin = DiracAdmin() + from DIRAC import exit as DIRACExit, gLogger + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -gLogger.setLevel('ALWAYS') + diracAdmin = DiracAdmin() -result = diracAdmin.getSiteMask(printOutput=True, status="Active") -if result['OK']: - DIRACExit(0) -else: - print(result['Message']) - DIRACExit(2) + gLogger.setLevel('ALWAYS') + + result = diracAdmin.getSiteMask(printOutput=True, status="Active") + if result['OK']: + DIRACExit(0) + else: + print(result['Message']) + DIRACExit(2) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_list_hosts.py b/src/DIRAC/Interfaces/scripts/dirac_admin_list_hosts.py index b95fd7a9d5a..cb34b525f48 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_list_hosts.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_list_hosts.py @@ -9,32 +9,40 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.registerSwitch("e", "extended", "Show extended info") -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() +@DIRACScript() +def main(): + Script.registerSwitch("e", "extended", "Show extended info") -from DIRAC import exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] -extendedInfo = False + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -for unprocSw in Script.getUnprocessedSwitches(): - if unprocSw[0] in ('e', 'extended'): - extendedInfo = True + from DIRAC import exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + extendedInfo = False -if not extendedInfo: - result = diracAdmin.csListHosts() - for host in result['Value']: - print(" %s" % host) -else: - result = diracAdmin.csDescribeHosts() - print(diracAdmin.pPrint.pformat(result['Value'])) + for unprocSw in Script.getUnprocessedSwitches(): + if unprocSw[0] in ('e', 'extended'): + extendedInfo = True -for error in errorList: - print("ERROR %s: %s" % error) + if not extendedInfo: + result = diracAdmin.csListHosts() + for host in result['Value']: + print(" %s" % host) + else: + result = diracAdmin.csDescribeHosts() + print(diracAdmin.pPrint.pformat(result['Value'])) -DIRACExit(exitCode) + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_list_users.py b/src/DIRAC/Interfaces/scripts/dirac_admin_list_users.py index 4834f52a293..ff01b65eb04 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_list_users.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_list_users.py @@ -12,61 +12,66 @@ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.registerSwitch("e", "extended", "Show extended info") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... [Group] ...' % Script.scriptName, - 'Arguments:', - ' Group: Only users from this group (default: all)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) == 0: - args = ['all'] +@DIRACScript() +def main(): + Script.registerSwitch("e", "extended", "Show extended info") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... [Group] ...' % Script.scriptName, + 'Arguments:', + ' Group: Only users from this group (default: all)'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] -extendedInfo = False + if len(args) == 0: + args = ['all'] -for unprocSw in Script.getUnprocessedSwitches(): - if unprocSw[0] in ('e', 'extended'): - extendedInfo = True + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + extendedInfo = False + for unprocSw in Script.getUnprocessedSwitches(): + if unprocSw[0] in ('e', 'extended'): + extendedInfo = True -def printUsersInGroup(group=False): - result = diracAdmin.csListUsers(group) - if result['OK']: - if group: - print("Users in group %s:" % group) - else: - print("All users registered:") - for username in result['Value']: - print(" %s" % username) + def printUsersInGroup(group=False): + result = diracAdmin.csListUsers(group) + if result['OK']: + if group: + print("Users in group %s:" % group) + else: + print("All users registered:") + for username in result['Value']: + print(" %s" % username) + def describeUsersInGroup(group=False): + result = diracAdmin.csListUsers(group) + if result['OK']: + if group: + print("Users in group %s:" % group) + else: + print("All users registered:") + result = diracAdmin.csDescribeUsers(result['Value']) + print(diracAdmin.pPrint.pformat(result['Value'])) -def describeUsersInGroup(group=False): - result = diracAdmin.csListUsers(group) - if result['OK']: - if group: - print("Users in group %s:" % group) + for group in args: + if 'all' in args: + group = False + if not extendedInfo: + printUsersInGroup(group) else: - print("All users registered:") - result = diracAdmin.csDescribeUsers(result['Value']) - print(diracAdmin.pPrint.pformat(result['Value'])) + describeUsersInGroup(group) + for error in errorList: + print("ERROR %s: %s" % error) -for group in args: - if 'all' in args: - group = False - if not extendedInfo: - printUsersInGroup(group) - else: - describeUsersInGroup(group) + DIRAC.exit(exitCode) -for error in errorList: - print("ERROR %s: %s" % error) -DIRAC.exit(exitCode) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_modify_user.py b/src/DIRAC/Interfaces/scripts/dirac_admin_modify_user.py index c5db766b6ef..259eaf6b253 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_modify_user.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_modify_user.py @@ -14,60 +14,68 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.registerSwitch("p:", "property=", "Add property to the user =") -Script.registerSwitch("f", "force", "create the user if it doesn't exist") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... user DN group [group] ...' % Script.scriptName, - 'Arguments:', - ' user: User name (mandatory)', - ' DN: DN of the User (mandatory)', - ' group: Add the user to the group (mandatory)'])) -Script.parseCommandLine(ignoreErrors=True) +@DIRACScript() +def main(): + Script.registerSwitch("p:", "property=", "Add property to the user =") + Script.registerSwitch("f", "force", "create the user if it doesn't exist") -args = Script.getPositionalArgs() + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... user DN group [group] ...' % Script.scriptName, + 'Arguments:', + ' user: User name (mandatory)', + ' DN: DN of the User (mandatory)', + ' group: Add the user to the group (mandatory)'])) + Script.parseCommandLine(ignoreErrors=True) -if len(args) < 3: - Script.showHelp(exitCode=1) + args = Script.getPositionalArgs() -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -forceCreation = False -errorList = [] + if len(args) < 3: + Script.showHelp(exitCode=1) -userProps = {} -for unprocSw in Script.getUnprocessedSwitches(): - if unprocSw[0] in ("f", "force"): - forceCreation = True - elif unprocSw[0] in ("p", "property"): - prop = unprocSw[1] - pl = prop.split("=") - if len(pl) < 2: - errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) - exitCode = 255 - else: - pName = pl[0] - pValue = "=".join(pl[1:]) - print("Setting property %s to %s" % (pName, pValue)) - userProps[pName] = pValue + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + forceCreation = False + errorList = [] + + userProps = {} + for unprocSw in Script.getUnprocessedSwitches(): + if unprocSw[0] in ("f", "force"): + forceCreation = True + elif unprocSw[0] in ("p", "property"): + prop = unprocSw[1] + pl = prop.split("=") + if len(pl) < 2: + errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) + exitCode = 255 + else: + pName = pl[0] + pValue = "=".join(pl[1:]) + print("Setting property %s to %s" % (pName, pValue)) + userProps[pName] = pValue -userName = args[0] -userProps['DN'] = args[1] -userProps['Groups'] = args[2:] + userName = args[0] + userProps['DN'] = args[1] + userProps['Groups'] = args[2:] -if not diracAdmin.csModifyUser(userName, userProps, createIfNonExistant=forceCreation): - errorList.append(("modify user", "Cannot modify user %s" % userName)) - exitCode = 255 -else: - result = diracAdmin.csCommitChanges() - if not result['OK']: - errorList.append(("commit", result['Message'])) + if not diracAdmin.csModifyUser(userName, userProps, createIfNonExistant=forceCreation): + errorList.append(("modify user", "Cannot modify user %s" % userName)) exitCode = 255 + else: + result = diracAdmin.csCommitChanges() + if not result['OK']: + errorList.append(("commit", result['Message'])) + exitCode = 255 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) -for error in errorList: - print("ERROR %s: %s" % error) -DIRAC.exit(exitCode) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_pilot_summary.py b/src/DIRAC/Interfaces/scripts/dirac_admin_pilot_summary.py index 3d98f135156..cb82d5540f2 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_pilot_summary.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_pilot_summary.py @@ -13,16 +13,24 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() +@DIRACScript() +def main(): + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -result = diracAdmin.getPilotSummary() -if result['OK']: - DIRAC.exit(0) -else: - print(result['Message']) - DIRAC.exit(2) + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + + result = diracAdmin.getPilotSummary() + if result['OK']: + DIRAC.exit(0) + else: + print(result['Message']) + DIRAC.exit(2) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py b/src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py index 08f14c594d5..affbb31801f 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py @@ -14,41 +14,49 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC ID of the Job'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC ID of the Job'])) + Script.parseCommandLine(ignoreErrors=True) -if len(args) < 1: - Script.showHelp() + args = Script.getPositionalArgs() -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] + if len(args) < 1: + Script.showHelp() -for job in args: + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] - try: - job = int(job) - except Exception as x: - errorList.append(('Expected integer for jobID', job)) - exitCode = 2 - continue + for job in args: - result = diracAdmin.resetJob(job) - if result['OK']: - print('Reset Job %s' % (job)) - else: - errorList.append((job, result['Message'])) - exitCode = 2 + try: + job = int(job) + except Exception as x: + errorList.append(('Expected integer for jobID', job)) + exitCode = 2 + continue -for error in errorList: - print("ERROR %s: %s" % error) + result = diracAdmin.resetJob(job) + if result['OK']: + print('Reset Job %s' % (job)) + else: + errorList.append((job, result['Message'])) + exitCode = 2 -DIRAC.exit(exitCode) + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_service_ports.py b/src/DIRAC/Interfaces/scripts/dirac_admin_service_ports.py index 07fdd6dee6f..150215eafa3 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_service_ports.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_service_ports.py @@ -14,24 +14,32 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... [Setup]' % Script.scriptName, - 'Arguments:', - ' Setup: Name of the setup'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -setup = '' -if args: - setup = args[0] - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -result = diracAdmin.getServicePorts(setup, printOutput=True) -if result['OK']: - DIRAC.exit(0) -else: - print(result['Message']) - DIRAC.exit(2) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... [Setup]' % Script.scriptName, + 'Arguments:', + ' Setup: Name of the setup'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + setup = '' + if args: + setup = args[0] + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + result = diracAdmin.getServicePorts(setup, printOutput=True) + if result['OK']: + DIRAC.exit(0) + else: + print(result['Message']) + DIRAC.exit(2) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_set_site_protocols.py b/src/DIRAC/Interfaces/scripts/dirac_admin_set_site_protocols.py index 7b57194c52b..fdeb0a2a90a 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_set_site_protocols.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_set_site_protocols.py @@ -14,31 +14,39 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.registerSwitch("", "Site=", "Site for which protocols are to be set (mandatory)") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Protocol ...' % Script.scriptName, - 'Arguments:', - ' Protocol: SE access protocol (mandatory)'])) -Script.parseCommandLine(ignoreErrors=True) - -site = None -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "site": - site = switch[1] - -args = Script.getPositionalArgs() - -if not site or not args: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -result = diracAdmin.setSiteProtocols(site, args, printOutput=True) -if not result['OK']: - print('ERROR: %s' % result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) + +@DIRACScript() +def main(): + Script.registerSwitch("", "Site=", "Site for which protocols are to be set (mandatory)") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Protocol ...' % Script.scriptName, + 'Arguments:', + ' Protocol: SE access protocol (mandatory)'])) + Script.parseCommandLine(ignoreErrors=True) + + site = None + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "site": + site = switch[1] + + args = Script.getPositionalArgs() + + if not site or not args: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + result = diracAdmin.setSiteProtocols(site, args, printOutput=True) + if not result['OK']: + print('ERROR: %s' % result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py index 37cfd46c9bc..b632b915d81 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py @@ -14,31 +14,39 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Site ...' % Script.scriptName, - 'Arguments:', - ' Site: Name of the Site'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Site ...' % Script.scriptName, + 'Arguments:', + ' Site: Name of the Site'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] + if len(args) < 1: + Script.showHelp() -for site in args: + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] - result = diracAdmin.getSiteSection(site, printOutput=True) - if not result['OK']: - errorList.append((site, result['Message'])) - exitCode = 2 + for site in args: -for error in errorList: - print("ERROR %s: %s" % error) + result = diracAdmin.getSiteSection(site, printOutput=True) + if not result['OK']: + errorList.append((site, result['Message'])) + exitCode = 2 -DIRAC.exit(exitCode) + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_site_mask_logging.py b/src/DIRAC/Interfaces/scripts/dirac_admin_site_mask_logging.py index f17d6e1c757..f70d9566a4e 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_site_mask_logging.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_site_mask_logging.py @@ -15,31 +15,39 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Site ...' % Script.scriptName, - 'Arguments:', - ' Site: Name of the Site'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Site ...' % Script.scriptName, + 'Arguments:', + ' Site: Name of the Site'])) -if len(args) < 1: - Script.showHelp() + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] + if len(args) < 1: + Script.showHelp() -for site in args: - result = diracAdmin.getSiteMaskLogging(site, printOutput=True) - if not result['OK']: - errorList.append((site, result['Message'])) - exitCode = 2 + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] -for error in errorList: - print("ERROR %s: %s" % error) + for site in args: + result = diracAdmin.getSiteMaskLogging(site, printOutput=True) + if not result['OK']: + errorList.append((site, result['Message'])) + exitCode = 2 -DIRAC.exit(exitCode) + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_sync_users_from_file.py b/src/DIRAC/Interfaces/scripts/dirac_admin_sync_users_from_file.py index f184be0c8fe..4640befe3c0 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_sync_users_from_file.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_sync_users_from_file.py @@ -13,53 +13,60 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript __RCSID__ = "$Id$" -Script.registerSwitch("t", "test", "Only test. Don't commit changes") -Script.setUsageMessage( - '\n'.join( - [ - __doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... UserCfg' % - Script.scriptName, - 'Arguments:', - ' UserCfg: Cfg FileName with Users as sections containing DN, Groups, and other properties as options'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getExtraCLICFGFiles() +@DIRACScript() +def main(): + Script.registerSwitch("t", "test", "Only test. Don't commit changes") + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... UserCfg' % + Script.scriptName, + 'Arguments:', + ' UserCfg: Cfg FileName with Users as sections containing DN, Groups, and other properties as options' + ])) + Script.parseCommandLine(ignoreErrors=True) -if len(args) < 1: - Script.showHelp() + args = Script.getExtraCLICFGFiles() -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -testOnly = False -errorList = [] + if len(args) < 1: + Script.showHelp() -for unprocSw in Script.getUnprocessedSwitches(): - if unprocSw[0] in ("t", "test"): - testOnly = True + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + testOnly = False + errorList = [] -try: - usersCFG = CFG().loadFromFile(args[0]) -except Exception as e: - errorList.append("file open", "Can't parse file %s: %s" % (args[0], str(e))) - errorCode = 1 -else: - if not diracAdmin.csSyncUsersWithCFG(usersCFG): - errorList.append(("modify users", "Cannot sync with %s" % args[0])) - exitCode = 255 + for unprocSw in Script.getUnprocessedSwitches(): + if unprocSw[0] in ("t", "test"): + testOnly = True -if not exitCode and not testOnly: - result = diracAdmin.csCommitChanges() - if not result['OK']: - errorList.append(("commit", result['Message'])) - exitCode = 255 + try: + usersCFG = CFG().loadFromFile(args[0]) + except Exception as e: + errorList.append("file open", "Can't parse file %s: %s" % (args[0], str(e))) + errorCode = 1 + else: + if not diracAdmin.csSyncUsersWithCFG(usersCFG): + errorList.append(("modify users", "Cannot sync with %s" % args[0])) + exitCode = 255 -for error in errorList: - print("ERROR %s: %s" % error) + if not exitCode and not testOnly: + result = diracAdmin.csCommitChanges() + if not result['OK']: + errorList.append(("commit", result['Message'])) + exitCode = 255 -DIRAC.exit(exitCode) + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_get_file.py b/src/DIRAC/Interfaces/scripts/dirac_dms_get_file.py index 228e6ca8713..735d546353c 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_dms_get_file.py +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_get_file.py @@ -14,32 +14,40 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, - 'Arguments:', - ' LFN: Logical File Name or file containing LFNs'])) -Script.parseCommandLine(ignoreErrors=True) -lfns = Script.getPositionalArgs() - -if len(lfns) < 1: - Script.showHelp() - -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 - -if len(lfns) == 1: - try: - with open(lfns[0], 'r') as f: - lfns = f.read().splitlines() - except Exception: - pass - -result = dirac.getFile(lfns, printOutput=True) -if not result['OK']: - print('ERROR %s' % (result['Message'])) - exitCode = 2 - -DIRAC.exit(exitCode) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, + 'Arguments:', + ' LFN: Logical File Name or file containing LFNs'])) + Script.parseCommandLine(ignoreErrors=True) + lfns = Script.getPositionalArgs() + + if len(lfns) < 1: + Script.showHelp() + + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + + if len(lfns) == 1: + try: + with open(lfns[0], 'r') as f: + lfns = f.read().splitlines() + except Exception: + pass + + result = dirac.getFile(lfns, printOutput=True) + if not result['OK']: + print('ERROR %s' % (result['Message'])) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_accessURL.py b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_accessURL.py index d0acb761a94..289f881a11c 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_accessURL.py +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_accessURL.py @@ -14,45 +14,53 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... LFN SE [PROTO]' % Script.scriptName, - 'Arguments:', - ' LFN: Logical File Name or file containing LFNs (mandatory)', - ' SE: Valid DIRAC SE (mandatory)', - ' PROTO: Optional protocol for accessURL'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -# pylint: disable=wrong-import-position -from DIRAC.Interfaces.API.Dirac import Dirac - -if len(args) < 2: - Script.showHelp(exitCode=1) - -if len(args) > 3: - print('Only one LFN SE pair will be considered') - -dirac = Dirac() -exitCode = 0 - -lfn = args[0] -seName = args[1] -proto = False -if len(args) > 2: - proto = args[2] - -try: - with open(lfn, 'r') as f: - lfns = f.read().splitlines() -except IOError: - lfns = [lfn] - -for lfn in lfns: - result = dirac.getAccessURL(lfn, seName, protocol=proto, printOutput=True) - if not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... LFN SE [PROTO]' % Script.scriptName, + 'Arguments:', + ' LFN: Logical File Name or file containing LFNs (mandatory)', + ' SE: Valid DIRAC SE (mandatory)', + ' PROTO: Optional protocol for accessURL'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + # pylint: disable=wrong-import-position + from DIRAC.Interfaces.API.Dirac import Dirac + + if len(args) < 2: + Script.showHelp(exitCode=1) + + if len(args) > 3: + print('Only one LFN SE pair will be considered') + + dirac = Dirac() + exitCode = 0 + + lfn = args[0] + seName = args[1] + proto = False + if len(args) > 2: + proto = args[2] + + try: + with open(lfn, 'r') as f: + lfns = f.read().splitlines() + except IOError: + lfns = [lfn] + + for lfn in lfns: + result = dirac.getAccessURL(lfn, seName, protocol=proto, printOutput=True) + if not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_metadata.py b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_metadata.py index fed4db4d6ff..39e534022c6 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_metadata.py +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_metadata.py @@ -14,33 +14,41 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, - 'Arguments:', - ' LFN: Logical File Name or file containing LFNs'])) -Script.parseCommandLine(ignoreErrors=True) -lfns = Script.getPositionalArgs() - -if len(lfns) < 1: - Script.showHelp() - -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 -errorList = [] - -if len(lfns) == 1: - try: - with open(lfns[0], 'r') as f: - lfns = f.read().splitlines() - except BaseException: - pass - -result = dirac.getLfnMetadata(lfns, printOutput=True) -if not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, + 'Arguments:', + ' LFN: Logical File Name or file containing LFNs'])) + Script.parseCommandLine(ignoreErrors=True) + lfns = Script.getPositionalArgs() + + if len(lfns) < 1: + Script.showHelp() + + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + errorList = [] + + if len(lfns) == 1: + try: + with open(lfns[0], 'r') as f: + lfns = f.read().splitlines() + except BaseException: + pass + + result = dirac.getLfnMetadata(lfns, printOutput=True) + if not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_replicas.py b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_replicas.py index 6067d91cf0c..1e4f142f200 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_replicas.py +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_replicas.py @@ -14,39 +14,47 @@ import DIRAC from DIRAC.Core.Base import Script - -Script.registerSwitch('a', "All", " Also show inactive replicas") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, - 'Arguments:', - ' LFN: Logical File Name or file containing LFNs'])) -Script.parseCommandLine(ignoreErrors=True) -lfns = Script.getPositionalArgs() -switches = Script.getUnprocessedSwitches() - -active = True -for switch in switches: - opt = switch[0].lower() - if opt in ("a", "all"): - active = False -if len(lfns) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 - -if len(lfns) == 1: - try: - with open(lfns[0], 'r') as f: - lfns = f.read().splitlines() - except BaseException: - pass - -result = dirac.getReplicas(lfns, active=active, printOutput=True) -if not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch('a', "All", " Also show inactive replicas") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, + 'Arguments:', + ' LFN: Logical File Name or file containing LFNs'])) + Script.parseCommandLine(ignoreErrors=True) + lfns = Script.getPositionalArgs() + switches = Script.getUnprocessedSwitches() + + active = True + for switch in switches: + opt = switch[0].lower() + if opt in ("a", "all"): + active = False + if len(lfns) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + + if len(lfns) == 1: + try: + with open(lfns[0], 'r') as f: + lfns = f.read().splitlines() + except BaseException: + pass + + result = dirac.getReplicas(lfns, active=active, printOutput=True) + if not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_accessURL.py b/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_accessURL.py index 805725dc573..dce18d093bb 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_accessURL.py +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_accessURL.py @@ -14,38 +14,46 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... PFN SE' % Script.scriptName, - 'Arguments:', - ' PFN: Physical File Name or file containing PFNs (mandatory)', - ' SE: Valid DIRAC SE (mandatory)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 2: - Script.showHelp(exitCode=1) - -if len(args) > 2: - print('Only one PFN SE pair will be considered') - -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 - -pfn = args[0] -seName = args[1] -try: - with open(pfn, 'r') as f: - pfns = f.read().splitlines() -except BaseException: - pfns = [pfn] - -for pfn in pfns: - result = dirac.getPhysicalFileAccessURL(pfn, seName, printOutput=True) - if not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... PFN SE' % Script.scriptName, + 'Arguments:', + ' PFN: Physical File Name or file containing PFNs (mandatory)', + ' SE: Valid DIRAC SE (mandatory)'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 2: + Script.showHelp(exitCode=1) + + if len(args) > 2: + print('Only one PFN SE pair will be considered') + + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + + pfn = args[0] + seName = args[1] + try: + with open(pfn, 'r') as f: + pfns = f.read().splitlines() + except BaseException: + pfns = [pfn] + + for pfn in pfns: + result = dirac.getPhysicalFileAccessURL(pfn, seName, printOutput=True) + if not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_metadata.py b/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_metadata.py index c2be191f7b9..db31f525148 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_metadata.py +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_metadata.py @@ -14,39 +14,47 @@ import DIRAC from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... PFN SE' % Script.scriptName, - 'Arguments:', - ' PFN: Physical File Name or file containing PFNs (mandatory)', - ' SE: Valid DIRAC SE (mandatory)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 2: - Script.showHelp(exitCode=1) - -if len(args) > 2: - print('Only one PFN SE pair will be considered') - -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 - -pfn = args[0] -seName = args[1] -try: - f = open(pfn, 'r') - pfns = f.read().splitlines() - f.close() -except BaseException: - pfns = [pfn] - -for pfn in pfns: - result = dirac.getPhysicalFileMetadata(pfn, seName, printOutput=True) - if not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... PFN SE' % Script.scriptName, + 'Arguments:', + ' PFN: Physical File Name or file containing PFNs (mandatory)', + ' SE: Valid DIRAC SE (mandatory)'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 2: + Script.showHelp(exitCode=1) + + if len(args) > 2: + print('Only one PFN SE pair will be considered') + + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + + pfn = args[0] + seName = args[1] + try: + f = open(pfn, 'r') + pfns = f.read().splitlines() + f.close() + except BaseException: + pfns = [pfn] + + for pfn in pfns: + result = dirac.getPhysicalFileMetadata(pfn, seName, printOutput=True) + if not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_replicate_lfn.py b/src/DIRAC/Interfaces/scripts/dirac_dms_replicate_lfn.py index 1fe1ad70b00..ac6cd2c2ae3 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_dms_replicate_lfn.py +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_replicate_lfn.py @@ -14,52 +14,60 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... LFN Dest [Source [Cache]]' % Script.scriptName, - 'Arguments:', - ' LFN: Logical File Name or file containing LFNs (mandatory)', - ' Dest: Valid DIRAC SE (mandatory)', - ' Source: Valid DIRAC SE', - ' Cache: Local directory to be used as cache'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 2 or len(args) > 4: - Script.showHelp(exitCode=1) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... LFN Dest [Source [Cache]]' % Script.scriptName, + 'Arguments:', + ' LFN: Logical File Name or file containing LFNs (mandatory)', + ' Dest: Valid DIRAC SE (mandatory)', + ' Source: Valid DIRAC SE', + ' Cache: Local directory to be used as cache'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -lfn = args[0] -seName = args[1] -sourceSE = '' -localCache = '' -if len(args) > 2: - sourceSE = args[2] -if len(args) == 4: - localCache = args[3] + if len(args) < 2 or len(args) > 4: + Script.showHelp(exitCode=1) -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 + lfn = args[0] + seName = args[1] + sourceSE = '' + localCache = '' + if len(args) > 2: + sourceSE = args[2] + if len(args) == 4: + localCache = args[3] -try: - f = open(lfn, 'r') - lfns = f.read().splitlines() - f.close() -except BaseException: - lfns = [lfn] + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 -finalResult = {"Failed": [], "Successful": []} -for lfn in lfns: - result = dirac.replicateFile(lfn, seName, sourceSE, localCache, printOutput=True) - if not result['OK']: - finalResult["Failed"].append(lfn) - print('ERROR %s' % (result['Message'])) - exitCode = 2 - else: - finalResult["Successful"].append(lfn) + try: + f = open(lfn, 'r') + lfns = f.read().splitlines() + f.close() + except BaseException: + lfns = [lfn] -if len(lfns) > 1: - print(finalResult) + finalResult = {"Failed": [], "Successful": []} + for lfn in lfns: + result = dirac.replicateFile(lfn, seName, sourceSE, localCache, printOutput=True) + if not result['OK']: + finalResult["Failed"].append(lfn) + print('ERROR %s' % (result['Message'])) + exitCode = 2 + else: + finalResult["Successful"].append(lfn) -DIRAC.exit(exitCode) + if len(lfns) > 1: + print(finalResult) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_framework_ping_service.py b/src/DIRAC/Interfaces/scripts/dirac_framework_ping_service.py index c809d16498f..2e3184ad3c5 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_framework_ping_service.py +++ b/src/DIRAC/Interfaces/scripts/dirac_framework_ping_service.py @@ -14,48 +14,55 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... System Service|System/Agent' % Script.scriptName, - 'Arguments:', - ' System: Name of the DIRAC system (ie: WorkloadManagement) (mandatory)', - ' Service: Name of the DIRAC service (ie: Matcher) (mandatory)', - ' url: URL of the service to ping (instead of System and Service)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -system = None -service = None -url = None -if len(args) == 1: - # it is a URL - if args[0].startswith('dips://'): - url = args[0] - # It is System/Service - else: - sys_serv = args[0].split('/') - if len(sys_serv) != 2: - Script.showHelp(exitCode=1) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... System Service|System/Agent' % Script.scriptName, + 'Arguments:', + ' System: Name of the DIRAC system (ie: WorkloadManagement) (mandatory)', + ' Service: Name of the DIRAC service (ie: Matcher) (mandatory)', + ' url: URL of the service to ping (instead of System and Service)'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + system = None + service = None + url = None + if len(args) == 1: + # it is a URL + if args[0].startswith('dips://'): + url = args[0] + # It is System/Service else: - system, service = sys_serv + sys_serv = args[0].split('/') + if len(sys_serv) != 2: + Script.showHelp(exitCode=1) + else: + system, service = sys_serv + + elif len(args) == 2: + system, service = args[0], args[1] + else: + Script.showHelp(exitCode=1) -elif len(args) == 2: - system, service = args[0], args[1] -else: - Script.showHelp(exitCode=1) + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + result = dirac.pingService(system, service, printOutput=True, url=url) -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 + if not result: + print('ERROR: Null result from ping()') + exitCode = 2 + elif not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 -result = dirac.pingService(system, service, printOutput=True, url=url) + DIRAC.exit(exitCode) -if not result: - print('ERROR: Null result from ping()') - exitCode = 2 -elif not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 -DIRAC.exit(exitCode) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_framework_self_ping.py b/src/DIRAC/Interfaces/scripts/dirac_framework_self_ping.py index 54027ccb28a..613b6342084 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_framework_self_ping.py +++ b/src/DIRAC/Interfaces/scripts/dirac_framework_self_ping.py @@ -15,18 +15,26 @@ import sys import os import time +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -with open(os.devnull, 'w') as redirectStdout, open(os.devnull, 'w') as redirectStderr: - from DIRAC import gLogger - from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData - gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true') - gLogger.setLevel('FATAL') - from DIRAC.Core.DISET.RPCClient import RPCClient - - rpc = RPCClient('dips://localhost:%s' % sys.argv[1]) - res = rpc.ping() - time.sleep(0.1) - if not res['OK']: - sys.exit(1) - else: - sys.exit(0) + +@DIRACScript() +def main(): + with open(os.devnull, 'w') as redirectStdout, open(os.devnull, 'w') as redirectStderr: + from DIRAC import gLogger + from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true') + gLogger.setLevel('FATAL') + from DIRAC.Core.DISET.RPCClient import RPCClient + + rpc = RPCClient('dips://localhost:%s' % sys.argv[1]) + res = rpc.ping() + time.sleep(0.1) + if not res['OK']: + sys.exit(1) + else: + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_repo_monitor.py b/src/DIRAC/Interfaces/scripts/dirac_repo_monitor.py index ee799e3b9c4..1804946127f 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_repo_monitor.py +++ b/src/DIRAC/Interfaces/scripts/dirac_repo_monitor.py @@ -10,26 +10,34 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... RepoDir' % Script.scriptName, - 'Arguments:', - ' RepoDir: Location of Job Repository'])) -Script.parseCommandLine(ignoreErrors=False) -args = Script.getPositionalArgs() -if len(args) != 1: - Script.showHelp() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... RepoDir' % Script.scriptName, + 'Arguments:', + ' RepoDir: Location of Job Repository'])) + Script.parseCommandLine(ignoreErrors=False) + args = Script.getPositionalArgs() -repoLocation = args[0] -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac(withRepo=True, repoLocation=repoLocation) + if len(args) != 1: + Script.showHelp() -exitCode = 0 -result = dirac.monitorRepository(printOutput=True) -if not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 + repoLocation = args[0] + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac(withRepo=True, repoLocation=repoLocation) -DIRAC.exit(exitCode) + exitCode = 0 + result = dirac.monitorRepository(printOutput=True) + if not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_utils_file_adler.py b/src/DIRAC/Interfaces/scripts/dirac_utils_file_adler.py index 8feec1f20f0..620fe5b2ff2 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_utils_file_adler.py +++ b/src/DIRAC/Interfaces/scripts/dirac_utils_file_adler.py @@ -14,25 +14,33 @@ import DIRAC from DIRAC.Core.Utilities.Adler import fileAdler from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... File ...' % Script.scriptName, - 'Arguments:', - ' File: File Name'])) -Script.parseCommandLine(ignoreErrors=False) -files = Script.getPositionalArgs() -if len(files) == 0: - Script.showHelp() - -exitCode = 0 - -for fa in files: - adler = fileAdler(fa) - if adler: - print(fa.rjust(100), adler.ljust(10)) # pylint: disable=no-member - else: - print('ERROR %s: Failed to get adler' % fa) - exitCode = 2 - -DIRAC.exit(exitCode) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... File ...' % Script.scriptName, + 'Arguments:', + ' File: File Name'])) + Script.parseCommandLine(ignoreErrors=False) + files = Script.getPositionalArgs() + if len(files) == 0: + Script.showHelp() + + exitCode = 0 + + for fa in files: + adler = fileAdler(fa) + if adler: + print(fa.rjust(100), adler.ljust(10)) # pylint: disable=no-member + else: + print('ERROR %s: Failed to get adler' % fa) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_utils_file_md5.py b/src/DIRAC/Interfaces/scripts/dirac_utils_file_md5.py index 7ff4399fcf2..cdb70483909 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_utils_file_md5.py +++ b/src/DIRAC/Interfaces/scripts/dirac_utils_file_md5.py @@ -15,29 +15,37 @@ import DIRAC from DIRAC.Core.Utilities.File import makeGuid from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... File ...' % Script.scriptName, - 'Arguments:', - ' File: File Name'])) -Script.parseCommandLine(ignoreErrors=False) -files = Script.getPositionalArgs() -if len(files) == 0: - Script.showHelp() - -exitCode = 0 - -for file in files: - try: - md5 = makeGuid(file) - if md5: - print(file.rjust(100), md5.ljust(10)) - else: - print('ERROR %s: Failed to get md5' % file) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... File ...' % Script.scriptName, + 'Arguments:', + ' File: File Name'])) + Script.parseCommandLine(ignoreErrors=False) + files = Script.getPositionalArgs() + if len(files) == 0: + Script.showHelp() + + exitCode = 0 + + for file in files: + try: + md5 = makeGuid(file) + if md5: + print(file.rjust(100), md5.ljust(10)) + else: + print('ERROR %s: Failed to get md5' % file) + exitCode = 2 + except Exception as x: + print('ERROR %s: Failed to get md5' % file, str(x)) exitCode = 2 - except Exception as x: - print('ERROR %s: Failed to get md5' % file, str(x)) - exitCode = 2 -DIRAC.exit(exitCode) + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py b/src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py index e2684d72d74..72a42c1799d 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py @@ -6,7 +6,8 @@ """ Report Normalized CPU length of queue - This script was used by the dirac-pilot script to set the CPUTime limit for the matching but now this is no more the case + This script was used by the dirac-pilot script to set the CPUTime limit for + the matching but now this is no more the case. """ from __future__ import print_function from __future__ import absolute_import @@ -17,28 +18,38 @@ import DIRAC from DIRAC.Core.Base import Script from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import queueNormalizedCPU +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Queue ...' % Script.scriptName, - 'Arguments:', - ' Queue: GlueCEUniqueID of the Queue (ie, juk.nikhef.nl:8443/cream-pbs-lhcb)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Queue ...' % Script.scriptName, + 'Arguments:', + ' Queue: GlueCEUniqueID of the Queue (ie, juk.nikhef.nl:8443/cream-pbs-lhcb)' + ])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -exitCode = 0 + if len(args) < 1: + Script.showHelp() -for ceUniqueID in args: + exitCode = 0 - normCPU = queueNormalizedCPU(ceUniqueID) + for ceUniqueID in args: - if not normCPU['OK']: - print('ERROR %s:' % ceUniqueID, normCPU['Message']) - exitCode = 2 - continue - print(ceUniqueID, normCPU['Value']) + normCPU = queueNormalizedCPU(ceUniqueID) -DIRAC.exit(exitCode) + if not normCPU['OK']: + print('ERROR %s:' % ceUniqueID, normCPU['Message']) + exitCode = 2 + continue + print(ceUniqueID, normCPU['Value']) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py b/src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py index 313653d1508..4a3e2f4627a 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py @@ -15,28 +15,38 @@ import DIRAC from DIRAC.Core.Base import Script from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import getQueueNormalization +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Queue ...' % Script.scriptName, - 'Arguments:', - ' Queue: GlueCEUniqueID of the Queue (ie, juk.nikhef.nl:8443/cream-pbs-lhcb)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Queue ...' % Script.scriptName, + 'Arguments:', + ' Queue: GlueCEUniqueID of the Queue (ie, juk.nikhef.nl:8443/cream-pbs-lhcb)', + ])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -exitCode = 0 + if len(args) < 1: + Script.showHelp() -for ceUniqueID in args: + exitCode = 0 - cpuNorm = getQueueNormalization(ceUniqueID) + for ceUniqueID in args: - if not cpuNorm['OK']: - print('ERROR %s:' % ceUniqueID, cpuNorm['Message']) - exitCode = 2 - continue - print(ceUniqueID, cpuNorm['Value']) + cpuNorm = getQueueNormalization(ceUniqueID) -DIRAC.exit(exitCode) + if not cpuNorm['OK']: + print('ERROR %s:' % ceUniqueID, cpuNorm['Message']) + exitCode = 2 + continue + print(ceUniqueID, cpuNorm['Value']) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py index 11b3e89b191..89fdf413266 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py @@ -14,31 +14,39 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] + if len(args) < 1: + Script.showHelp() -for job in parseArguments(args): + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] - result = dirac.getJobAttributes(job, printOutput=True) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 + for job in parseArguments(args): -for error in errorList: - print("ERROR %s: %s" % error) + result = dirac.getJobAttributes(job, printOutput=True) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 -DIRAC.exit(exitCode) + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_delete.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_delete.py index 7d02057e3bc..7f69e3a538a 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_delete.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_delete.py @@ -17,21 +17,22 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file") -Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group") +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() + Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file") + Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group") - -if __name__ == "__main__": + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments from DIRAC.Core.Utilities.Time import toString, date, day @@ -72,3 +73,7 @@ exitCode = 2 DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py index d6c1229b011..3759ac87a2e 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py @@ -16,40 +16,48 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.registerSwitch("D:", "Dir=", "Store the output in this directory") -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] - -outputDir = None -for sw, v in Script.getUnprocessedSwitches(): - if sw in ('D', 'Dir'): - outputDir = v - -for job in parseArguments(args): - - result = dirac.getInputSandbox(job, outputDir=outputDir) - if result['OK']: - if os.path.exists('InputSandbox%s' % job): - print('Job input sandbox retrieved in InputSandbox%s/' % (job)) - else: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.registerSwitch("D:", "Dir=", "Store the output in this directory") + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] + + outputDir = None + for sw, v in Script.getUnprocessedSwitches(): + if sw in ('D', 'Dir'): + outputDir = v + + for job in parseArguments(args): + + result = dirac.getInputSandbox(job, outputDir=outputDir) + if result['OK']: + if os.path.exists('InputSandbox%s' % job): + print('Job input sandbox retrieved in InputSandbox%s/' % (job)) + else: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py index 0872c0bbc3e..d655b7a15e2 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py @@ -14,41 +14,49 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -original = False -Script.registerSwitch('O', 'Original', 'Gets the original JDL') -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -for switch in Script.getUnprocessedSwitches(): - if switch[0] == 'Original' or switch[0] == 'O': - original = True +@DIRACScript() +def main(): + original = False + Script.registerSwitch('O', 'Original', 'Gets the original JDL') + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -for switch in Script.getUnprocessedSwitches(): - if switch[0] == 'Original': - original = True + for switch in Script.getUnprocessedSwitches(): + if switch[0] == 'Original' or switch[0] == 'O': + original = True -if len(args) < 1: - Script.showHelp(exitCode=1) + for switch in Script.getUnprocessedSwitches(): + if switch[0] == 'Original': + original = True -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] + if len(args) < 1: + Script.showHelp(exitCode=1) -for job in parseArguments(args): + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] - result = dirac.getJobJDL(job, original=original, printOutput=True) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 + for job in parseArguments(args): -for error in errorList: - print("ERROR %s: %s" % error) + result = dirac.getJobJDL(job, original=original, printOutput=True) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 -DIRAC.exit(exitCode) + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py index bb92070909f..3f860b2031e 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py @@ -18,101 +18,109 @@ import DIRAC from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID or a name of the file with JobID per line'])) - -Script.registerSwitch("D:", "Dir=", "Store the output in this directory") -Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file") -Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group") - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -from DIRAC.Core.Utilities.Time import toString, date, day -from DIRAC.Core.Utilities.File import mkDir - -dirac = Dirac() -exitCode = 0 -errorList = [] - -outputDir = None -group = None -jobs = [] -for sw, value in Script.getUnprocessedSwitches(): - if sw in ('D', 'Dir'): - outputDir = value - elif sw.lower() in ('f', 'file'): - if os.path.exists(value): - jFile = open(value) - jobs += jFile.read().split() - jFile.close() - elif sw.lower() in ('g', 'jobgroup'): - group = value - jobDate = toString(date() - 30 * day) - - # Choose jobs in final state, no more than 30 days old - result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Done') - if not result['OK']: - if "No jobs selected" not in result['Message']: - print("Error:", result['Message']) - DIRAC.exit(-1) - else: - jobs += result['Value'] - result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Failed') - if not result['OK']: - if "No jobs selected" not in result['Message']: - print("Error:", result['Message']) - DIRAC.exit(-1) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID or a name of the file with JobID per line'])) + + Script.registerSwitch("D:", "Dir=", "Store the output in this directory") + Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file") + Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group") + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + from DIRAC.Core.Utilities.Time import toString, date, day + from DIRAC.Core.Utilities.File import mkDir + + dirac = Dirac() + exitCode = 0 + errorList = [] + + outputDir = None + group = None + jobs = [] + for sw, value in Script.getUnprocessedSwitches(): + if sw in ('D', 'Dir'): + outputDir = value + elif sw.lower() in ('f', 'file'): + if os.path.exists(value): + jFile = open(value) + jobs += jFile.read().split() + jFile.close() + elif sw.lower() in ('g', 'jobgroup'): + group = value + jobDate = toString(date() - 30 * day) + + # Choose jobs in final state, no more than 30 days old + result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Done') + if not result['OK']: + if "No jobs selected" not in result['Message']: + print("Error:", result['Message']) + DIRAC.exit(-1) + else: + jobs += result['Value'] + result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Failed') + if not result['OK']: + if "No jobs selected" not in result['Message']: + print("Error:", result['Message']) + DIRAC.exit(-1) + else: + jobs += result['Value'] + + for arg in parseArguments(args): + if os.path.isdir(arg): + print("Output for job %s already retrieved, remove the output directory to redownload" % arg) else: - jobs += result['Value'] + jobs.append(arg) -for arg in parseArguments(args): - if os.path.isdir(arg): - print("Output for job %s already retrieved, remove the output directory to redownload" % arg) - else: - jobs.append(arg) + if not jobs: + print("No jobs selected") + DIRAC.exit(0) -if not jobs: - print("No jobs selected") - DIRAC.exit(0) + if group: + if outputDir: + outputDir = os.path.join(outputDir, group) + else: + outputDir = group -if group: if outputDir: - outputDir = os.path.join(outputDir, group) + mkDir(outputDir) else: - outputDir = group + outputDir = os.getcwd() -if outputDir: - mkDir(outputDir) -else: - outputDir = os.getcwd() + jobs = [str(job) for job in jobs] + doneJobs = os.listdir(outputDir) + todoJobs = [job for job in jobs if job not in doneJobs] -jobs = [str(job) for job in jobs] -doneJobs = os.listdir(outputDir) -todoJobs = [job for job in jobs if job not in doneJobs] + for job in todoJobs: -for job in todoJobs: + result = dirac.getOutputSandbox(job, outputDir=outputDir) - result = dirac.getOutputSandbox(job, outputDir=outputDir) + jobDir = str(job) + if outputDir: + jobDir = os.path.join(outputDir, job) + if result['OK']: + if os.path.exists(jobDir): + print('Job output sandbox retrieved in %s/' % (jobDir)) + else: + if os.path.exists('%s' % jobDir): + shutil.rmtree(jobDir) + errorList.append((job, result['Message'])) + exitCode = 2 - jobDir = str(job) - if outputDir: - jobDir = os.path.join(outputDir, job) - if result['OK']: - if os.path.exists(jobDir): - print('Job output sandbox retrieved in %s/' % (jobDir)) - else: - if os.path.exists('%s' % jobDir): - shutil.rmtree(jobDir) - errorList.append((job, result['Message'])) - exitCode = 2 + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) -for error in errorList: - print("ERROR %s: %s" % error) -DIRAC.exit(exitCode) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py index f1b2ac6c1b7..60d79d8c88c 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py @@ -14,39 +14,47 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.registerSwitch("D:", "Dir=", "Store the output in this directory") -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] - -outputDir = '' -for sw, v in Script.getUnprocessedSwitches(): - if sw in ('D', 'Dir'): - outputDir = v - -for job in parseArguments(args): - - result = dirac.getJobOutputData(job, destinationDir=outputDir) - if result['OK']: - print('Job %s output data retrieved' % (job)) - else: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.registerSwitch("D:", "Dir=", "Store the output in this directory") + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] + + outputDir = '' + for sw, v in Script.getUnprocessedSwitches(): + if sw in ('D', 'Dir'): + outputDir = v + + for job in parseArguments(args): + + result = dirac.getJobOutputData(job, destinationDir=outputDir) + if result['OK']: + print('Job %s output data retrieved' % (job)) + else: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_kill.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_kill.py index 9180d137e3c..e5ce171f569 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_kill.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_kill.py @@ -14,26 +14,34 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + if len(args) < 1: + Script.showHelp(exitCode=1) -result = Dirac().killJob(parseArguments(args)) -if result['OK']: - print('Killed jobs %s' % ','.join([str(j) for j in result['Value']])) - exitCode = 0 -else: - print('ERROR', result['Message']) - exitCode = 2 + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -DIRAC.exit(exitCode) + result = Dirac().killJob(parseArguments(args)) + if result['OK']: + print('Killed jobs %s' % ','.join([str(j) for j in result['Value']])) + exitCode = 0 + else: + print('ERROR', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py index 8bfb170ae09..581b0dd882f 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py @@ -14,31 +14,39 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] + if len(args) < 1: + Script.showHelp(exitCode=1) -for job in parseArguments(args): + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] - result = dirac.getJobLoggingInfo(job, printOutput=True) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 + for job in parseArguments(args): -for error in errorList: - print("ERROR %s: %s" % error) + result = dirac.getJobLoggingInfo(job, printOutput=True) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 -DIRAC.exit(exitCode) + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py index 95a43e6f54a..938abf9eeaa 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py @@ -17,31 +17,39 @@ from DIRAC import gLogger from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] + if len(args) < 1: + Script.showHelp(exitCode=1) -for job in parseArguments(args): + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] - result = dirac.getJobParameters(job, printOutput=True) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 + for job in parseArguments(args): -for error in errorList: - print("ERROR %s: %s" % error) + result = dirac.getJobParameters(job, printOutput=True) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 -DIRAC.exit(exitCode) + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py index a80c5e86136..fe7b6dd3d93 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py @@ -14,31 +14,39 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] + if len(args) < 1: + Script.showHelp(exitCode=1) -for job in parseArguments(args): + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] - result = dirac.peekJob(job, printOutput=True) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 + for job in parseArguments(args): -for error in errorList: - print("ERROR %s: %s" % error) + result = dirac.peekJob(job, printOutput=True) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 -DIRAC.exit(exitCode) + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_reschedule.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_reschedule.py index 1dd2731e8cf..40483652e07 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_reschedule.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_reschedule.py @@ -14,32 +14,40 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] - -result = dirac.rescheduleJob(parseArguments(args)) -if result['OK']: - print('Rescheduled job %s' % ','.join([str(j) for j in result['Value']])) -else: - errorList.append((j, result['Message'])) - print(result['Message']) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] + + result = dirac.rescheduleJob(parseArguments(args)) + if result['OK']: + print('Rescheduled job %s' % ','.join([str(j) for j in result['Value']])) + else: + errorList.append((j, result['Message'])) + print(result['Message']) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_status.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_status.py index 04290da1397..1cbb2f53070 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_status.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_status.py @@ -16,55 +16,63 @@ import DIRAC from DIRAC import exit as DIRACExit from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Utilities.Time import toString, date, day -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - '\nUsage:\n', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - '\nArguments:\n', - ' JobID: DIRAC Job ID'])) -Script.registerSwitch("f:", "File=", "Get status for jobs with IDs from the file") -Script.registerSwitch("g:", "JobGroup=", "Get status for jobs in the given group") +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + '\nUsage:\n', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + '\nArguments:\n', + ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() + Script.registerSwitch("f:", "File=", "Get status for jobs with IDs from the file") + Script.registerSwitch("g:", "JobGroup=", "Get status for jobs in the given group") -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -jobs = [] -for key, value in Script.getUnprocessedSwitches(): - if key.lower() in ('f', 'file'): - if os.path.exists(value): - jFile = open(value) - jobs += jFile.read().split() - jFile.close() - elif key.lower() in ('g', 'jobgroup'): - jobDate = toString(date() - 30 * day) - # Choose jobs no more than 30 days old - result = dirac.selectJobs(jobGroup=value, date=jobDate) - if not result['OK']: - print("Error:", result['Message']) - DIRACExit(-1) - jobs += result['Value'] + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 -if len(args) < 1 and not jobs: - Script.showHelp(exitCode=1) + jobs = [] + for key, value in Script.getUnprocessedSwitches(): + if key.lower() in ('f', 'file'): + if os.path.exists(value): + jFile = open(value) + jobs += jFile.read().split() + jFile.close() + elif key.lower() in ('g', 'jobgroup'): + jobDate = toString(date() - 30 * day) + # Choose jobs no more than 30 days old + result = dirac.selectJobs(jobGroup=value, date=jobDate) + if not result['OK']: + print("Error:", result['Message']) + DIRACExit(-1) + jobs += result['Value'] -if len(args) > 0: - jobs += parseArguments(args) + if len(args) < 1 and not jobs: + Script.showHelp(exitCode=1) -result = dirac.getJobStatus(jobs) -if result['OK']: - for job in result['Value']: - print('JobID=' + str(job), end=' ') - for status in result['Value'][job].items(): - print('%s=%s;' % status, end=' ') - print() -else: - exitCode = 2 - print("ERROR: %s" % result['Message']) + if len(args) > 0: + jobs += parseArguments(args) -DIRAC.exit(exitCode) + result = dirac.getJobStatus(jobs) + if result['OK']: + for job in result['Value']: + print('JobID=' + str(job), end=' ') + for status in result['Value'][job].items(): + print('%s=%s;' % status, end=' ') + print() + else: + exitCode = 2 + print("ERROR: %s" % result['Message']) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py index cd0d8a81b6b..c5e40fb7284 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py @@ -16,65 +16,71 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JDL ...' % Script.scriptName, + 'Arguments:', + ' JDL: Path to JDL file'])) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JDL ...' % Script.scriptName, - 'Arguments:', - ' JDL: Path to JDL file'])) - -Script.registerSwitch("f:", "File=", "Writes job ids to file ") -Script.registerSwitch("r:", "UseJobRepo=", "Use the job repository") -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC.Interfaces.API.Dirac import Dirac -unprocessed_switches = Script.getUnprocessedSwitches() -use_repo = False -repo_name = "" -for sw, value in unprocessed_switches: - if sw.lower() in ["r", "usejobrepo"]: - use_repo = True - repo_name = value - repo_name = repo_name.replace(".cfg", ".repo") -dirac = Dirac(use_repo, repo_name) -exitCode = 0 -errorList = [] - -jFile = None -for sw, value in unprocessed_switches: - if sw.lower() in ('f', 'file'): - if os.path.isfile(value): - print('Appending job ids to existing logfile: %s' % value) - if not os.access(value, os.W_OK): - print('Existing logfile %s must be writable by user.' % value) - jFile = open(value, 'a') - -for jdl in args: - - result = dirac.submitJob(jdl) - if result['OK']: - print('JobID = %s' % (result['Value'])) - if jFile is not None: - # parametric jobs - if isinstance(result['Value'], list): - jFile.write('\n'.join(str(p) for p in result['Value'])) - jFile.write('\n') - else: - jFile.write(str(result['Value']) + '\n') - else: - errorList.append((jdl, result['Message'])) - exitCode = 2 - -if jFile is not None: - jFile.close() - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) + Script.registerSwitch("f:", "File=", "Writes job ids to file ") + Script.registerSwitch("r:", "UseJobRepo=", "Use the job repository") + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC.Interfaces.API.Dirac import Dirac + unprocessed_switches = Script.getUnprocessedSwitches() + use_repo = False + repo_name = "" + for sw, value in unprocessed_switches: + if sw.lower() in ["r", "usejobrepo"]: + use_repo = True + repo_name = value + repo_name = repo_name.replace(".cfg", ".repo") + dirac = Dirac(use_repo, repo_name) + exitCode = 0 + errorList = [] + + jFile = None + for sw, value in unprocessed_switches: + if sw.lower() in ('f', 'file'): + if os.path.isfile(value): + print('Appending job ids to existing logfile: %s' % value) + if not os.access(value, os.W_OK): + print('Existing logfile %s must be writable by user.' % value) + jFile = open(value, 'a') + + for jdl in args: + + result = dirac.submitJob(jdl) + if result['OK']: + print('JobID = %s' % (result['Value'])) + if jFile is not None: + # parametric jobs + if isinstance(result['Value'], list): + jFile.write('\n'.join(str(p) for p in result['Value'])) + jFile.write('\n') + else: + jFile.write(str(result['Value']) + '\n') + else: + errorList.append((jdl, result['Message'])) + exitCode = 2 + + if jFile is not None: + jFile.close() + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py b/src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py index 9b6daa53357..fb6569930c8 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py @@ -17,97 +17,104 @@ import DIRAC from DIRAC.Core.Base import Script - -Script.registerSwitch("", "Status=", "Primary status") -Script.registerSwitch("", "MinorStatus=", "Secondary status") -Script.registerSwitch("", "ApplicationStatus=", "Application status") -Script.registerSwitch("", "Site=", "Execution site") -Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)") -Script.registerSwitch("", "JobGroup=", "Select jobs for specified job group") -Script.registerSwitch("", "Date=", "Date in YYYY-MM-DD format, if not specified default is today") -Script.registerSwitch("", "File=", "File name,if not specified default is std.out ") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... String ...' % Script.scriptName, - 'Arguments:', - ' String: string to search for'])) - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -# Default values -status = None -minorStatus = None -appStatus = None -site = None -owner = None -jobGroup = None -date = None -filename = 'std.out' - -if len(args) != 1: - Script.showHelp() - -searchstring = str(args[0]) - -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "status": - status = switch[1] - elif switch[0].lower() == "minorstatus": - minorStatus = switch[1] - elif switch[0].lower() == "applicationstatus": - appStatus = switch[1] - elif switch[0].lower() == "site": - site = switch[1] - elif switch[0].lower() == "owner": - owner = switch[1] - elif switch[0].lower() == "jobgroup": - jobGroup = switch[1] - elif switch[0].lower() == "date": - date = switch[1] - elif switch[0].lower() == "file": - filename = switch[1] - -selDate = date -if not date: - selDate = 'Today' - -from DIRAC.Interfaces.API.Dirac import Dirac - -dirac = Dirac() -exitCode = 0 -errorList = [] -resultDict = {} - -result = dirac.selectJobs(status=status, minorStatus=minorStatus, applicationStatus=appStatus, - site=site, owner=owner, jobGroup=jobGroup, date=date) -if result['OK']: - jobs = result['Value'] -else: - print("Error in selectJob", result['Message']) - DIRAC.exit(2) - -for job in jobs: - - result = dirac.getOutputSandbox(job) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch("", "Status=", "Primary status") + Script.registerSwitch("", "MinorStatus=", "Secondary status") + Script.registerSwitch("", "ApplicationStatus=", "Application status") + Script.registerSwitch("", "Site=", "Execution site") + Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)") + Script.registerSwitch("", "JobGroup=", "Select jobs for specified job group") + Script.registerSwitch("", "Date=", "Date in YYYY-MM-DD format, if not specified default is today") + Script.registerSwitch("", "File=", "File name,if not specified default is std.out ") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... String ...' % Script.scriptName, + 'Arguments:', + ' String: string to search for'])) + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + # Default values + status = None + minorStatus = None + appStatus = None + site = None + owner = None + jobGroup = None + date = None + filename = 'std.out' + + if len(args) != 1: + Script.showHelp() + + searchstring = str(args[0]) + + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "status": + status = switch[1] + elif switch[0].lower() == "minorstatus": + minorStatus = switch[1] + elif switch[0].lower() == "applicationstatus": + appStatus = switch[1] + elif switch[0].lower() == "site": + site = switch[1] + elif switch[0].lower() == "owner": + owner = switch[1] + elif switch[0].lower() == "jobgroup": + jobGroup = switch[1] + elif switch[0].lower() == "date": + date = switch[1] + elif switch[0].lower() == "file": + filename = switch[1] + + selDate = date + if not date: + selDate = 'Today' + + from DIRAC.Interfaces.API.Dirac import Dirac + + dirac = Dirac() + exitCode = 0 + errorList = [] + resultDict = {} + + result = dirac.selectJobs(status=status, minorStatus=minorStatus, applicationStatus=appStatus, + site=site, owner=owner, jobGroup=jobGroup, date=date) if result['OK']: - if os.path.exists('%s' % job): - - lines = [] - try: - lines = open(os.path.join(job, filename)).readlines() - except Exception as x: - errorList.append((job, x)) - for line in lines: - if line.count(searchstring): - resultDict[job] = line - rmtree("%s" % (job)) + jobs = result['Value'] else: - errorList.append((job, result['Message'])) - exitCode = 2 + print("Error in selectJob", result['Message']) + DIRAC.exit(2) + + for job in jobs: + + result = dirac.getOutputSandbox(job) + if result['OK']: + if os.path.exists('%s' % job): + + lines = [] + try: + lines = open(os.path.join(job, filename)).readlines() + except Exception as x: + errorList.append((job, x)) + for line in lines: + if line.count(searchstring): + resultDict[job] = line + rmtree("%s" % (job)) + else: + errorList.append((job, result['Message'])) + exitCode = 2 + + for result in resultDict.items(): + print(result) -for result in resultDict.items(): - print(result) + DIRAC.exit(exitCode) -DIRAC.exit(exitCode) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_select_jobs.py b/src/DIRAC/Interfaces/scripts/dirac_wms_select_jobs.py index 145ea6c7cd5..ac9f9199a2f 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_select_jobs.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_select_jobs.py @@ -14,120 +14,126 @@ import DIRAC from DIRAC import gLogger from DIRAC.Core.Base import Script - -maxJobs = 100 -Script.registerSwitch("", "Status=", "Primary status") -Script.registerSwitch("", "MinorStatus=", "Secondary status") -Script.registerSwitch("", "ApplicationStatus=", "Application status") -Script.registerSwitch("", "Site=", "Execution site") -Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)") -Script.registerSwitch("", "JobGroup=", "Select jobs for specified job group") -Script.registerSwitch("", "Date=", "Date in YYYY-MM-DD format, if not specified default is today") -Script.registerSwitch("", "Maximum=", "Maximum number of jobs shown (default %d, 0 means all)" % maxJobs) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName])) -Script.parseCommandLine(ignoreErrors=True) - -args = Script.getPositionalArgs() - -# Default values -status = None -minorStatus = None -appStatus = None -site = None -owner = None -jobGroups = [] -date = None - -if args: - Script.showHelp() - -exitCode = 0 - -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "status": - status = switch[1] - elif switch[0].lower() == "minorstatus": - minorStatus = switch[1] - elif switch[0].lower() == "applicationstatus": - appStatus = switch[1] - elif switch[0].lower() == "site": - site = switch[1] - elif switch[0].lower() == "owner": - owner = switch[1] - elif switch[0].lower() == "jobgroup": - for jg in switch[1].split(','): - if jg.isdigit(): - jobGroups.append('%08d' % int(jg)) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + maxJobs = 100 + Script.registerSwitch("", "Status=", "Primary status") + Script.registerSwitch("", "MinorStatus=", "Secondary status") + Script.registerSwitch("", "ApplicationStatus=", "Application status") + Script.registerSwitch("", "Site=", "Execution site") + Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)") + Script.registerSwitch("", "JobGroup=", "Select jobs for specified job group") + Script.registerSwitch("", "Date=", "Date in YYYY-MM-DD format, if not specified default is today") + Script.registerSwitch("", "Maximum=", "Maximum number of jobs shown (default %d, 0 means all)" % maxJobs) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName])) + Script.parseCommandLine(ignoreErrors=True) + + args = Script.getPositionalArgs() + + # Default values + status = None + minorStatus = None + appStatus = None + site = None + owner = None + jobGroups = [] + date = None + + if args: + Script.showHelp() + + exitCode = 0 + + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "status": + status = switch[1] + elif switch[0].lower() == "minorstatus": + minorStatus = switch[1] + elif switch[0].lower() == "applicationstatus": + appStatus = switch[1] + elif switch[0].lower() == "site": + site = switch[1] + elif switch[0].lower() == "owner": + owner = switch[1] + elif switch[0].lower() == "jobgroup": + for jg in switch[1].split(','): + if jg.isdigit(): + jobGroups.append('%08d' % int(jg)) + else: + jobGroups.append(jg) + elif switch[0].lower() == "date": + date = switch[1] + elif switch[0] == 'Maximum': + try: + maxJobs = int(switch[1]) + except TypeError: + gLogger.fatal("Invalid max number of jobs", switch[1]) + DIRAC.exit(1) + + selDate = date + if not date: + selDate = 'Today' + conditions = {'Status': status, + 'MinorStatus': minorStatus, + 'ApplicationStatus': appStatus, + 'Owner': owner, + 'JobGroup': ','.join(str(jg) for jg in jobGroups), + 'Date': selDate} + + from DIRAC.Interfaces.API.Dirac import Dirac + + dirac = Dirac() + jobs = [] + + if jobGroups: + for jobGroup in jobGroups: + res = dirac.selectJobs(status=status, + minorStatus=minorStatus, + applicationStatus=appStatus, + site=site, + owner=owner, + jobGroup=jobGroup, + date=date, + printErrors=False) + if res['OK']: + jobs.extend(res['Value']) else: - jobGroups.append(jg) - elif switch[0].lower() == "date": - date = switch[1] - elif switch[0] == 'Maximum': - try: - maxJobs = int(switch[1]) - except TypeError: - gLogger.fatal("Invalid max number of jobs", switch[1]) - DIRAC.exit(1) - -selDate = date -if not date: - selDate = 'Today' -conditions = {'Status': status, - 'MinorStatus': minorStatus, - 'ApplicationStatus': appStatus, - 'Owner': owner, - 'JobGroup': ','.join(str(jg) for jg in jobGroups), - 'Date': selDate} - - -from DIRAC.Interfaces.API.Dirac import Dirac - -dirac = Dirac() -jobs = [] - -if jobGroups: - for jobGroup in jobGroups: + gLogger.error("Can't select jobs: ", res['Message']) + else: res = dirac.selectJobs(status=status, minorStatus=minorStatus, applicationStatus=appStatus, site=site, owner=owner, - jobGroup=jobGroup, date=date, printErrors=False) if res['OK']: jobs.extend(res['Value']) else: gLogger.error("Can't select jobs: ", res['Message']) -else: - res = dirac.selectJobs(status=status, - minorStatus=minorStatus, - applicationStatus=appStatus, - site=site, - owner=owner, - date=date, - printErrors=False) - if res['OK']: - jobs.extend(res['Value']) + + conds = ['%s = %s' % (n, v) for n, v in conditions.items() if v] + if maxJobs and len(jobs) > maxJobs: + jobs = jobs[:maxJobs] + constrained = ' (first %d shown) ' % maxJobs else: - gLogger.error("Can't select jobs: ", res['Message']) + constrained = ' ' + if jobs: + gLogger.notice('==> Selected %s jobs%swith conditions: %s\n%s' % (len(jobs), + constrained, + ', '.join(conds), + ','.join(jobs))) + else: + gLogger.notice('No jobs were selected with conditions:', ', '.join(conds)) -conds = ['%s = %s' % (n, v) for n, v in conditions.items() if v] -if maxJobs and len(jobs) > maxJobs: - jobs = jobs[:maxJobs] - constrained = ' (first %d shown) ' % maxJobs -else: - constrained = ' ' + DIRAC.exit(exitCode) -if jobs: - gLogger.notice('==> Selected %s jobs%swith conditions: %s\n%s' % (len(jobs), - constrained, - ', '.join(conds), - ','.join(jobs))) -else: - gLogger.notice('No jobs were selected with conditions:', ', '.join(conds)) -DIRAC.exit(exitCode) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_add_trans.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_add_trans.py index 76e7f5ede1a..35bccf4a045 100755 --- a/src/DIRAC/ProductionSystem/scripts/dirac_prod_add_trans.py +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_add_trans.py @@ -12,70 +12,76 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID transID [parentTransID]' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID', - ' transID: Transformation ID', - ' parentTransID: Parent Transformation ID' - ])) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID transID [parentTransID]' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID', + ' transID: Transformation ID', + ' parentTransID: Parent Transformation ID' + ])) -Script.parseCommandLine() + Script.parseCommandLine() -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient -prodClient = ProductionClient() -transClient = TransformationClient() + prodClient = ProductionClient() + transClient = TransformationClient() -# get arguments -args = Script.getPositionalArgs() -if len(args) == 3: - parentTransID = args[2] -elif len(args) == 2: - parentTransID = '' -else: - Script.showHelp(exitCode=1) + # get arguments + args = Script.getPositionalArgs() + if len(args) == 3: + parentTransID = args[2] + elif len(args) == 2: + parentTransID = '' + else: + Script.showHelp(exitCode=1) -prodID = args[0] -transID = args[1] + prodID = args[0] + transID = args[1] -res = transClient.getTransformation(transID) -if not res['OK']: - DIRAC.gLogger.error('Failed to get transformation %s: %s' % (transID, res['Message'])) - DIRAC.exit(-1) + res = transClient.getTransformation(transID) + if not res['OK']: + DIRAC.gLogger.error('Failed to get transformation %s: %s' % (transID, res['Message'])) + DIRAC.exit(-1) + + transID = res['Value']['TransformationID'] -transID = res['Value']['TransformationID'] + if parentTransID: + res = transClient.getTransformation(parentTransID) + if not res['OK']: + DIRAC.gLogger.error('Failed to get transformation %s: %s' % (parentTransID, res['Message'])) + DIRAC.exit(-1) + parentTransID = res['Value']['TransformationID'] -if parentTransID: - res = transClient.getTransformation(parentTransID) + res = prodClient.getProduction(prodID) if not res['OK']: - DIRAC.gLogger.error('Failed to get transformation %s: %s' % (parentTransID, res['Message'])) + DIRAC.gLogger.error('Failed to get production %s: %s' % (prodID, res['Message'])) DIRAC.exit(-1) - parentTransID = res['Value']['TransformationID'] -res = prodClient.getProduction(prodID) -if not res['OK']: - DIRAC.gLogger.error('Failed to get production %s: %s' % (prodID, res['Message'])) - DIRAC.exit(-1) + prodID = res['Value']['ProductionID'] + res = prodClient.addTransformationsToProduction(prodID, transID, parentTransID) + if not res['OK']: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) -prodID = res['Value']['ProductionID'] -res = prodClient.addTransformationsToProduction(prodID, transID, parentTransID) -if not res['OK']: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) + if parentTransID: + msg = 'Transformation %s successfully added to production %s with parent transformation %s' % \ + (transID, prodID, parentTransID) + else: + msg = 'Transformation %s successfully added to production %s with no parent transformation' % \ + (transID, prodID) -if parentTransID: - msg = 'Transformation %s successfully added to production %s with parent transformation %s' % \ - (transID, prodID, parentTransID) -else: - msg = 'Transformation %s successfully added to production %s with no parent transformation' % \ - (transID, prodID) + DIRAC.gLogger.notice(msg) -DIRAC.gLogger.notice(msg) + DIRAC.exit(0) -DIRAC.exit(0) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_clean.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_clean.py index c2de670411e..77e37bf3754 100755 --- a/src/DIRAC/ProductionSystem/scripts/dirac_prod_clean.py +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_clean.py @@ -11,30 +11,37 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)' - ])) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)' + ])) -Script.parseCommandLine() + Script.parseCommandLine() -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) -# get arguments -prodID = args[0] + # get arguments + prodID = args[0] -res = ProductionClient().setProductionStatus(prodID, 'Cleaned') -if not res['OK']: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(1) + res = ProductionClient().setProductionStatus(prodID, 'Cleaned') + if not res['OK']: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(1) -DIRAC.gLogger.notice('Production %s successully cleaned' % prodID) -DIRAC.exit(0) + DIRAC.gLogger.notice('Production %s successully cleaned' % prodID) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_delete.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_delete.py index c75d6d992c1..1a14f95bdaf 100755 --- a/src/DIRAC/ProductionSystem/scripts/dirac_prod_delete.py +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_delete.py @@ -11,33 +11,40 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)' - ])) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)' + ])) -Script.parseCommandLine() + Script.parseCommandLine() -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) -# get arguments -prodID = args[0] + # get arguments + prodID = args[0] -prodClient = ProductionClient() + prodClient = ProductionClient() -res = prodClient.deleteProduction(prodID) -if res['OK']: - DIRAC.gLogger.notice('Production %s successully deleted' % prodID) -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) + res = prodClient.deleteProduction(prodID) + if res['OK']: + DIRAC.gLogger.notice('Production %s successully deleted' % prodID) + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) -DIRAC.exit(0) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_get.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get.py index 4abca1c93f8..b312e701711 100755 --- a/src/DIRAC/ProductionSystem/scripts/dirac_prod_get.py +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get.py @@ -11,59 +11,66 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Utilities.PrettyPrint import printTable -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)' - ])) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)' + ])) -Script.parseCommandLine() + Script.parseCommandLine() -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient -prodClient = ProductionClient() + prodClient = ProductionClient() -# get arguments -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) -else: - prodID = args[0] - res = prodClient.getProduction(prodID) + # get arguments + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) + else: + prodID = args[0] + res = prodClient.getProduction(prodID) -fields = [ - 'ProductionName', - 'Status', - 'ProductionID', - 'CreationDate', - 'LastUpdate', - 'AuthorDN', - 'AuthorGroup'] -records = [] + fields = [ + 'ProductionName', + 'Status', + 'ProductionID', + 'CreationDate', + 'LastUpdate', + 'AuthorDN', + 'AuthorGroup'] + records = [] -if res['OK']: - prodList = res['Value'] - if not isinstance(res['Value'], list): - prodList = [res['Value']] - for prod in prodList: - records.append( - [ - str( - prod['ProductionName']), str( - prod['Status']), str( - prod['ProductionID']), str( - prod['CreationDate']), str( - prod['LastUpdate']), str( - prod['AuthorDN']), str( - prod['AuthorGroup'])]) -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) + if res['OK']: + prodList = res['Value'] + if not isinstance(res['Value'], list): + prodList = [res['Value']] + for prod in prodList: + records.append( + [ + str( + prod['ProductionName']), str( + prod['Status']), str( + prod['ProductionID']), str( + prod['CreationDate']), str( + prod['LastUpdate']), str( + prod['AuthorDN']), str( + prod['AuthorGroup'])]) + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) -printTable(fields, records) + printTable(fields, records) -DIRAC.exit(0) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py index 637cf7f776e..2966fcfb415 100755 --- a/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py @@ -12,45 +12,53 @@ import DIRAC from DIRAC.Core.Base import Script from DIRAC.Core.Utilities.PrettyPrint import printTable +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1]])) - -Script.parseCommandLine() - -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient - -prodClient = ProductionClient() -res = prodClient.getProductions() - -fields = [ - 'ProductionName', - 'Status', - 'ProductionID', - 'CreationDate', - 'LastUpdate', - 'AuthorDN', - 'AuthorGroup'] -records = [] - -if res['OK']: - prodList = res['Value'] - if not isinstance(res['Value'], list): - prodList = [res['Value']] - for prod in prodList: - records.append( - [ - str( - prod['ProductionName']), str( - prod['Status']), str( - prod['ProductionID']), str( - prod['CreationDate']), str( - prod['LastUpdate']), str( - prod['AuthorDN']), str( - prod['AuthorGroup'])]) -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) - -printTable(fields, records) - -DIRAC.exit(0) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1]])) + + Script.parseCommandLine() + + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + + prodClient = ProductionClient() + res = prodClient.getProductions() + + fields = [ + 'ProductionName', + 'Status', + 'ProductionID', + 'CreationDate', + 'LastUpdate', + 'AuthorDN', + 'AuthorGroup'] + records = [] + + if res['OK']: + prodList = res['Value'] + if not isinstance(res['Value'], list): + prodList = [res['Value']] + for prod in prodList: + records.append( + [ + str( + prod['ProductionName']), str( + prod['Status']), str( + prod['ProductionID']), str( + prod['CreationDate']), str( + prod['LastUpdate']), str( + prod['AuthorDN']), str( + prod['AuthorGroup'])]) + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) + + printTable(fields, records) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_description.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_description.py index be937db62ec..fe45903153b 100755 --- a/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_description.py +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_description.py @@ -11,36 +11,43 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)' - ])) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)' + ])) -Script.parseCommandLine() + Script.parseCommandLine() -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient -prodClient = ProductionClient() + prodClient = ProductionClient() -# get arguments -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) -else: - prodID = args[0] - res = prodClient.getProduction(prodID) + # get arguments + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) + else: + prodID = args[0] + res = prodClient.getProduction(prodID) -if res['OK']: - prod = res['Value'] -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) + if res['OK']: + prod = res['Value'] + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) -print('Description for production %s:\n' % prodID) -print(prod['Description']) + print('Description for production %s:\n' % prodID) + print(prod['Description']) -DIRAC.exit(0) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py index 4e9a276b5bb..c4335998215 100755 --- a/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py @@ -12,88 +12,92 @@ import DIRAC from DIRAC.Core.Base import Script from DIRAC.Core.Utilities.PrettyPrint import printTable +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)', + '\ne.g: %s 381' % Script.scriptName, + ])) + + Script.parseCommandLine() + + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) + + # get arguments + prodID = args[0] + + prodClient = ProductionClient() + transClient = TransformationClient() + + res = prodClient.getProductionTransformations(prodID) + transIDs = [] + + if res['OK']: + transList = res['Value'] + if not transList: + DIRAC.gLogger.notice('No transformation associated with production %s' % prodID) + DIRAC.exit(-1) + for trans in transList: + transIDs.append(trans['TransformationID']) + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)', - '\ne.g: %s 381' % Script.scriptName, - ])) + fields = [ + 'TransformationName', + 'Status', + 'F_Proc.', + 'F_Proc.(%)', + 'TransformationID', + 'ProductionID', + 'Prod_LastUpdate', + 'Prod_InsertedTime'] + records = [] -Script.parseCommandLine() + paramShowNames = ['TransformationID', 'TransformationName', 'Type', 'Status', 'Files_Total', 'Files_PercentProcessed', + 'Files_Processed', 'Files_Unused', 'Jobs_TotalCreated', 'Jobs_Waiting', + 'Jobs_Running', 'Jobs_Done', 'Jobs_Failed', 'Jobs_Stalled'] + resList = [] -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + res = transClient.getTransformationSummaryWeb({'TransformationID': transIDs}, [], 0, len(transIDs)) -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) + if not res['OK']: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) -# get arguments -prodID = args[0] + if res['Value']['TotalRecords'] > 0: + paramNames = res['Value']['ParameterNames'] + for paramValues in res['Value']['Records']: + paramShowValues = map(lambda pname: paramValues[paramNames.index(pname)], paramShowNames) + showDict = dict(zip(paramShowNames, paramShowValues)) + resList.append(showDict) -prodClient = ProductionClient() -transClient = TransformationClient() + for res in resList: + files_Processed = res['Files_Processed'] + files_PercentProcessed = res['Files_PercentProcessed'] + status = res['Status'] + type = res['Type'] + transName = res['TransformationName'] + transID = res['TransformationID'] + records.append([transName, status, str(files_Processed), str(files_PercentProcessed), str(transID), str(prodID), + str(trans['LastUpdate']), str(trans['InsertedTime'])]) -res = prodClient.getProductionTransformations(prodID) -transIDs = [] + printTable(fields, records) -if res['OK']: - transList = res['Value'] - if not transList: - DIRAC.gLogger.notice('No transformation associated with production %s' % prodID) - DIRAC.exit(-1) - for trans in transList: - transIDs.append(trans['TransformationID']) -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) - - -fields = [ - 'TransformationName', - 'Status', - 'F_Proc.', - 'F_Proc.(%)', - 'TransformationID', - 'ProductionID', - 'Prod_LastUpdate', - 'Prod_InsertedTime'] - -records = [] - -paramShowNames = ['TransformationID', 'TransformationName', 'Type', 'Status', 'Files_Total', 'Files_PercentProcessed', - 'Files_Processed', 'Files_Unused', 'Jobs_TotalCreated', 'Jobs_Waiting', - 'Jobs_Running', 'Jobs_Done', 'Jobs_Failed', 'Jobs_Stalled'] -resList = [] - - -res = transClient.getTransformationSummaryWeb({'TransformationID': transIDs}, [], 0, len(transIDs)) - -if not res['OK']: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) - -if res['Value']['TotalRecords'] > 0: - paramNames = res['Value']['ParameterNames'] - for paramValues in res['Value']['Records']: - paramShowValues = map(lambda pname: paramValues[paramNames.index(pname)], paramShowNames) - showDict = dict(zip(paramShowNames, paramShowValues)) - resList.append(showDict) - -for res in resList: - files_Processed = res['Files_Processed'] - files_PercentProcessed = res['Files_PercentProcessed'] - status = res['Status'] - type = res['Type'] - transName = res['TransformationName'] - transID = res['TransformationID'] - records.append([transName, status, str(files_Processed), str(files_PercentProcessed), str(transID), str(prodID), - str(trans['LastUpdate']), str(trans['InsertedTime'])]) - - -printTable(fields, records) - -DIRAC.exit(0) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_start.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_start.py index 32abec0037e..b55f2656767 100755 --- a/src/DIRAC/ProductionSystem/scripts/dirac_prod_start.py +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_start.py @@ -11,33 +11,40 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)' - ])) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)' + ])) -Script.parseCommandLine() + Script.parseCommandLine() -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) -# get arguments -prodID = args[0] + # get arguments + prodID = args[0] -prodClient = ProductionClient() + prodClient = ProductionClient() -res = prodClient.setProductionStatus(prodID, 'Active') -if res['OK']: - DIRAC.gLogger.notice('Production %s successully started' % prodID) -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) + res = prodClient.setProductionStatus(prodID, 'Active') + if res['OK']: + DIRAC.gLogger.notice('Production %s successully started' % prodID) + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) -DIRAC.exit(0) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_stop.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_stop.py index 00be7945736..03b163a0d0a 100755 --- a/src/DIRAC/ProductionSystem/scripts/dirac_prod_stop.py +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_stop.py @@ -11,33 +11,40 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)' - ])) +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)' + ])) -Script.parseCommandLine() + Script.parseCommandLine() -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) -# get arguments -prodID = args[0] + # get arguments + prodID = args[0] -prodClient = ProductionClient() + prodClient = ProductionClient() -res = prodClient.setProductionStatus(prodID, 'Stopped') -if res['OK']: - DIRAC.gLogger.notice('Production %s successully stopped' % prodID) -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) + res = prodClient.setProductionStatus(prodID, 'Stopped') + if res['OK']: + DIRAC.gLogger.notice('Production %s successully stopped' % prodID) + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) -DIRAC.exit(0) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_list_req_cache.py b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_list_req_cache.py index 8a2a1fb8372..6e50f8bd881 100755 --- a/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_list_req_cache.py +++ b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_list_req_cache.py @@ -7,15 +7,16 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Base.Script import parseCommandLine -Script.registerSwitch('', 'Full', ' Print full list of requests') -Script.setUsageMessage('\n'.join([__doc__, 'Usage:', ' %s [option|cfgfile]' % Script.scriptName])) -parseCommandLine() -from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient - -if __name__ == "__main__": +@DIRACScript() +def main(): + Script.registerSwitch('', 'Full', ' Print full list of requests') + Script.setUsageMessage('\n'.join([__doc__, 'Usage:', ' %s [option|cfgfile]' % Script.scriptName])) + parseCommandLine() + from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient fullPrint = False @@ -39,3 +40,7 @@ DIRAC.gLogger.always("Number of requests in the cache", len(reqCache)) DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_reqdb_summary.py b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_reqdb_summary.py index 7b6c200d5a7..68c7ddfe33b 100755 --- a/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_reqdb_summary.py +++ b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_reqdb_summary.py @@ -6,11 +6,14 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s [option|cfgfile]' % Script.scriptName])) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if __name__ == "__main__": + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__, + 'Usage:', + ' %s [option|cfgfile]' % Script.scriptName])) from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() @@ -47,3 +50,7 @@ DIRAC.gLogger.always("- '%s' %s" % (fState, fCount)) DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_request.py b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_request.py index 7361a18e522..bc6488f49e2 100755 --- a/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_request.py +++ b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_request.py @@ -7,6 +7,7 @@ import datetime import os +from DIRAC.Core.Utilities.DIRACScript import DIRACScript def convertDate(date): @@ -18,39 +19,40 @@ def convertDate(date): try: value = datetime.datetime.utcnow() - datetime.timedelta(hours=int(24 * float(date))) except Exception: + from DIRAC import gLogger gLogger.fatal("Invalid date", date) value = None return value -from DIRAC.Core.Base import Script -Script.registerSwitch('', 'Job=', ' JobID[,jobID2,...]') -Script.registerSwitch('', 'Transformation=', ' transformation ID') -Script.registerSwitch('', 'Tasks=', ' Associated to --Transformation, list of taskIDs') -Script.registerSwitch('', 'Verbose', ' Print more information') -Script.registerSwitch('', 'Terse', ' Only print request status') -Script.registerSwitch('', 'Full', ' Print full request content') -Script.registerSwitch('', 'Status=', ' Select all requests in a given status') -Script.registerSwitch('', 'Since=', - ' Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day') -Script.registerSwitch('', 'Until=', ' Associated to --Status, end date (default= now') -Script.registerSwitch('', 'Maximum=', ' Associated to --Status, max number of requests ') -Script.registerSwitch('', 'Reset', ' Reset Failed files to Waiting if any') -Script.registerSwitch('', 'Force', ' Force reset even if not Failed') -Script.registerSwitch('', 'All', ' (if --Status Failed) all requests, otherwise exclude irrecoverable failures') -Script.registerSwitch('', 'FixJob', ' Set job Done if the request is Done') -Script.registerSwitch('', 'Cancel', ' Cancel the request') -Script.registerSwitch('', 'ListJobs', ' List the corresponding jobs') -Script.registerSwitch('', 'TargetSE=', ' Select request only if that SE is in the targetSEs') -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s [option|cfgfile] [request[,request1,...]|' % Script.scriptName, - 'Arguments:', - ' request: a request ID or a unique request name', - ' : a file containing a list of requests (Comma-separated on each line)'])) +@DIRACScript() +def main(): + from DIRAC.Core.Base import Script + Script.registerSwitch('', 'Job=', ' JobID[,jobID2,...]') + Script.registerSwitch('', 'Transformation=', ' transformation ID') + Script.registerSwitch('', 'Tasks=', ' Associated to --Transformation, list of taskIDs') + Script.registerSwitch('', 'Verbose', ' Print more information') + Script.registerSwitch('', 'Terse', ' Only print request status') + Script.registerSwitch('', 'Full', ' Print full request content') + Script.registerSwitch('', 'Status=', ' Select all requests in a given status') + Script.registerSwitch('', 'Since=', + ' Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day') + Script.registerSwitch('', 'Until=', ' Associated to --Status, end date (default= now') + Script.registerSwitch('', 'Maximum=', ' Associated to --Status, max number of requests ') + Script.registerSwitch('', 'Reset', ' Reset Failed files to Waiting if any') + Script.registerSwitch('', 'Force', ' Force reset even if not Failed') + Script.registerSwitch('', 'All', ' (if --Status Failed) all requests, otherwise exclude irrecoverable failures') + Script.registerSwitch('', 'FixJob', ' Set job Done if the request is Done') + Script.registerSwitch('', 'Cancel', ' Cancel the request') + Script.registerSwitch('', 'ListJobs', ' List the corresponding jobs') + Script.registerSwitch('', 'TargetSE=', ' Select request only if that SE is in the targetSEs') + Script.setUsageMessage('\n'.join([__doc__, + 'Usage:', + ' %s [option|cfgfile] [request[,request1,...]|' % Script.scriptName, + 'Arguments:', + ' request: a request ID or a unique request name', + ' : a file containing a list of requests (Comma-separated on each line)'])) -# # execution -if __name__ == "__main__": from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() @@ -274,3 +276,7 @@ def convertDate(date): gLogger.notice('\nList of %d selected requests:' % len(okRequests)) for reqs in breakListIntoChunks(okRequests, 100): gLogger.notice(','.join(reqs)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py index 93405d5af58..2be8149d4b2 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py @@ -15,6 +15,7 @@ from DIRAC import gLogger, exit as DIRACExit, version from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.ResourceStatusSystem.Client import ResourceStatusClient from DIRAC.Core.Utilities.PrettyPrint import printTable @@ -153,8 +154,8 @@ def run(): tabularPrint(elements) -if __name__ == "__main__": - +@DIRACScript() +def main(): subLogger = gLogger.getSubLogger(__file__) # Script initialization @@ -167,3 +168,7 @@ def run(): # Bye DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py index 257679020df..532a21c3d93 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py @@ -23,6 +23,7 @@ import datetime from DIRAC import gLogger, exit as DIRACExit, S_OK, version from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.ResourceStatusSystem.Client import ResourceStatusClient from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Security.ProxyInfo import getProxyInfo @@ -387,7 +388,6 @@ def run(args, switchDictSet): """ Main function of the script """ - query = args[0] matches = 0 @@ -410,8 +410,8 @@ def run(args, switchDictSet): confirm(query, matches) -if __name__ == "__main__": - +@DIRACScript() +def main(): subLogger = gLogger.getSubLogger(__file__) # Script initialization @@ -427,3 +427,7 @@ def run(args, switchDictSet): # Bye DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py index 4ec9ff2f2f2..27eefbed06a 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py @@ -21,6 +21,7 @@ from DIRAC import gLogger, exit as DIRACExit, version from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Utilities import Time from DIRAC.Core.Utilities.PrettyPrint import printTable from DIRAC.ResourceStatusSystem.Utilities import Utils @@ -338,7 +339,9 @@ def run(args, switchDict): error(result['Message']) -if __name__ == "__main__": +@DIRACScript() +def main(): + global ResourceManagementClient subLogger = gLogger.getSubLogger(__file__) @@ -346,12 +349,18 @@ def run(args, switchDict): registerSwitches() registerUsageMessage() args, switchDict = parseSwitches() + ResourceManagementClient = getattr( Utils.voimport('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient'), - 'ResourceManagementClient') + 'ResourceManagementClient', + ) # Run script run(args, switchDict) # Bye DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py index 9fba2a9166a..61c291f19b1 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py @@ -14,6 +14,7 @@ from DIRAC import gLogger, exit as DIRACExit, S_OK, version from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.ResourceStatusSystem.Client import ResourceStatusClient from DIRAC.ResourceStatusSystem.PolicySystem import StateMachine @@ -232,8 +233,8 @@ def run(switchDict): DIRACExit(1) -if __name__ == "__main__": - +@DIRACScript() +def main(): subLogger = gLogger.getSubLogger(__file__) # Script initialization @@ -249,5 +250,6 @@ def run(switchDict): # Bye DIRACExit(0) -################################################################################ -# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py index cd447543343..db227f46189 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py @@ -21,6 +21,7 @@ # DIRAC from DIRAC import gLogger, exit as DIRACExit, S_OK, version from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient @@ -175,11 +176,19 @@ def setToken(user): return S_OK() +@DIRACScript() def main(): """ Main function of the script. Gets the username from the proxy loaded and sets the token taking into account that user and the switchDict parameters. """ + # Logger initialization + subLogger = gLogger.getSubLogger(__file__) + + # Script initialization + registerSwitches() + registerUsageMessage() + switchDict = parseSwitches() user = proxyUser() if not user['OK']: @@ -192,17 +201,8 @@ def main(): subLogger.error(res['Message']) DIRACExit(1) + DIRACExit(0) -if __name__ == '__main__': - - # Logger initialization - subLogger = gLogger.getSubLogger(__file__) - - # Script initialization - registerSwitches() - registerUsageMessage() - switchDict = parseSwitches() +if __name__ == "__main__": main() - - DIRACExit(0) diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py index d92f3c0c011..78817687abf 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py @@ -20,6 +20,7 @@ from DIRAC import version, gLogger, exit as DIRACExit, S_OK from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript subLogger = None switchDict = {} @@ -256,10 +257,14 @@ def run(): DIRACExit(1) -if __name__ == "__main__": - +@DIRACScript() +def main(): # Run script run() # Bye DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_file.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_file.py index d692338660a..92c6bcffd2f 100755 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_file.py +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_file.py @@ -20,73 +20,82 @@ from __future__ import print_function __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s LFN SE ...' % Script.scriptName, - 'Arguments:', - ' LFN: LFN of the staging file', - ' SE: Storage Element for the staging file' - ])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 2: - Script.showHelp() - -from DIRAC import exit as DIRACExit, gLogger - -lfn = args[0] -se = args[1] - -from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient -client = StorageManagerClient() -res = client.getCacheReplicas({'LFN': lfn, 'SE': se}) -if not res['OK']: - gLogger.error(res['Message']) -cacheReplicaInfo = res['Value'] -if cacheReplicaInfo: - replicaID = list(cacheReplicaInfo)[0] - outStr = "\n--------------------" - outStr += "\n%s: %s" % ('LFN'.ljust(8), cacheReplicaInfo[replicaID]['LFN'].ljust(100)) - outStr += "\n%s: %s" % ('SE'.ljust(8), cacheReplicaInfo[replicaID]['SE'].ljust(100)) - outStr += "\n%s: %s" % ('PFN'.ljust(8), cacheReplicaInfo[replicaID]['PFN'].ljust(100)) - outStr += "\n%s: %s" % ('Status'.ljust(8), cacheReplicaInfo[replicaID]['Status'].ljust(100)) - outStr += "\n%s: %s" % ('LastUpdate'.ljust(8), str(cacheReplicaInfo[replicaID]['LastUpdate']).ljust(100)) - outStr += "\n%s: %s" % ('Reason'.ljust(8), str(cacheReplicaInfo[replicaID]['Reason']).ljust(100)) - - resTasks = client.getTasks({'ReplicaID': replicaID}) - - if resTasks['OK']: - # print resTasks['Message'] - outStr += '\nJob IDs requesting this file to be staged:'.ljust(8) - tasks = resTasks['Value'] - for tid in tasks.keys(): - outStr += ' %s ' % (tasks[tid]['SourceTaskID']) - - resStageRequests = client.getStageRequests({'ReplicaID': replicaID}) - - if not resStageRequests['OK']: - gLogger.error(resStageRequests['Message']) - - if resStageRequests['Records']: - stageRequests = resStageRequests['Value'] - outStr += "\n------SRM staging request info--------------" - for info in stageRequests.values(): - outStr += "\n%s: %s" % ('SRM RequestID'.ljust(8), info['RequestID'].ljust(100)) - outStr += "\n%s: %s" % ('SRM StageStatus'.ljust(8), info['StageStatus'].ljust(100)) - outStr += "\n%s: %s" % ('SRM StageRequestSubmitTime'.ljust(8), str(info['StageRequestSubmitTime']).ljust(100)) - outStr += "\n%s: %s" % ('SRM StageRequestCompletedTime'.ljust(8), - str(info['StageRequestCompletedTime']).ljust(100)) - outStr += "\n%s: %s" % ('SRM PinExpiryTime'.ljust(8), str(info['PinExpiryTime']).ljust(100)) - outStr += "\n%s: %s sec" % ('SRM PinLength'.ljust(8), str(info['PinLength']).ljust(100)) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__, + 'Usage:', + ' %s LFN SE ...' % Script.scriptName, + 'Arguments:', + ' LFN: LFN of the staging file', + ' SE: Storage Element for the staging file' + ])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + if len(args) < 2: + Script.showHelp() + + from DIRAC import exit as DIRACExit, gLogger + + lfn = args[0] + se = args[1] + + from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + client = StorageManagerClient() + res = client.getCacheReplicas({'LFN': lfn, 'SE': se}) + if not res['OK']: + gLogger.error(res['Message']) + cacheReplicaInfo = res['Value'] + if cacheReplicaInfo: + replicaID = list(cacheReplicaInfo)[0] + outStr = "\n--------------------" + outStr += "\n%s: %s" % ('LFN'.ljust(8), cacheReplicaInfo[replicaID]['LFN'].ljust(100)) + outStr += "\n%s: %s" % ('SE'.ljust(8), cacheReplicaInfo[replicaID]['SE'].ljust(100)) + outStr += "\n%s: %s" % ('PFN'.ljust(8), cacheReplicaInfo[replicaID]['PFN'].ljust(100)) + outStr += "\n%s: %s" % ('Status'.ljust(8), cacheReplicaInfo[replicaID]['Status'].ljust(100)) + outStr += "\n%s: %s" % ('LastUpdate'.ljust(8), str(cacheReplicaInfo[replicaID]['LastUpdate']).ljust(100)) + outStr += "\n%s: %s" % ('Reason'.ljust(8), str(cacheReplicaInfo[replicaID]['Reason']).ljust(100)) + + resTasks = client.getTasks({'ReplicaID': replicaID}) + + if resTasks['OK']: + # print resTasks['Message'] + outStr += '\nJob IDs requesting this file to be staged:'.ljust(8) + tasks = resTasks['Value'] + for tid in tasks.keys(): + outStr += ' %s ' % (tasks[tid]['SourceTaskID']) + + resStageRequests = client.getStageRequests({'ReplicaID': replicaID}) + + if not resStageRequests['OK']: + gLogger.error(resStageRequests['Message']) + + if resStageRequests['Records']: + stageRequests = resStageRequests['Value'] + outStr += "\n------SRM staging request info--------------" + for info in stageRequests.values(): + outStr += "\n%s: %s" % ('SRM RequestID'.ljust(8), info['RequestID'].ljust(100)) + outStr += "\n%s: %s" % ('SRM StageStatus'.ljust(8), info['StageStatus'].ljust(100)) + outStr += "\n%s: %s" % ('SRM StageRequestSubmitTime'.ljust(8), str(info['StageRequestSubmitTime']).ljust(100)) + outStr += "\n%s: %s" % ('SRM StageRequestCompletedTime'.ljust(8), + str(info['StageRequestCompletedTime']).ljust(100)) + outStr += "\n%s: %s" % ('SRM PinExpiryTime'.ljust(8), str(info['PinExpiryTime']).ljust(100)) + outStr += "\n%s: %s sec" % ('SRM PinLength'.ljust(8), str(info['PinLength']).ljust(100)) + else: + outStr += '\nThere are no staging requests submitted to the site yet.'.ljust(8) else: - outStr += '\nThere are no staging requests submitted to the site yet.'.ljust(8) -else: - outStr = "\nThere is no such file requested for staging. Check for typo's!" - # Script.showHelp() -gLogger.notice(outStr) + outStr = "\nThere is no such file requested for staging. Check for typo's!" + # Script.showHelp() + gLogger.notice(outStr) + + DIRACExit(0) + + +if __name__ == "__main__": + main() -DIRACExit(0) ''' Example: dirac-stager-monitor-file.py /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst GRIDKA-RDST @@ -105,4 +114,4 @@ SRM StageRequestCompletedTime: None SRM PinExpiryTime: None SRM PinLength: 43200 -''' +''' # NOQA diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_jobs.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_jobs.py index 973a19dbd90..11b102977e0 100755 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_jobs.py +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_jobs.py @@ -12,62 +12,71 @@ from __future__ import print_function __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s jobID [jobID] [jobID] ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC job ID \n' - ])) -Script.parseCommandLine(ignoreErrors=False) -args = Script.getPositionalArgs() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s jobID [jobID] [jobID] ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC job ID \n' + ])) + Script.parseCommandLine(ignoreErrors=False) -if len(args) < 1: - Script.showHelp() + args = Script.getPositionalArgs() -from DIRAC import exit as DIRACExit, gLogger + if len(args) < 1: + Script.showHelp() -try: - jobIDs = [int(arg) for arg in args] -except BaseException: - gLogger.fatal('DIRAC Job IDs must be integers') - DIRACExit(2) + from DIRAC import exit as DIRACExit, gLogger -from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient -client = StorageManagerClient() + try: + jobIDs = [int(arg) for arg in args] + except BaseException: + gLogger.fatal('DIRAC Job IDs must be integers') + DIRACExit(2) + + from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + client = StorageManagerClient() + + outStr = "\n" + for jobID in jobIDs: + res = client.getTaskSummary(jobID) + if not res['OK']: + gLogger.error(res['Message']) + continue + if not res['Value']: + gLogger.notice('No info for job %s, probably gone from the stager...' % jobID) + continue + taskInfo = res['Value']['TaskInfo'] + replicaInfo = res['Value']['ReplicaInfo'] + outStr = "%s: %s" % ('JobID'.ljust(20), jobID) + outStr += "\n%s: %s" % ('Status'.ljust(20), taskInfo[str(jobID)]['Status']) + outStr += "\n%s: %s" % ('SubmitTime'.ljust(20), taskInfo[str(jobID)]['SubmitTime']) + outStr += "\n%s: %s" % ('CompleteTime'.ljust(20), taskInfo[str(jobID)]['CompleteTime']) + outStr += "\nStaging files for this job:" + if not res['Value']['ReplicaInfo']: + gLogger.notice('No info on files for the job = %s, that is odd' % jobID) + continue + else: + for lfn, metadata in replicaInfo.items(): + outStr += "\n\t--------------------" + outStr += "\n\t%s: %s" % ('LFN'.ljust(8), lfn.ljust(100)) + outStr += "\n\t%s: %s" % ('SE'.ljust(8), metadata['StorageElement'].ljust(100)) + outStr += "\n\t%s: %s" % ('PFN'.ljust(8), str(metadata['PFN']).ljust(100)) + outStr += "\n\t%s: %s" % ('Status'.ljust(8), metadata['Status'].ljust(100)) + outStr += "\n\t%s: %s" % ('Reason'.ljust(8), str(metadata['Reason']).ljust(100)) + outStr += "\n%s: %s" % ('LastUpdate'.ljust(8), str(metadata['LastUpdate']).ljust(100)) + outStr += "\n----------------------" + gLogger.notice(outStr) + DIRACExit(0) + + +if __name__ == "__main__": + main() -outStr = "\n" -for jobID in jobIDs: - res = client.getTaskSummary(jobID) - if not res['OK']: - gLogger.error(res['Message']) - continue - if not res['Value']: - gLogger.notice('No info for job %s, probably gone from the stager...' % jobID) - continue - taskInfo = res['Value']['TaskInfo'] - replicaInfo = res['Value']['ReplicaInfo'] - outStr = "%s: %s" % ('JobID'.ljust(20), jobID) - outStr += "\n%s: %s" % ('Status'.ljust(20), taskInfo[str(jobID)]['Status']) - outStr += "\n%s: %s" % ('SubmitTime'.ljust(20), taskInfo[str(jobID)]['SubmitTime']) - outStr += "\n%s: %s" % ('CompleteTime'.ljust(20), taskInfo[str(jobID)]['CompleteTime']) - outStr += "\nStaging files for this job:" - if not res['Value']['ReplicaInfo']: - gLogger.notice('No info on files for the job = %s, that is odd' % jobID) - continue - else: - for lfn, metadata in replicaInfo.items(): - outStr += "\n\t--------------------" - outStr += "\n\t%s: %s" % ('LFN'.ljust(8), lfn.ljust(100)) - outStr += "\n\t%s: %s" % ('SE'.ljust(8), metadata['StorageElement'].ljust(100)) - outStr += "\n\t%s: %s" % ('PFN'.ljust(8), str(metadata['PFN']).ljust(100)) - outStr += "\n\t%s: %s" % ('Status'.ljust(8), metadata['Status'].ljust(100)) - outStr += "\n\t%s: %s" % ('Reason'.ljust(8), str(metadata['Reason']).ljust(100)) - outStr += "\n%s: %s" % ('LastUpdate'.ljust(8), str(metadata['LastUpdate']).ljust(100)) - outStr += "\n----------------------" - gLogger.notice(outStr) -DIRACExit(0) ''' Example: dirac-stager-monitor-jobs.py 5688643 5688644 @@ -138,5 +147,4 @@ Status : Offline Reason : None -------------------- - -''' +''' # NOQA diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_request.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_request.py index 4d6c10a8986..d9d46033334 100755 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_request.py +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_request.py @@ -13,49 +13,57 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Request ...' % Script.scriptName, - 'Arguments:', - ' Request: ID of the Stage request in the StorageManager'])) -Script.parseCommandLine(ignoreErrors=False) - -args = Script.getPositionalArgs() - -if not len(args) == 1: - Script.showHelp() - -from DIRAC import exit as DIRACExit, gLogger - -try: - taskID = int(args[0]) -except BaseException: - gLogger.fatal('Stage requestID must be an integer') - DIRACExit(2) - -from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient -client = StorageManagerClient() - -res = client.getTaskSummary(taskID) -if not res['OK']: - gLogger.error(res['Message']) - DIRACExit(2) -taskInfo = res['Value']['TaskInfo'] -replicaInfo = res['Value']['ReplicaInfo'] -outStr = "%s: %s" % ('TaskID'.ljust(20), taskID) -outStr += "\n%s: %s" % ('Status'.ljust(20), taskInfo[taskID]['Status']) -outStr += "\n%s: %s" % ('Source'.ljust(20), taskInfo[taskID]['Source']) -outStr += "\n%s: %s" % ('SourceTaskID'.ljust(20), taskInfo[taskID]['SourceTaskID']) -outStr += "\n%s: %s" % ('CallBackMethod'.ljust(20), taskInfo[taskID]['CallBackMethod']) -outStr += "\n%s: %s" % ('SubmitTime'.ljust(20), taskInfo[taskID]['SubmitTime']) -outStr += "\n%s: %s" % ('CompleteTime'.ljust(20), taskInfo[taskID]['CompleteTime']) -for lfn, metadata in replicaInfo.items(): - outStr += "\n" - outStr += "\n\t%s: %s" % ('LFN'.ljust(8), lfn.ljust(100)) - outStr += "\n\t%s: %s" % ('SE'.ljust(8), metadata['StorageElement'].ljust(100)) - outStr += "\n\t%s: %s" % ('PFN'.ljust(8), str(metadata['PFN']).ljust(100)) - outStr += "\n\t%s: %s" % ('Size'.ljust(8), str(metadata['FileSize']).ljust(100)) - outStr += "\n\t%s: %s" % ('Status'.ljust(8), metadata['Status'].ljust(100)) - outStr += "\n\t%s: %s" % ('Reason'.ljust(8), str(metadata['Reason']).ljust(100)) -gLogger.notice(outStr) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Request ...' % Script.scriptName, + 'Arguments:', + ' Request: ID of the Stage request in the StorageManager'])) + Script.parseCommandLine(ignoreErrors=False) + + args = Script.getPositionalArgs() + + if not len(args) == 1: + Script.showHelp() + + from DIRAC import exit as DIRACExit, gLogger + + try: + taskID = int(args[0]) + except BaseException: + gLogger.fatal('Stage requestID must be an integer') + DIRACExit(2) + + from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + client = StorageManagerClient() + + res = client.getTaskSummary(taskID) + if not res['OK']: + gLogger.error(res['Message']) + DIRACExit(2) + taskInfo = res['Value']['TaskInfo'] + replicaInfo = res['Value']['ReplicaInfo'] + outStr = "%s: %s" % ('TaskID'.ljust(20), taskID) + outStr += "\n%s: %s" % ('Status'.ljust(20), taskInfo[taskID]['Status']) + outStr += "\n%s: %s" % ('Source'.ljust(20), taskInfo[taskID]['Source']) + outStr += "\n%s: %s" % ('SourceTaskID'.ljust(20), taskInfo[taskID]['SourceTaskID']) + outStr += "\n%s: %s" % ('CallBackMethod'.ljust(20), taskInfo[taskID]['CallBackMethod']) + outStr += "\n%s: %s" % ('SubmitTime'.ljust(20), taskInfo[taskID]['SubmitTime']) + outStr += "\n%s: %s" % ('CompleteTime'.ljust(20), taskInfo[taskID]['CompleteTime']) + for lfn, metadata in replicaInfo.items(): + outStr += "\n" + outStr += "\n\t%s: %s" % ('LFN'.ljust(8), lfn.ljust(100)) + outStr += "\n\t%s: %s" % ('SE'.ljust(8), metadata['StorageElement'].ljust(100)) + outStr += "\n\t%s: %s" % ('PFN'.ljust(8), str(metadata['PFN']).ljust(100)) + outStr += "\n\t%s: %s" % ('Size'.ljust(8), str(metadata['FileSize']).ljust(100)) + outStr += "\n\t%s: %s" % ('Status'.ljust(8), metadata['Status'].ljust(100)) + outStr += "\n\t%s: %s" % ('Reason'.ljust(8), str(metadata['Reason']).ljust(100)) + gLogger.notice(outStr) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py index f8a91e5f729..29ee3c7315a 100755 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py @@ -12,148 +12,146 @@ from __future__ import print_function _RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import gConfig, gLogger, exit as DIRACExit, S_OK, version subLogger = None -Script.setUsageMessage( - '\n'.join( - [ - __doc__.split('\n')[1], - 'Usage:', - ' %s [--status=] [--se=] [--limit=] [--showJobs=YES] ...' % - Script.scriptName, - 'Arguments:', - ' status: file status=(New, Offline, Waiting, Failed, StageSubmitted, Staged).', - ' se: storage element', - ' showJobs: whether to ALSO list the jobs asking for these files to be staged', - ' WARNING: Query may be heavy, please use --limit switch!'])) - - -def registerSwitches(): - ''' - Registers all switches that can be used while calling the script from the - command line interface. - ''' - - switches = (('status=', 'Filter per file status=(New, Offline, Waiting, Failed, StageSubmitted, Staged).' - '\n If not used, all status values will be taken into account'), - ('se=', 'Filter per Storage Element. If not used, all storage elements will be taken into account.'), - ('limit=', 'Limit the number of entries returned.'), - ('showJobs=', 'Whether to ALSO list the jobs asking for these files to be staged'), - ) - - for switch in switches: - Script.registerSwitch('', switch[0], switch[1]) - - -def parseSwitches(): - ''' - Parses the arguments passed by the user - ''' - - Script.parseCommandLine(ignoreErrors=True) - args = Script.getPositionalArgs() - if args: - subLogger.error("Found the following positional args '%s', but we only accept switches" % args) - subLogger.error("Please, check documentation below") - Script.showHelp(exitCode=1) - - switches = dict(Script.getUnprocessedSwitches()) - - for key in ('status', 'se', 'limit'): - if key not in switches: - subLogger.warn("You're not using switch --%s, query may take long!" % key) - - if 'status' in switches and switches['status'] not in ( - 'New', 'Offline', 'Waiting', 'Failed', 'StageSubmitted', 'Staged'): - subLogger.error("Found \"%s\" as Status value. Incorrect value used!" % switches['status']) - subLogger.error("Please, check documentation below") - Script.showHelp(exitCode=1) - - subLogger.debug("The switches used are:") - map(subLogger.debug, switches.items()) - - return switches - -# ............................................................................... - - -def run(): - - from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient - client = StorageManagerClient() - queryDict = {} - - if 'status' in switchDict: - queryDict['Status'] = str(switchDict['status']) - - if 'se' in switchDict: - queryDict['SE'] = str(switchDict['se']) - - # weird: if there are no switches (dictionary is empty), then the --limit is ignored!! - # must FIX that in StorageManagementDB.py! - # ugly fix: - newer = '1903-08-02 06:24:38' # select newer than - if 'limit' in switchDict: - gLogger.notice("Query limited to %s entries" % switchDict['limit']) - res = client.getCacheReplicas(queryDict, None, newer, None, None, int(switchDict['limit'])) - else: - res = client.getCacheReplicas(queryDict) - - if not res['OK']: - gLogger.error(res['Message']) - outStr = "\n" - if res['Records']: - replicas = res['Value'] - outStr += " %s" % ("Status".ljust(15)) - outStr += " %s" % ("LastUpdate".ljust(20)) - outStr += " %s" % ("LFN".ljust(80)) - outStr += " %s" % ("SE".ljust(10)) - outStr += " %s" % ("Reason".ljust(10)) - if 'showJobs' in switchDict: - outStr += " %s" % ("Jobs".ljust(10)) - outStr += " %s" % ("PinExpiryTime".ljust(15)) - outStr += " %s" % ("PinLength(sec)".ljust(15)) - outStr += "\n" - - for crid, info in replicas.items(): - outStr += " %s" % (info['Status'].ljust(15)) - outStr += " %s" % (str(info['LastUpdate']).ljust(20)) - outStr += " %s" % (info['LFN'].ljust(30)) - outStr += " %s" % (info['SE'].ljust(15)) - outStr += " %s" % (str(info['Reason']).ljust(10)) - - # Task info + +@DIRACScript() +def main(): + Script.setUsageMessage( + '\n'.join( + [ + __doc__.split('\n')[1], + 'Usage:', + ' %s [--status=] [--se=] [--limit=] [--showJobs=YES] ...' % + Script.scriptName, + 'Arguments:', + ' status: file status=(New, Offline, Waiting, Failed, StageSubmitted, Staged).', + ' se: storage element', + ' showJobs: whether to ALSO list the jobs asking for these files to be staged', + ' WARNING: Query may be heavy, please use --limit switch!'])) + + def registerSwitches(): + ''' + Registers all switches that can be used while calling the script from the + command line interface. + ''' + + switches = (('status=', 'Filter per file status=(New, Offline, Waiting, Failed, StageSubmitted, Staged).' + '\n If not used, all status values will be taken into account'), + ('se=', 'Filter per Storage Element. If not used, all storage elements will be taken into account.'), + ('limit=', 'Limit the number of entries returned.'), + ('showJobs=', 'Whether to ALSO list the jobs asking for these files to be staged'), + ) + + for switch in switches: + Script.registerSwitch('', switch[0], switch[1]) + + def parseSwitches(): + ''' + Parses the arguments passed by the user + ''' + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + if args: + subLogger.error("Found the following positional args '%s', but we only accept switches" % args) + subLogger.error("Please, check documentation below") + Script.showHelp(exitCode=1) + + switches = dict(Script.getUnprocessedSwitches()) + + for key in ('status', 'se', 'limit'): + if key not in switches: + subLogger.warn("You're not using switch --%s, query may take long!" % key) + + if 'status' in switches and switches['status'] not in ( + 'New', 'Offline', 'Waiting', 'Failed', 'StageSubmitted', 'Staged'): + subLogger.error("Found \"%s\" as Status value. Incorrect value used!" % switches['status']) + subLogger.error("Please, check documentation below") + Script.showHelp(exitCode=1) + + subLogger.debug("The switches used are:") + map(subLogger.debug, switches.items()) + + return switches + + # ............................................................................... + + def run(): + + from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + client = StorageManagerClient() + queryDict = {} + + if 'status' in switchDict: + queryDict['Status'] = str(switchDict['status']) + + if 'se' in switchDict: + queryDict['SE'] = str(switchDict['se']) + + # weird: if there are no switches (dictionary is empty), then the --limit is ignored!! + # must FIX that in StorageManagementDB.py! + # ugly fix: + newer = '1903-08-02 06:24:38' # select newer than + if 'limit' in switchDict: + gLogger.notice("Query limited to %s entries" % switchDict['limit']) + res = client.getCacheReplicas(queryDict, None, newer, None, None, int(switchDict['limit'])) + else: + res = client.getCacheReplicas(queryDict) + + if not res['OK']: + gLogger.error(res['Message']) + outStr = "\n" + if res['Records']: + replicas = res['Value'] + outStr += " %s" % ("Status".ljust(15)) + outStr += " %s" % ("LastUpdate".ljust(20)) + outStr += " %s" % ("LFN".ljust(80)) + outStr += " %s" % ("SE".ljust(10)) + outStr += " %s" % ("Reason".ljust(10)) if 'showJobs' in switchDict: - resTasks = client.getTasks({'ReplicaID': crid}) - if resTasks['OK']: - if resTasks['Value']: - tasks = resTasks['Value'] - jobs = [] - for tid in tasks: - jobs.append(tasks[tid]['SourceTaskID']) - outStr += ' %s ' % (str(jobs).ljust(10)) - else: - outStr += ' %s ' % (" --- ".ljust(10)) - # Stage request info - # what if there's no request to the site yet? - resStageRequests = client.getStageRequests({'ReplicaID': crid}) - if not resStageRequests['OK']: - gLogger.error(resStageRequests['Message']) - if resStageRequests['Records']: - stageRequests = resStageRequests['Value'] - for info in stageRequests.values(): - outStr += " %s" % (str(info['PinExpiryTime']).ljust(20)) - outStr += " %s" % (str(info['PinLength']).ljust(10)) + outStr += " %s" % ("Jobs".ljust(10)) + outStr += " %s" % ("PinExpiryTime".ljust(15)) + outStr += " %s" % ("PinLength(sec)".ljust(15)) outStr += "\n" - gLogger.notice(outStr) - else: - gLogger.notice("No entries") - - -if __name__ == "__main__": + for crid, info in replicas.items(): + outStr += " %s" % (info['Status'].ljust(15)) + outStr += " %s" % (str(info['LastUpdate']).ljust(20)) + outStr += " %s" % (info['LFN'].ljust(30)) + outStr += " %s" % (info['SE'].ljust(15)) + outStr += " %s" % (str(info['Reason']).ljust(10)) + + # Task info + if 'showJobs' in switchDict: + resTasks = client.getTasks({'ReplicaID': crid}) + if resTasks['OK']: + if resTasks['Value']: + tasks = resTasks['Value'] + jobs = [] + for tid in tasks: + jobs.append(tasks[tid]['SourceTaskID']) + outStr += ' %s ' % (str(jobs).ljust(10)) + else: + outStr += ' %s ' % (" --- ".ljust(10)) + # Stage request info + # what if there's no request to the site yet? + resStageRequests = client.getStageRequests({'ReplicaID': crid}) + if not resStageRequests['OK']: + gLogger.error(resStageRequests['Message']) + if resStageRequests['Records']: + stageRequests = resStageRequests['Value'] + for info in stageRequests.values(): + outStr += " %s" % (str(info['PinExpiryTime']).ljust(20)) + outStr += " %s" % (str(info['PinLength']).ljust(10)) + outStr += "\n" + + gLogger.notice(outStr) + else: + gLogger.notice("No entries") subLogger = gLogger.getSubLogger(__file__) @@ -164,20 +162,25 @@ def run(): # Bye DIRACExit(0) + + +if __name__ == "__main__": + main() + + ''' Example: dirac-stager-show-requests.py --status=Staged --se=GRIDKA-RDST --limit=10 --showJobs=YES Query limited to 10 entries - Status LastUpdate LFN SE Reason Jobs PinExpiryTime PinLength - Staged 2013-06-05 20:10:50 /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00054816_1.full.dst GRIDKA-RDST None ['48498752'] 2013-06-05 22:10:50 86400 - Staged 2013-06-06 15:54:29 /lhcb/LHCb/Collision12/FULL.DST/00020846/0001/00020846_00013202_1.full.dst GRIDKA-RDST None ['48516851'] 2013-06-06 16:54:29 43200 - Staged 2013-06-07 02:35:41 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032726_1.full.dst GRIDKA-RDST None ['48520736'] 2013-06-07 03:35:41 43200 - Staged 2013-06-06 04:16:50 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00030567_1.full.dst GRIDKA-RDST None ['48510852'] 2013-06-06 06:16:50 86400 - Staged 2013-06-07 03:44:04 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032699_1.full.dst GRIDKA-RDST None ['48520737'] 2013-06-07 04:44:04 43200 - Staged 2013-06-05 23:37:46 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032576_1.full.dst GRIDKA-RDST None ['48508687'] 2013-06-06 01:37:46 86400 - Staged 2013-06-10 08:50:09 /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056424_1.full.dst GRIDKA-RDST None ['48518896'] 2013-06-10 09:50:09 43200 - Staged 2013-06-06 11:03:25 /lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00022161_1.full.dst GRIDKA-RDST None ['48515583'] 2013-06-06 12:03:25 43200 - Staged 2013-06-06 11:11:50 /lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00029215_1.full.dst GRIDKA-RDST None ['48515072'] 2013-06-06 12:11:50 43200 - Staged 2013-06-07 03:19:26 /lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00022323_1.full.dst GRIDKA-RDST None ['48515600'] 2013-06-07 04:19:26 43200 - ''' -################################################################################ +Status LastUpdate LFN SE Reason Jobs PinExpiryTime PinLength +Staged 2013-06-05 20:10:50 /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00054816_1.full.dst GRIDKA-RDST None ['48498752'] 2013-06-05 22:10:50 86400 +Staged 2013-06-06 15:54:29 /lhcb/LHCb/Collision12/FULL.DST/00020846/0001/00020846_00013202_1.full.dst GRIDKA-RDST None ['48516851'] 2013-06-06 16:54:29 43200 +Staged 2013-06-07 02:35:41 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032726_1.full.dst GRIDKA-RDST None ['48520736'] 2013-06-07 03:35:41 43200 +Staged 2013-06-06 04:16:50 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00030567_1.full.dst GRIDKA-RDST None ['48510852'] 2013-06-06 06:16:50 86400 +Staged 2013-06-07 03:44:04 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032699_1.full.dst GRIDKA-RDST None ['48520737'] 2013-06-07 04:44:04 43200 +Staged 2013-06-05 23:37:46 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032576_1.full.dst GRIDKA-RDST None ['48508687'] 2013-06-06 01:37:46 86400 +Staged 2013-06-10 08:50:09 /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056424_1.full.dst GRIDKA-RDST None ['48518896'] 2013-06-10 09:50:09 43200 +Staged 2013-06-06 11:03:25 /lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00022161_1.full.dst GRIDKA-RDST None ['48515583'] 2013-06-06 12:03:25 43200 +Staged 2013-06-06 11:11:50 /lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00029215_1.full.dst GRIDKA-RDST None ['48515072'] 2013-06-06 12:11:50 43200 +Staged 2013-06-07 03:19:26 /lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00022323_1.full.dst GRIDKA-RDST None ['48515600'] 2013-06-07 04:19:26 43200 +''' # NOQA diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_show_stats.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_show_stats.py index fdaf34bbddc..008fd9eb98c 100755 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_show_stats.py +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_show_stats.py @@ -30,46 +30,54 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import gConfig, gLogger, exit as DIRACExit, S_OK, version -Script.setUsageMessage(__doc__) -Script.parseCommandLine(ignoreErrors=False) -from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient -client = StorageManagerClient() -res = client.getCacheReplicasSummary() -if not res['OK']: - gLogger.fatal(res['Message']) - DIRACExit(2) -stagerInfo = res['Value'] -outStr = "\n" -outStr += " %s" % ("Status".ljust(20)) -outStr += " %s" % ("SE".ljust(20)) -outStr += " %s" % ("NumberOfFiles".ljust(20)) -outStr += " %s" % ("Size(GB)".ljust(20)) -outStr += " \n--------------------------------------------------------------------------\n" -if stagerInfo: - for info in stagerInfo.values(): - outStr += " %s" % (info['Status'].ljust(20)) - outStr += " %s" % (info['SE'].ljust(20)) - outStr += " %s" % (str(info['NumFiles']).ljust(20)) - outStr += " %s\n" % (str(info['SumFiles']).ljust(20)) -else: - outStr += " %s" % ("Nothing to see here...Bye") -outStr += " %s" % ("\nWARNING: the Size for files with Status=New is not yet determined at the point of selection!\n") -outStr += "--------------------- current status of the SE Caches from the DB-----------" -res = client.getSubmittedStagePins() -if not res['OK']: - gLogger.fatal(res['Message']) - DIRACExit(2) -storageElementUsage = res['Value'] -if storageElementUsage: - for storageElement in storageElementUsage.keys(): - seDict = storageElementUsage[storageElement] - seDict['TotalSize'] = int(seDict['TotalSize'] / (1000 * 1000 * 1000.0)) - outStr += " \n %s: %s replicas with a size of %.3f GB." % ( - storageElement.ljust(15), str(seDict['Replicas']).rjust(6), seDict['TotalSize']) -else: - outStr += " %s" % "\nStageRequest.getStorageUsage: No active stage/pin requests found." -gLogger.notice(outStr) -DIRACExit(0) +@DIRACScript() +def main(): + Script.setUsageMessage(__doc__) + Script.parseCommandLine(ignoreErrors=False) + from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + client = StorageManagerClient() + + res = client.getCacheReplicasSummary() + if not res['OK']: + gLogger.fatal(res['Message']) + DIRACExit(2) + stagerInfo = res['Value'] + outStr = "\n" + outStr += " %s" % ("Status".ljust(20)) + outStr += " %s" % ("SE".ljust(20)) + outStr += " %s" % ("NumberOfFiles".ljust(20)) + outStr += " %s" % ("Size(GB)".ljust(20)) + outStr += " \n--------------------------------------------------------------------------\n" + if stagerInfo: + for info in stagerInfo.values(): + outStr += " %s" % (info['Status'].ljust(20)) + outStr += " %s" % (info['SE'].ljust(20)) + outStr += " %s" % (str(info['NumFiles']).ljust(20)) + outStr += " %s\n" % (str(info['SumFiles']).ljust(20)) + else: + outStr += " %s" % ("Nothing to see here...Bye") + outStr += " \nWARNING: the Size for files with Status=New is not yet determined at the point of selection!\n" + outStr += "--------------------- current status of the SE Caches from the DB-----------" + res = client.getSubmittedStagePins() + if not res['OK']: + gLogger.fatal(res['Message']) + DIRACExit(2) + storageElementUsage = res['Value'] + if storageElementUsage: + for storageElement in storageElementUsage.keys(): + seDict = storageElementUsage[storageElement] + seDict['TotalSize'] = int(seDict['TotalSize'] / (1000 * 1000 * 1000.0)) + outStr += " \n %s: %s replicas with a size of %.3f GB." % ( + storageElement.ljust(15), str(seDict['Replicas']).rjust(6), seDict['TotalSize']) + else: + outStr += " %s" % "\nStageRequest.getStorageUsage: No active stage/pin requests found." + gLogger.notice(outStr) + DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_stage_files.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_stage_files.py index 443443d06a0..77a19c0d04d 100755 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_stage_files.py +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_stage_files.py @@ -14,68 +14,78 @@ from __future__ import print_function __RCSID__ = "$Id$" from DIRAC.Core.Base import Script -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s ' % Script.scriptName, - 'Arguments:', - ' : LFN to Stage (or local file with list of LFNs)', - ' : Name of Storage Element'])) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s ' % Script.scriptName, + 'Arguments:', + ' : LFN to Stage (or local file with list of LFNs)', + ' : Name of Storage Element'])) -if len(args) < 2: - Script.showHelp() + Script.parseCommandLine(ignoreErrors=True) -seName = args[1] -fileName = args[0] + args = Script.getPositionalArgs() -import os -from DIRAC import exit as DIRACExit, gLogger -from DIRAC.Interfaces.API.Dirac import Dirac -from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + if len(args) < 2: + Script.showHelp() -stageLfns = {} + seName = args[1] + fileName = args[0] -if os.path.exists(fileName): - try: - lfnFile = open(fileName) - lfns = [k.strip() for k in lfnFile.readlines()] - lfnFile.close() - except Exception: - gLogger.exception('Can not open file', fileName) + import os + from DIRAC import exit as DIRACExit, gLogger + from DIRAC.Interfaces.API.Dirac import Dirac + from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + + stageLfns = {} + + if os.path.exists(fileName): + try: + lfnFile = open(fileName) + lfns = [k.strip() for k in lfnFile.readlines()] + lfnFile.close() + except Exception: + gLogger.exception('Can not open file', fileName) + DIRACExit(-1) + else: + lfns = args[:len(args) - 1] + + stageLfns[seName] = lfns + stagerClient = StorageManagerClient() + + res = stagerClient.setRequest(stageLfns, 'WorkloadManagement', + 'updateJobFromStager@WorkloadManagement/JobStateUpdate', + 0) # fake JobID = 0 + if not res['OK']: + gLogger.error(res['Message']) DIRACExit(-1) -else: - lfns = args[:len(args) - 1] - -stageLfns[seName] = lfns -stagerClient = StorageManagerClient() - -res = stagerClient.setRequest(stageLfns, 'WorkloadManagement', - 'updateJobFromStager@WorkloadManagement/JobStateUpdate', - 0) # fake JobID = 0 -if not res['OK']: - gLogger.error(res['Message']) - DIRACExit(-1) -else: - gLogger.notice("Stage request submitted for LFNs:\n %s" % lfns) - gLogger.notice("SE= %s" % seName) - gLogger.notice("You can check their status and progress with dirac-stager-monitor-file ") + else: + gLogger.notice("Stage request submitted for LFNs:\n %s" % lfns) + gLogger.notice("SE= %s" % seName) + gLogger.notice("You can check their status and progress with dirac-stager-monitor-file ") + + DIRACExit() + + +if __name__ == "__main__": + main() + '''Example1: dirac-stager-stage-files.py filesToStage.txt GRIDKA-RDST Stage request submitted for LFNs: - ['/lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00023458_1.full.dst', '/lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032669_1.full.dst', '/lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032666_1.full.dst'] +['/lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00023458_1.full.dst', '/lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032669_1.full.dst', '/lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032666_1.full.dst'] SE= GRIDKA-RDST You can check their status and progress with dirac-stager-monitor-file Example2: dirac-stager-stage-files.py /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032641_1.full.dst GRIDKA-RDST Stage request submitted for LFNs: - ['/lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032641_1.full.dst'] +['/lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032641_1.full.dst'] SE= GRIDKA-RDST You can check their status and progress with dirac-stager-monitor-file -''' - -DIRACExit() +''' # NOQA diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py b/src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py index 668ce1bcae2..eb965c8c94f 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py @@ -29,32 +29,7 @@ from urllib.error import URLError as url_library_URLError # pylint: disable=no-name-in-module,import-error from DIRAC.Core.Base import Script -Script.parseCommandLine(ignoreErrors=False) - -Script.registerSwitch('D:', 'Download=', 'Defines data acquisition as DownloadInputData') -Script.registerSwitch('P:', 'Protocol=', 'Defines data acquisition as InputDataByProtocol') - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - '\nUsage:', - 'dirac-production-runjoblocal [Data imput mode] [job ID]' - '\nArguments:', - ' Download (Job ID): Defines data aquisition as DownloadInputData', - ' Protocol (Job ID): Defines data acquisition as InputDataByProtocol\n'])) - -from DIRAC import S_OK -from DIRAC.Core.Utilities.File import mkDir - - -_downloadinputdata = False -_jobID = None - -for switch in Script.getUnprocessedSwitches(): - if switch[0] in ('D', 'Download'): - _downloadinputdata = True - _jobID = switch[1] - if switch[0] in ('I', 'Protocol'): - _downloadinputdata = False - _jobID = switch[1] +from DIRAC.Core.Utilities.DIRACScript import DIRACScript def __runSystemDefaults(jobID, vo): @@ -63,9 +38,8 @@ def __runSystemDefaults(jobID, vo): the path for the other functions. """ - + from DIRAC.Core.Utilities.File import mkDir tempdir = str(vo) + "job" + str(jobID) + "temp" - mkDir(tempdir) basepath = os.getcwd() return basepath + os.path.sep + tempdir + os.path.sep @@ -88,6 +62,7 @@ def __modifyJobDescription(jobID, basepath, downloadinputdata): uses InputDataByProtocol """ + from DIRAC import S_OK if not downloadinputdata: from xml.etree import ElementTree as et archive = et.parse(basepath + "InputSandbox" + str(jobID) + os.path.sep + "jobDescription.xml") @@ -160,7 +135,30 @@ def __runJobLocally(jobID, basepath, vo): localJob.runLocal() -if __name__ == "__main__": +@DIRACScript() +def main(): + Script.parseCommandLine(ignoreErrors=False) + + Script.registerSwitch('D:', 'Download=', 'Defines data acquisition as DownloadInputData') + Script.registerSwitch('P:', 'Protocol=', 'Defines data acquisition as InputDataByProtocol') + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + '\nUsage:', + 'dirac-production-runjoblocal [Data imput mode] [job ID]' + '\nArguments:', + ' Download (Job ID): Defines data aquisition as DownloadInputData', + ' Protocol (Job ID): Defines data acquisition as InputDataByProtocol\n'])) + _downloadinputdata = False + _jobID = None + + for switch in Script.getUnprocessedSwitches(): + if switch[0] in ('D', 'Download'): + _downloadinputdata = True + _jobID = switch[1] + if switch[0] in ('I', 'Protocol'): + _downloadinputdata = False + _jobID = switch[1] + from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import Extensions ext = Extensions() _vo = ext.getCSExtensions()[0] @@ -182,3 +180,7 @@ def __runJobLocally(jobID, basepath, vo): finally: os.chdir(_dir) os.rename(_dir + '.dirac.cfg.old', _dir + '.dirac.cfg') + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_add_files.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_add_files.py index 1465acf2029..b3cb8e739ab 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_add_files.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_add_files.py @@ -12,49 +12,57 @@ import os import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s TransID ' % Script.scriptName - ])) - -Script.parseCommandLine() - -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient - -args = Script.getPositionalArgs() -if len(args) < 2: - Script.showHelp(exitCode=1) - -# get arguments -inputFileName = args[1] - -lfns = [] -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - inputFile.close() - lfns.extend([lfn.strip() for lfn in string.splitlines()]) -else: - lfns.append(inputFileName) - -tc = TransformationClient() -res = tc.addFilesToTransformation(args[0], lfns) # Files added here - -if not res['OK']: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(2) - -successfullyAdded = 0 -alreadyPresent = 0 -for lfn, message in res['Value']['Successful'].items(): - if message == 'Added': - successfullyAdded += 1 - elif message == 'Present': - alreadyPresent += 1 - -if successfullyAdded > 0: - DIRAC.gLogger.notice("Successfully added %d files" % successfullyAdded) -if alreadyPresent > 0: - DIRAC.gLogger.notice("Already present %d files" % alreadyPresent) -DIRAC.exit(0) + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s TransID ' % Script.scriptName + ])) + + Script.parseCommandLine() + + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + args = Script.getPositionalArgs() + if len(args) < 2: + Script.showHelp(exitCode=1) + + # get arguments + inputFileName = args[1] + + lfns = [] + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + inputFile.close() + lfns.extend([lfn.strip() for lfn in string.splitlines()]) + else: + lfns.append(inputFileName) + + tc = TransformationClient() + res = tc.addFilesToTransformation(args[0], lfns) # Files added here + + if not res['OK']: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(2) + + successfullyAdded = 0 + alreadyPresent = 0 + for lfn, message in res['Value']['Successful'].items(): + if message == 'Added': + successfullyAdded += 1 + elif message == 'Present': + alreadyPresent += 1 + + if successfullyAdded > 0: + DIRAC.gLogger.notice("Successfully added %d files" % successfullyAdded) + if alreadyPresent > 0: + DIRAC.gLogger.notice("Already present %d files" % alreadyPresent) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py index efcda0a5f21..ce6af535edc 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py @@ -8,22 +8,31 @@ import sys from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs -parseCommandLine() +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if not getPositionalArgs(): - print('Usage: dirac-transformation-archive transID [transID] [transID]') - sys.exit() -else: - transIDs = [int(arg) for arg in getPositionalArgs()] -from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient +@DIRACScript() +def main(): + parseCommandLine() -agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', - 'Transformation/TransformationCleaningAgent', - 'dirac-transformation-archive') -agent.initialize() + if not getPositionalArgs(): + print('Usage: dirac-transformation-archive transID [transID] [transID]') + sys.exit() + else: + transIDs = [int(arg) for arg in getPositionalArgs()] -client = TransformationClient() -for transID in transIDs: - agent.archiveTransformation(transID) + from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', + 'Transformation/TransformationCleaningAgent', + 'dirac-transformation-archive') + agent.initialize() + + client = TransformationClient() + for transID in transIDs: + agent.archiveTransformation(transID) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py index 97f8a93ab08..acfb5e4ca3e 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py @@ -8,22 +8,31 @@ import sys from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs -parseCommandLine() +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient -if not getPositionalArgs(): - print('Usage: dirac-transformation-clean transID [transID] [transID]') - sys.exit() -else: - transIDs = [int(arg) for arg in getPositionalArgs()] +@DIRACScript() +def main(): + parseCommandLine() -agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', - 'Transformation/TransformationCleaningAgent', - 'dirac-transformation-clean') -agent.initialize() + from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient -client = TransformationClient() -for transID in transIDs: - agent.cleanTransformation(transID) + if not getPositionalArgs(): + print('Usage: dirac-transformation-clean transID [transID] [transID]') + sys.exit() + else: + transIDs = [int(arg) for arg in getPositionalArgs()] + + agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', + 'Transformation/TransformationCleaningAgent', + 'dirac-transformation-clean') + agent.initialize() + + client = TransformationClient() + for transID in transIDs: + agent.cleanTransformation(transID) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_cli.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_cli.py index 562dac4f19f..2484a5d7d20 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_cli.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_cli.py @@ -6,18 +6,24 @@ from __future__ import print_function from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage(""" + +@DIRACScript() +def main(): + Script.setUsageMessage(""" Launch the Transformation shell Usage: %s [option] """ % Script.scriptName) + Script.parseCommandLine(ignoreErrors=False) + from DIRAC.TransformationSystem.Client.TransformationCLI import TransformationCLI -Script.parseCommandLine(ignoreErrors=False) + cli = TransformationCLI() + cli.cmdloop() -from DIRAC.TransformationSystem.Client.TransformationCLI import TransformationCLI -cli = TransformationCLI() -cli.cmdloop() +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_get_files.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_get_files.py index 6024974e45e..9f26d525735 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_get_files.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_get_files.py @@ -11,26 +11,35 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s TransID' % Script.scriptName - ])) -Script.parseCommandLine() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s TransID' % Script.scriptName + ])) -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + Script.parseCommandLine() -args = Script.getPositionalArgs() -if len(args) != 1: - Script.showHelp(exitCode=1) + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient -tc = TransformationClient() -res = tc.getTransformationFiles({'TransformationID': args[0]}) + args = Script.getPositionalArgs() + if len(args) != 1: + Script.showHelp(exitCode=1) -if not res['OK']: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(2) + tc = TransformationClient() + res = tc.getTransformationFiles({'TransformationID': args[0]}) -for transfile in res['Value']: - DIRAC.gLogger.notice(transfile['LFN']) + if not res['OK']: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(2) + + for transfile in res['Value']: + DIRAC.gLogger.notice(transfile['LFN']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_recover_data.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_recover_data.py index 8e3f1cd98f0..a4f9acb9420 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_recover_data.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_recover_data.py @@ -5,7 +5,7 @@ from __future__ import print_function from DIRAC import S_OK, gLogger from DIRAC.Core.Base import Script - +from DIRAC.Core.Utilities.DIRACScript import DIRACScript __RCSID__ = '$Id$' @@ -33,7 +33,8 @@ def registerSwitches(self): ' %s [option|cfgfile] ...\n' % Script.scriptName])) -if __name__ == '__main__': +@DIRACScript() +def main(): PARAMS = Params() PARAMS.registerSwitches() Script.parseCommandLine(ignoreErrors=False) @@ -56,3 +57,7 @@ def registerSwitches(self): gLogger.notice('Found transformation: %s' % TRANS_INFO_DICT) DRA.treatTransformation(PARAMS.transID, TRANS_INFO_DICT) exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py index 24fc4765fa9..131ea60dce5 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py @@ -8,22 +8,31 @@ import sys from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs -parseCommandLine() +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if not getPositionalArgs(): - print('Usage: dirac-transformation-remove-output transID [transID] [transID]') - sys.exit() -else: - transIDs = [int(arg) for arg in getPositionalArgs()] -from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient +@DIRACScript() +def main(): + parseCommandLine() -agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', - 'Transformation/TransformationCleaningAgent', - 'dirac-transformation-remove-output') -agent.initialize() + if not getPositionalArgs(): + print('Usage: dirac-transformation-remove-output transID [transID] [transID]') + sys.exit() + else: + transIDs = [int(arg) for arg in getPositionalArgs()] -client = TransformationClient() -for transID in transIDs: - agent.removeTransformationOutput(transID) + from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', + 'Transformation/TransformationCleaningAgent', + 'dirac-transformation-remove-output') + agent.initialize() + + client = TransformationClient() + for transID in transIDs: + agent.removeTransformationOutput(transID) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_verify_outputdata.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_verify_outputdata.py index 99eb84bf263..037e5072409 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_verify_outputdata.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_verify_outputdata.py @@ -8,22 +8,31 @@ import sys from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs -parseCommandLine() +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if not getPositionalArgs(): - print('Usage: dirac-transformation-verify-outputdata transID [transID] [transID]') - sys.exit() -else: - transIDs = [int(arg) for arg in getPositionalArgs()] -from DIRAC.TransformationSystem.Agent.ValidateOutputDataAgent import ValidateOutputDataAgent -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient +@DIRACScript() +def main(): + parseCommandLine() -agent = ValidateOutputDataAgent('Transformation/ValidateOutputDataAgent', - 'Transformation/ValidateOutputDataAgent', - 'dirac-transformation-verify-outputdata') -agent.initialize() + if not getPositionalArgs(): + print('Usage: dirac-transformation-verify-outputdata transID [transID] [transID]') + sys.exit() + else: + transIDs = [int(arg) for arg in getPositionalArgs()] -client = TransformationClient() -for transID in transIDs: - agent.checkTransformationIntegrity(transID) + from DIRAC.TransformationSystem.Agent.ValidateOutputDataAgent import ValidateOutputDataAgent + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + agent = ValidateOutputDataAgent('Transformation/ValidateOutputDataAgent', + 'Transformation/ValidateOutputDataAgent', + 'dirac-transformation-verify-outputdata') + agent.initialize() + + client = TransformationClient() + for transID in transIDs: + agent.checkTransformationIntegrity(transID) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_kill_pilot.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_kill_pilot.py index b3a33782a16..5b49af94d6e 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_kill_pilot.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_kill_pilot.py @@ -13,27 +13,35 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s ' % Script.scriptName])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s ' % Script.scriptName])) -if len(args) < 1: - Script.showHelp(exitCode=1) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -pilotRef = args[0] + if len(args) < 1: + Script.showHelp(exitCode=1) -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 + pilotRef = args[0] -result = diracAdmin.killPilot(pilotRef) -if not result['OK']: - DIRAC.gLogger.error('Failed to kill pilot', pilotRef) - DIRAC.gLogger.error(result['Message']) - exitCode = 1 + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 -DIRAC.exit(exitCode) + result = diracAdmin.killPilot(pilotRef) + if not result['OK']: + DIRAC.gLogger.error('Failed to kill pilot', pilotRef) + DIRAC.gLogger.error(result['Message']) + exitCode = 1 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_pilot_logging_info.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_pilot_logging_info.py index 7ab7dcb7cd9..882e1a854e4 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_pilot_logging_info.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_pilot_logging_info.py @@ -10,6 +10,7 @@ import DIRAC from DIRAC import S_OK, gLogger from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript uuid = None jobid = None @@ -33,30 +34,38 @@ def setJobID(optVal): return S_OK() -Script.registerSwitch('u:', 'uuid=', 'get PilotsLogging for given Pilot UUID', setUUID) -Script.registerSwitch('j:', 'jobid=', 'get PilotsLogging for given Job ID', setJobID) +@DIRACScript() +def main(): + global uuid + global jobid + Script.registerSwitch('u:', 'uuid=', 'get PilotsLogging for given Pilot UUID', setUUID) + Script.registerSwitch('j:', 'jobid=', 'get PilotsLogging for given Job ID', setJobID) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s option value ' % Script.scriptName, - 'Only one option (either uuid or jobid) should be used.'])) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s option value ' % Script.scriptName, + 'Only one option (either uuid or jobid) should be used.'])) -Script.parseCommandLine() + Script.parseCommandLine() -from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient + from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient -if jobid: - result = PilotManagerClient().getPilots(jobid) + if jobid: + result = PilotManagerClient().getPilots(jobid) + if not result['OK']: + gLogger.error(result['Message']) + DIRAC.exit(1) + gLogger.debug(result['Value']) + uuid = list(result['Value'])[0] + + result = PilotManagerClient().getPilotLoggingInfo(uuid) if not result['OK']: gLogger.error(result['Message']) DIRAC.exit(1) - gLogger.debug(result['Value']) - uuid = list(result['Value'])[0] + gLogger.notice(result['Value']) + + DIRAC.exit(0) -result = PilotManagerClient().getPilotLoggingInfo(uuid) -if not result['OK']: - gLogger.error(result['Message']) - DIRAC.exit(1) -gLogger.notice(result['Value']) -DIRAC.exit(0) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py index 7fb404b0680..f8d161fc4a1 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py @@ -20,6 +20,7 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getUsernameForDN from DIRAC.WorkloadManagementSystem.Client.MatcherClient import MatcherClient +from DIRAC.Core.Utilities.DIRACScript import DIRACScript verbose = False @@ -39,67 +40,75 @@ def setTaskQueueID(optVal): return S_OK() -Script.registerSwitch("v", "verbose", "give max details about task queues", setVerbose) -Script.registerSwitch("t:", "taskQueue=", "show this task queue only", setTaskQueueID) +@DIRACScript() +def main(): + global verbose + global taskQueueID + Script.registerSwitch("v", "verbose", "give max details about task queues", setVerbose) + Script.registerSwitch("t:", "taskQueue=", "show this task queue only", setTaskQueueID) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ' % Script.scriptName])) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ' % Script.scriptName])) -Script.parseCommandLine(initializeMonitor=False) + Script.parseCommandLine(initializeMonitor=False) -result = MatcherClient().getActiveTaskQueues() -if not result['OK']: - gLogger.error(result['Message']) - sys.exit(1) + result = MatcherClient().getActiveTaskQueues() + if not result['OK']: + gLogger.error(result['Message']) + sys.exit(1) -tqDict = result['Value'] + tqDict = result['Value'] -if not verbose: - fields = ['TaskQueue', 'Jobs', 'CPUTime', 'Owner', 'OwnerGroup', 'Sites', - 'Platforms', 'SubmitPools', 'Setup', 'Priority'] - records = [] + if not verbose: + fields = ['TaskQueue', 'Jobs', 'CPUTime', 'Owner', 'OwnerGroup', 'Sites', + 'Platforms', 'SubmitPools', 'Setup', 'Priority'] + records = [] - for tqId in sorted(tqDict): - if taskQueueID and tqId != taskQueueID: - continue - record = [str(tqId)] - tqData = tqDict[tqId] - for key in fields[1:]: - if key == 'Owner': - value = tqData.get('OwnerDN', '-') - if value != '-': - result = getUsernameForDN(value) - if not result['OK']: - value = 'Unknown' + for tqId in sorted(tqDict): + if taskQueueID and tqId != taskQueueID: + continue + record = [str(tqId)] + tqData = tqDict[tqId] + for key in fields[1:]: + if key == 'Owner': + value = tqData.get('OwnerDN', '-') + if value != '-': + result = getUsernameForDN(value) + if not result['OK']: + value = 'Unknown' + else: + value = result['Value'] + else: + value = tqData.get(key, '-') + if isinstance(value, list): + if len(value) > 1: + record.append(str(value[0]) + '...') else: - value = result['Value'] - else: - value = tqData.get(key, '-') - if isinstance(value, list): - if len(value) > 1: - record.append(str(value[0]) + '...') + record.append(str(value[0])) else: - record.append(str(value[0])) - else: - record.append(str(value)) - records.append(record) - - printTable(fields, records) -else: - fields = ['Key', 'Value'] - for tqId in sorted(tqDict): - if taskQueueID and tqId != taskQueueID: - continue - gLogger.notice("\n==> TQ %s" % tqId) - records = [] - tqData = tqDict[tqId] - for key in sorted(tqData): - value = tqData[key] - if isinstance(value, list): - records.append([key, {"Value": value, 'Just': 'L'}]) - else: - value = str(value) - records.append([key, {"Value": value, 'Just': 'L'}]) - - printTable(fields, records, numbering=False) + record.append(str(value)) + records.append(record) + + printTable(fields, records) + else: + fields = ['Key', 'Value'] + for tqId in sorted(tqDict): + if taskQueueID and tqId != taskQueueID: + continue + gLogger.notice("\n==> TQ %s" % tqId) + records = [] + tqData = tqDict[tqId] + for key in sorted(tqData): + value = tqData[key] + if isinstance(value, list): + records.append([key, {"Value": value, 'Just': 'L'}]) + else: + value = str(value) + records.append([key, {"Value": value, 'Just': 'L'}]) + + printTable(fields, records, numbering=False) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_sync_pilot.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_sync_pilot.py index 91cdd2f258b..f87cdb4c3c5 100644 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_sync_pilot.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_sync_pilot.py @@ -16,52 +16,61 @@ from DIRAC.Core.Base import Script -Script.parseCommandLine() - -from DIRAC import gLogger, exit as DIRACExit -from DIRAC.WorkloadManagementSystem.Utilities.PilotCStoJSONSynchronizer import PilotCStoJSONSynchronizer - -ps = PilotCStoJSONSynchronizer() - -gLogger.verbose("Parameters for this sync:") -gLogger.verbose("repo=" + ps.pilotRepo) -gLogger.verbose("VO repo=" + ps.pilotVORepo) -gLogger.verbose("projectDir=" + ps.projectDir) -gLogger.verbose("pilotScriptsPath=" + ps.pilotScriptPath) -gLogger.verbose("pilotVOScriptsPath=" + ps.pilotVOScriptPath) -gLogger.verbose("pilotRepoBranch=" + ps.pilotRepoBranch) -gLogger.verbose("pilotVORepoBranch=" + ps.pilotVORepoBranch) - -# pilot.json -res = ps.getCSDict() -if not res['OK']: - DIRACExit(1) -pilotDict = res['Value'] -print(json.dumps(pilotDict, indent=4, sort_keys=True)) # just print here as formatting is important -with open('pilot.json', 'w') as jf: - json.dump(pilotDict, jf) - -# pilot files -res = ps.syncScripts() -if not res['OK']: - DIRACExit(1) -gLogger.always(res['Value']) -tarPath, tarFiles = res['Value'] - -allFiles = [tarPath] + tarFiles + ['pilot.json'] - -# checksums -checksumDict = {} -for pFile in allFiles: - filename = os.path.basename(pFile) - with open(pFile, 'rb') as fp: - checksumDict[filename] = hashlib.sha512(fp.read()).hexdigest() - cksPath = 'checksums.sha512' -with open(cksPath, 'wt') as chksums: - for filename, chksum in sorted(checksumDict.items()): - # same as the output from sha512sum commands - chksums.write('%s %s\n' % (chksum, filename)) - -allFiles = allFiles + [cksPath] - -print(allFiles) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.parseCommandLine() + + from DIRAC import gLogger, exit as DIRACExit + from DIRAC.WorkloadManagementSystem.Utilities.PilotCStoJSONSynchronizer import PilotCStoJSONSynchronizer + + ps = PilotCStoJSONSynchronizer() + + gLogger.verbose("Parameters for this sync:") + gLogger.verbose("repo=" + ps.pilotRepo) + gLogger.verbose("VO repo=" + ps.pilotVORepo) + gLogger.verbose("projectDir=" + ps.projectDir) + gLogger.verbose("pilotScriptsPath=" + ps.pilotScriptPath) + gLogger.verbose("pilotVOScriptsPath=" + ps.pilotVOScriptPath) + gLogger.verbose("pilotRepoBranch=" + ps.pilotRepoBranch) + gLogger.verbose("pilotVORepoBranch=" + ps.pilotVORepoBranch) + + # pilot.json + res = ps.getCSDict() + if not res['OK']: + DIRACExit(1) + pilotDict = res['Value'] + print(json.dumps(pilotDict, indent=4, sort_keys=True)) # just print here as formatting is important + with open('pilot.json', 'w') as jf: + json.dump(pilotDict, jf) + + # pilot files + res = ps.syncScripts() + if not res['OK']: + DIRACExit(1) + gLogger.always(res['Value']) + tarPath, tarFiles = res['Value'] + + allFiles = [tarPath] + tarFiles + ['pilot.json'] + + # checksums + checksumDict = {} + for pFile in allFiles: + filename = os.path.basename(pFile) + with open(pFile, 'rb') as fp: + checksumDict[filename] = hashlib.sha512(fp.read()).hexdigest() + cksPath = 'checksums.sha512' + with open(cksPath, 'wt') as chksums: + for filename, chksum in sorted(checksumDict.items()): + # same as the output from sha512sum commands + chksums.write('%s %s\n' % (chksum, filename)) + + allFiles = allFiles + [cksPath] + + print(allFiles) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py index 6c6d22ce472..7ae1c197130 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py @@ -20,86 +20,92 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + # Register workflow parameter switch + Script.registerSwitch('p:', 'parameter=', 'Parameters that are passed directly to the workflow') + Script.parseCommandLine() + + # from DIRAC.Core.Workflow.Parameter import * + from DIRAC import gLogger + from DIRAC.Core.Workflow.Workflow import fromXMLFile + from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport + from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient + from DIRAC.RequestManagementSystem.Client.Request import Request + + # Forcing the current directory to be the first in the PYTHONPATH + sys.path.insert(0, os.path.realpath('.')) + gLogger.showHeaders(True) + + def jobexec(jobxml, wfParameters): + jobfile = os.path.abspath(jobxml) + if not os.path.exists(jobfile): + gLogger.warn('Path to specified workflow %s does not exist' % (jobfile)) + sys.exit(1) + workflow = fromXMLFile(jobfile) + gLogger.debug(workflow) + code = workflow.createCode() + gLogger.debug(code) + jobID = 0 + if 'JOBID' in os.environ: + jobID = os.environ['JOBID'] + gLogger.info('DIRAC JobID %s is running at site %s' % (jobID, DIRAC.siteName())) + + workflow.addTool('JobReport', JobReport(jobID)) + workflow.addTool('AccountingReport', DataStoreClient()) + workflow.addTool('Request', Request()) + + # Propagate the command line parameters to the workflow if any + for pName, pValue in wfParameters.items(): + workflow.setValue(pName, pValue) + + # Propagate the command line parameters to the workflow module instances of each step + for stepdefinition in workflow.step_definitions.values(): + for moduleInstance in stepdefinition.module_instances: + for pName, pValue in wfParameters.items(): + if moduleInstance.parameters.find(pName): + moduleInstance.parameters.setValue(pName, pValue) + + return workflow.execute() + + positionalArgs = Script.getPositionalArgs() + if len(positionalArgs) != 1: + gLogger.debug('Positional arguments were %s' % (positionalArgs)) + DIRAC.abort(1, "Must specify the Job XML file description") -# Register workflow parameter switch -Script.registerSwitch('p:', 'parameter=', 'Parameters that are passed directly to the workflow') -Script.parseCommandLine() - -# from DIRAC.Core.Workflow.Parameter import * -from DIRAC import gLogger -from DIRAC.Core.Workflow.Workflow import fromXMLFile -from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport -from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient -from DIRAC.RequestManagementSystem.Client.Request import Request - -# Forcing the current directory to be the first in the PYTHONPATH -sys.path.insert(0, os.path.realpath('.')) -gLogger.showHeaders(True) - - -def jobexec(jobxml, wfParameters): - jobfile = os.path.abspath(jobxml) - if not os.path.exists(jobfile): - gLogger.warn('Path to specified workflow %s does not exist' % (jobfile)) - sys.exit(1) - workflow = fromXMLFile(jobfile) - gLogger.debug(workflow) - code = workflow.createCode() - gLogger.debug(code) - jobID = 0 if 'JOBID' in os.environ: - jobID = os.environ['JOBID'] - gLogger.info('DIRAC JobID %s is running at site %s' % (jobID, DIRAC.siteName())) - - workflow.addTool('JobReport', JobReport(jobID)) - workflow.addTool('AccountingReport', DataStoreClient()) - workflow.addTool('Request', Request()) - - # Propagate the command line parameters to the workflow if any - for pName, pValue in wfParameters.items(): - workflow.setValue(pName, pValue) - - # Propagate the command line parameters to the workflow module instances of each step - for stepdefinition in workflow.step_definitions.values(): - for moduleInstance in stepdefinition.module_instances: - for pName, pValue in wfParameters.items(): - if moduleInstance.parameters.find(pName): - moduleInstance.parameters.setValue(pName, pValue) - - return workflow.execute() - - -positionalArgs = Script.getPositionalArgs() -if len(positionalArgs) != 1: - gLogger.debug('Positional arguments were %s' % (positionalArgs)) - DIRAC.abort(1, "Must specify the Job XML file description") - -if 'JOBID' in os.environ: - gLogger.info('JobID: %s' % (os.environ['JOBID'])) - -jobXMLfile = positionalArgs[0] -parList = Script.getUnprocessedSwitches() -parDict = {} -for switch, parameter in parList: - if switch == "p": - name, value = parameter.split('=') - value = value.strip() - - # The comma separated list in curly brackets is interpreted as a list - if value.startswith("{"): - value = value[1:-1].replace('"', '').replace(" ", '').split(',') - value = ';'.join(value) - - parDict[name] = value - -gLogger.debug('PYTHONPATH:\n%s' % ('\n'.join(sys.path))) -jobExec = jobexec(jobXMLfile, parDict) -if not jobExec['OK']: - gLogger.debug('Workflow execution finished with errors, exiting') - if jobExec['Errno']: - sys.exit(jobExec['Errno']) + gLogger.info('JobID: %s' % (os.environ['JOBID'])) + + jobXMLfile = positionalArgs[0] + parList = Script.getUnprocessedSwitches() + parDict = {} + for switch, parameter in parList: + if switch == "p": + name, value = parameter.split('=') + value = value.strip() + + # The comma separated list in curly brackets is interpreted as a list + if value.startswith("{"): + value = value[1:-1].replace('"', '').replace(" ", '').split(',') + value = ';'.join(value) + + parDict[name] = value + + gLogger.debug('PYTHONPATH:\n%s' % ('\n'.join(sys.path))) + jobExec = jobexec(jobXMLfile, parDict) + if not jobExec['OK']: + gLogger.debug('Workflow execution finished with errors, exiting') + if jobExec['Errno']: + sys.exit(jobExec['Errno']) + else: + sys.exit(1) else: - sys.exit(1) -else: - gLogger.debug('Workflow execution successful, exiting') - sys.exit(0) + gLogger.debug('Workflow execution successful, exiting') + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_cpu_normalization.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_cpu_normalization.py index 0b86b86084c..bcc1131a099 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_cpu_normalization.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_cpu_normalization.py @@ -31,65 +31,72 @@ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.registerSwitch("U", "Update", "Update dirac.cfg with the resulting value") -Script.registerSwitch("R:", "Reconfig=", "Update given configuration file with the resulting value") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ' % Script.scriptName])) +@DIRACScript() +def main(): + Script.registerSwitch("U", "Update", "Update dirac.cfg with the resulting value") + Script.registerSwitch("R:", "Reconfig=", "Update given configuration file with the resulting value") -Script.parseCommandLine(ignoreErrors=True) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ' % Script.scriptName])) -update = False -configFile = None + Script.parseCommandLine(ignoreErrors=True) -for unprocSw in Script.getUnprocessedSwitches(): - if unprocSw[0] in ("U", "Update"): - update = True - elif unprocSw[0] in ("R", "Reconfig"): - configFile = unprocSw[1] + update = False + configFile = None + for unprocSw in Script.getUnprocessedSwitches(): + if unprocSw[0] in ("U", "Update"): + update = True + elif unprocSw[0] in ("R", "Reconfig"): + configFile = unprocSw[1] -if __name__ == "__main__": + if __name__ == "__main__": + + from DIRAC import gLogger, gConfig + from DIRAC.WorkloadManagementSystem.Client.DIRACbenchmark import singleDiracBenchmark + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + from DIRAC.Core.Utilities import MJF - from DIRAC import gLogger, gConfig - from DIRAC.WorkloadManagementSystem.Client.DIRACbenchmark import singleDiracBenchmark - from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations - from DIRAC.Core.Utilities import MJF + mjf = MJF.MJF() + mjf.updateConfig() - mjf = MJF.MJF() - mjf.updateConfig() + db12JobFeature = mjf.getJobFeature('db12') + hs06JobFeature = mjf.getJobFeature('hs06') - db12JobFeature = mjf.getJobFeature('db12') - hs06JobFeature = mjf.getJobFeature('hs06') + result = singleDiracBenchmark(1) - result = singleDiracBenchmark(1) + if result is None: + gLogger.error('Cannot make benchmark measurements') + DIRAC.exit(1) - if result is None: - gLogger.error('Cannot make benchmark measurements') - DIRAC.exit(1) + db12Measured = round(result['NORM'], 1) + corr = Operations().getValue('JobScheduling/CPUNormalizationCorrection', 1.) + norm = round(result['NORM'] / corr, 1) - db12Measured = round(result['NORM'], 1) - corr = Operations().getValue('JobScheduling/CPUNormalizationCorrection', 1.) - norm = round(result['NORM'] / corr, 1) + gLogger.notice('Estimated CPU power is %.1f HS06' % norm) - gLogger.notice('Estimated CPU power is %.1f HS06' % norm) + if update: + gConfig.setOptionValue('/LocalSite/CPUScalingFactor', hs06JobFeature if hs06JobFeature else norm) # deprecate? + gConfig.setOptionValue('/LocalSite/CPUNormalizationFactor', norm) # deprecate? + gConfig.setOptionValue('/LocalSite/DB12measured', db12Measured) - if update: - gConfig.setOptionValue('/LocalSite/CPUScalingFactor', hs06JobFeature if hs06JobFeature else norm) # deprecate? - gConfig.setOptionValue('/LocalSite/CPUNormalizationFactor', norm) # deprecate? - gConfig.setOptionValue('/LocalSite/DB12measured', db12Measured) + # Set DB12 to use by default. Remember db12JobFeature is still in /LocalSite/JOBFEATURES/db12 + if db12JobFeature is not None: + gConfig.setOptionValue('/LocalSite/DB12', db12JobFeature) + else: + gConfig.setOptionValue('/LocalSite/DB12', db12Measured) - # Set DB12 to use by default. Remember db12JobFeature is still in /LocalSite/JOBFEATURES/db12 - if db12JobFeature is not None: - gConfig.setOptionValue('/LocalSite/DB12', db12JobFeature) - else: - gConfig.setOptionValue('/LocalSite/DB12', db12Measured) + if configFile: + gConfig.dumpLocalCFGToFile(configFile) + else: + gConfig.dumpLocalCFGToFile(gConfig.diracConfigFilePath) - if configFile: - gConfig.dumpLocalCFGToFile(configFile) - else: - gConfig.dumpLocalCFGToFile(gConfig.diracConfigFilePath) + DIRAC.exit() - DIRAC.exit() + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_queue_cpu_time.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_queue_cpu_time.py index 24de49176f2..01c59ac7120 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_queue_cpu_time.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_queue_cpu_time.py @@ -13,22 +13,29 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.registerSwitch("C:", "CPUNormalizationFactor=", "CPUNormalizationFactor, in case it is known") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile]' % Script.scriptName])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -CPUNormalizationFactor = 0.0 -for unprocSw in Script.getUnprocessedSwitches(): - if unprocSw[0] in ("C", "CPUNormalizationFactor"): - CPUNormalizationFactor = float(unprocSw[1]) +@DIRACScript() +def main(): + Script.registerSwitch("C:", "CPUNormalizationFactor=", "CPUNormalizationFactor, in case it is known") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile]' % Script.scriptName])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + CPUNormalizationFactor = 0.0 + for unprocSw in Script.getUnprocessedSwitches(): + if unprocSw[0] in ("C", "CPUNormalizationFactor"): + CPUNormalizationFactor = float(unprocSw[1]) -if __name__ == "__main__": from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import getCPUTime cpuTime = getCPUTime(CPUNormalizationFactor) # I hate this kind of output... PhC print("CPU time left determined as", cpuTime) DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn.py index 16df0b29653..db174e18c27 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn.py @@ -15,9 +15,11 @@ import DIRAC import DIRAC.Core.Base.Script as Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if __name__ == "__main__": +@DIRACScript() +def main(): site = 'BOINC.World.org' status = ["Running"] minorStatus = None @@ -195,3 +197,7 @@ node = result[job]['Node'] jobID = result[job].get('LocalJobID') gLogger.always('%s (%s): %s - %s' % (node, jobID, job, status)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn_parameters.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn_parameters.py index 67917ca76ce..99c82f06239 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn_parameters.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn_parameters.py @@ -9,17 +9,13 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import gLogger from DIRAC.WorkloadManagementSystem.Utilities import JobParameters -Script.setUsageMessage('\n'.join(['Get the parameters (Memory and Number of processors) of a worker node', - 'Usage:', - ' %s [option]... [cfgfile]' % Script.scriptName, - 'Arguments:', - ' cfgfile: DIRAC Cfg with description of the configuration (optional)'])) - ceName = '' ceType = '' +Queue = '' def setCEName(args): @@ -37,21 +33,34 @@ def setQueue(args): Queue = args -Script.registerSwitch("N:", "Name=", "Computing Element Name (Mandatory)", setCEName) -Script.registerSwitch("S:", "Site=", "Site Name (Mandatory)", setSite) -Script.registerSwitch("Q:", "Queue=", "Queue Name (Mandatory)", setQueue) -Script.parseCommandLine(ignoreErrors=True) +@DIRACScript() +def main(): + global ceName + global Site + global Queue + Script.setUsageMessage('\n'.join(['Get the parameters (Memory and Number of processors) of a worker node', + 'Usage:', + ' %s [option]... [cfgfile]' % Script.scriptName, + 'Arguments:', + ' cfgfile: DIRAC Cfg with description of the configuration (optional)'])) + + Script.registerSwitch("N:", "Name=", "Computing Element Name (Mandatory)", setCEName) + Script.registerSwitch("S:", "Site=", "Site Name (Mandatory)", setSite) + Script.registerSwitch("Q:", "Queue=", "Queue Name (Mandatory)", setQueue) + Script.parseCommandLine(ignoreErrors=True) + gLogger.info("Getting number of processors") + numberOfProcessor = JobParameters.getNumberOfProcessors(Site, ceName, Queue) -gLogger.info("Getting number of processors") -numberOfProcessor = JobParameters.getNumberOfProcessors(Site, ceName, Queue) + gLogger.info("Getting memory (RAM) from MJF") + maxRAM = JobParameters.getMemoryFromMJF() + if not maxRAM: + gLogger.info("maxRAM could not be found in MJF, using JobParameters.getMemoryFromProc()") + maxRAM = JobParameters.getMemoryFromProc() + # just communicating it back + gLogger.notice(numberOfProcessor, maxRAM) -gLogger.info("Getting memory (RAM) from MJF") -maxRAM = JobParameters.getMemoryFromMJF() -if not maxRAM: - gLogger.info("maxRAM could not be found in MJF, using JobParameters.getMemoryFromProc()") - maxRAM = JobParameters.getMemoryFromProc() -# just communicating it back -gLogger.notice(numberOfProcessor, maxRAM) +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_match.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_match.py index faab667f379..62524f8aaf1 100644 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_match.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_match.py @@ -9,6 +9,7 @@ from DIRAC.Core.Base import Script from DIRAC import S_OK, gLogger, exit as DIRACExit +from DIRAC.Core.Utilities.DIRACScript import DIRACScript __RCSID__ = "$Id$" @@ -28,36 +29,37 @@ def setSites(optVal_): return S_OK() -description = """Get computing resources capable to execute a job with the given description. - -Note that only statically defined computing resource parameters are considered although sites -can fail matching due to their dynamic state, e.g. occupancy by other jobs. Also input data -proximity is not taken into account. -""" - -Script.setUsageMessage('\n'.join([description, - 'Usage:', - ' %s [option]... ' % Script.scriptName])) +@DIRACScript() +def main(): + global fullMatch + global sites + description = """Get computing resources capable to execute a job with the given description. -Script.registerSwitch("F", "full-match", "Check all the matching criteria", setFullMatch) -Script.registerSwitch("S:", "site=", "Check matching for these sites (comma separated list)", setSites) + Note that only statically defined computing resource parameters are considered although sites + can fail matching due to their dynamic state, e.g. occupancy by other jobs. Also input data + proximity is not taken into account. + """ + Script.setUsageMessage('\n'.join([description, + 'Usage:', + ' %s [option]... ' % Script.scriptName])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() + Script.registerSwitch("F", "full-match", "Check all the matching criteria", setFullMatch) + Script.registerSwitch("S:", "site=", "Check matching for these sites (comma separated list)", setSites) -if len(args) == 0: - gLogger.error("Error: No job description provided") - Script.showHelp(exitCode=1) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup -from DIRAC.ConfigurationSystem.Client.Helpers import Resources -from DIRAC.Core.Utilities.PrettyPrint import printTable -from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus -from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus -from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved, matchQueue + if len(args) == 0: + gLogger.error("Error: No job description provided") + Script.showHelp(exitCode=1) + from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup + from DIRAC.ConfigurationSystem.Client.Helpers import Resources + from DIRAC.Core.Utilities.PrettyPrint import printTable + from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus + from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus + from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved, matchQueue -if __name__ == '__main__': with open(args[0]) as f: jdl = f.read() @@ -112,3 +114,7 @@ def setSites(optVal_): records.append((site, ce, queueInfo['Queue'], status, 'No', result['Value']['Reason'])) gLogger.notice(printTable(fields, records, sortField='Site', columnSeparator=' ', printOut=False)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py index 5220ac331fe..fadf153bb63 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py @@ -14,6 +14,7 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript def _stringInList(subStr, sList): @@ -24,62 +25,68 @@ def _stringInList(subStr, sList): return resList -parameters = ['OwnerDN', 'StartExecTime', 'EndExecTime'] -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, - 'Arguments:', - ' PilotID: Grid ID of the pilot'])) -Script.registerSwitch('', 'Parameters=', ' List of strings to be matched by job parameters or attributes') -Script.parseCommandLine(ignoreErrors=True) -for switch in Script.getUnprocessedSwitches(): - if switch[0] == 'Parameters': - parameters += [par for par in switch[1].split(',')] -parameters = [(i, par.lower()) for i, par in enumerate(parameters) if par] -args = Script.getPositionalArgs() +@DIRACScript() +def main(): + parameters = ['OwnerDN', 'StartExecTime', 'EndExecTime'] + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, + 'Arguments:', + ' PilotID: Grid ID of the pilot'])) + Script.registerSwitch('', 'Parameters=', ' List of strings to be matched by job parameters or attributes') + Script.parseCommandLine(ignoreErrors=True) + for switch in Script.getUnprocessedSwitches(): + if switch[0] == 'Parameters': + parameters += [par for par in switch[1].split(',')] + parameters = [(i, par.lower()) for i, par in enumerate(parameters) if par] + args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp() + if len(args) < 1: + Script.showHelp() -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -from DIRAC.Interfaces.API.Dirac import Dirac -diracAdmin = DiracAdmin() -dirac = Dirac() -errorList = [] + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + from DIRAC.Interfaces.API.Dirac import Dirac + diracAdmin = DiracAdmin() + dirac = Dirac() + errorList = [] -for gridID in args: + for gridID in args: - result = {} - res = diracAdmin.getPilotInfo(gridID) - if not res['OK']: - errorList.append((gridID, res['Message'])) - else: - jobIDs = set([int(jobID) for jobID in res['Value'][gridID]['Jobs']]) - totCPU = 0 - totWall = 0 - effRequested = False - for jobID in sorted(jobIDs): - result.setdefault(jobID, {}) - for func in (dirac.getJobParameters, dirac.getJobAttributes): - res = func(jobID) - if not res['OK']: - errorList.append(('Job %d' % jobID, res['Message'])) - else: - params = res['Value'] - if 'TotalCPUTime(s)' in params: - totCPU += float(params['TotalCPUTime(s)']) - totWall += float(params['WallClockTime(s)']) - params['CPUEfficiency'] = '%s %%' % ( - 100. * params['TotalCPUTime(s)'] / params['WallClockTime(s)']) - for i, par in parameters: - for param in [p for p in _stringInList(str(par), str(params)) if not _stringInList(str(p), - str(result[jobID]))]: - if param == 'CPUEfficiency': - effRequested = True - result[jobID]['%d.%s' % (i, param)] = params[param] - if effRequested: - result['CPUEfficiency'] = '%s %%' % (100. * totCPU / totWall) - print(diracAdmin.pPrint.pformat({gridID: result})) + result = {} + res = diracAdmin.getPilotInfo(gridID) + if not res['OK']: + errorList.append((gridID, res['Message'])) + else: + jobIDs = set([int(jobID) for jobID in res['Value'][gridID]['Jobs']]) + totCPU = 0 + totWall = 0 + effRequested = False + for jobID in sorted(jobIDs): + result.setdefault(jobID, {}) + for func in (dirac.getJobParameters, dirac.getJobAttributes): + res = func(jobID) + if not res['OK']: + errorList.append(('Job %d' % jobID, res['Message'])) + else: + params = res['Value'] + if 'TotalCPUTime(s)' in params: + totCPU += float(params['TotalCPUTime(s)']) + totWall += float(params['WallClockTime(s)']) + params['CPUEfficiency'] = '%s %%' % ( + 100. * params['TotalCPUTime(s)'] / params['WallClockTime(s)']) + for i, par in parameters: + for param in [p for p in _stringInList(str(par), str(params)) + if not _stringInList(str(p), str(result[jobID]))]: + if param == 'CPUEfficiency': + effRequested = True + result[jobID]['%d.%s' % (i, param)] = params[param] + if effRequested: + result['CPUEfficiency'] = '%s %%' % (100. * totCPU / totWall) + print(diracAdmin.pPrint.pformat({gridID: result})) -for error in errorList: - print("ERROR %s: %s" % error) + for error in errorList: + print("ERROR %s: %s" % error) + + +if __name__ == "__main__": + main() From 704dfa00aef55c511b00474645daba1c07d00d12 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 13 Jan 2021 20:27:06 +0100 Subject: [PATCH 17/27] Fix filename in py3CheckDirs.txt --- tests/py3CheckDirs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/py3CheckDirs.txt b/tests/py3CheckDirs.txt index 2c872f3af71..1a3ad63d306 100644 --- a/tests/py3CheckDirs.txt +++ b/tests/py3CheckDirs.txt @@ -15,6 +15,6 @@ src/DIRAC/DataManagementSystem/Service/TornadoFileCatalogHandler.py src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py src/DIRAC/FrameworkSystem/Client/MonitoringClientIOLoop.py src/DIRAC/FrameworkSystem/Client/MonitoringClient.py -src/DIRAC/FrameworkSystem/scripts/dirac-install-tornado-service.py +src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py src/DIRAC/Resources/Catalog/FileCatalogClient.py src/DIRAC/Resources/Storage/GFAL2_StorageBase.py From 6567cc9b4c9fb5e62459a016004792294b8edc8a Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 13 Jan 2021 20:57:55 +0100 Subject: [PATCH 18/27] PY3: Update ComponentInstaller to account for _ in script names --- src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py index 2452b6ad430..f21e897bebc 100644 --- a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py +++ b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py @@ -1914,11 +1914,11 @@ def installComponent(self, componentType, system, component, extensions, compone [[ "%(componentType)s" = "agent" ]] && renice 20 -p $$ #%(bashVariables)s # -exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py \ +exec python $DIRAC/DIRAC/Core/scripts/dirac_%(componentType)s.py \ %(system)s/%(component)s --cfg %(componentCfg)s < /dev/null """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), 'bashVariables': bashVars, - 'componentType': componentType, + 'componentType': componentType.replace("-", "_"), 'system': system, 'component': component, 'componentCfg': componentCfg}) @@ -2541,7 +2541,7 @@ def installTornado(self): exec 2>&1 # # - exec python $DIRAC/DIRAC/Core/Tornado/scripts/tornado-start-all.py -ddd + exec python $DIRAC/DIRAC/Core/Tornado/scripts/tornado_start_all.py -ddd """ % {'bashrc': os.path.join(self.instancePath, 'bashrc')}) os.chmod(runFile, self.gDefaultPerms) From 850a3d5df22f5133799e286a31351bccc06a6b13 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 13 Jan 2021 22:14:02 +0100 Subject: [PATCH 19/27] Use global variables in some required places --- src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py | 3 ++- .../ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py | 2 +- .../ResourceStatusSystem/scripts/dirac_rss_set_status.py | 4 +++- src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py | 3 +++ .../scripts/dirac_stager_monitor_requests.py | 1 + 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py index 532a21c3d93..4183046b2f0 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py @@ -31,7 +31,6 @@ from DIRAC.Core.Utilities.PrettyPrint import printTable subLogger = None -switchDict = {} def registerSwitches(): @@ -412,6 +411,8 @@ def run(args, switchDictSet): @DIRACScript() def main(): + global subLogger + subLogger = gLogger.getSubLogger(__file__) # Script initialization diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py index 27eefbed06a..7f5b0e97c40 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py @@ -28,7 +28,6 @@ subLogger = None -switchDict = {} def registerSwitches(): @@ -341,6 +340,7 @@ def run(args, switchDict): @DIRACScript() def main(): + global subLogger global ResourceManagementClient subLogger = gLogger.getSubLogger(__file__) diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py index 61c291f19b1..45125e5205d 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py @@ -22,7 +22,6 @@ subLogger = None -switchDict = {} def registerSwitches(): @@ -235,6 +234,9 @@ def run(switchDict): @DIRACScript() def main(): + global subLogger + global registerUsageMessage + subLogger = gLogger.getSubLogger(__file__) # Script initialization diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py index db227f46189..05fd2afa032 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py @@ -182,6 +182,9 @@ def main(): Main function of the script. Gets the username from the proxy loaded and sets the token taking into account that user and the switchDict parameters. """ + global subLogger + global switchDict + # Logger initialization subLogger = gLogger.getSubLogger(__file__) diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py index 29ee3c7315a..840109942bb 100755 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py @@ -81,6 +81,7 @@ def parseSwitches(): # ............................................................................... def run(): + global subLogger from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient client = StorageManagerClient() From 780368e8287395873ee2f8707a9e3186b98c597f Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 13 Jan 2021 22:45:47 +0100 Subject: [PATCH 20/27] Fix typo --- .../ConfigurationSystem/scripts/dirac_configuration_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py index a70a2a9bd6c..25223b11b4d 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py @@ -16,7 +16,7 @@ # * root starts with "/" -@DIRACScript +@DIRACScript() def main(): Script.parseCommandLine() from DIRAC.ConfigurationSystem.Client.CSShellCLI import CSShellCLI From 6ba6fb7c5eadfee08de70b6b8ae0d2687f7c1e8e Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 14 Jan 2021 10:08:14 +0100 Subject: [PATCH 21/27] PY3: Convert dirac-platform and add shell scripts to setup.cfg --- setup.cfg | 9 ++++++++- src/DIRAC/Core/scripts/dirac_platform.py | 14 +++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index bf1d9871735..7b05ccc16c1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,6 +18,12 @@ python_requires = >=3.8 package_dir= =src packages = find: +# TODO: This should be treated as a legacy workaround and eventually removed +scripts = + src/DIRAC/Core/scripts/genAllCAs.sh + src/DIRAC/Core/scripts/dirac-cert-convert.sh + src/DIRAC/Core/scripts/install_site.sh + src/DIRAC/Core/scripts/genRevokedCerts.sh install_requires = boto3 botocore @@ -39,7 +45,7 @@ install_requires = six sqlalchemy subprocess32 - tornado =5.1.1+dirac.1 + tornado ~=5.1.1 [options.packages.find] where=src @@ -99,6 +105,7 @@ console_scripts = dirac-info = DIRAC.Core.scripts.dirac_info:main dirac-install-db = DIRAC.Core.scripts.dirac_install_db:main dirac-install-web-portal = DIRAC.Core.scripts.dirac_install_web_portal:main + dirac-platform = DIRAC.Core.scripts.dirac_platform:main dirac-service = DIRAC.Core.scripts.dirac_service:main dirac-setup-site = DIRAC.Core.scripts.dirac_setup_site:main dirac-version = DIRAC.Core.scripts.dirac_version:main diff --git a/src/DIRAC/Core/scripts/dirac_platform.py b/src/DIRAC/Core/scripts/dirac_platform.py index fa0b9f1deef..373f287dc1e 100755 --- a/src/DIRAC/Core/scripts/dirac_platform.py +++ b/src/DIRAC/Core/scripts/dirac_platform.py @@ -127,5 +127,17 @@ def getPlatformString(): return platformString + + def main(): + print(getPlatformString()) +else: + from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + + @DIRACScript() + def main(): + print(getPlatformString()) + + if __name__ == "__main__": - print(getPlatformString()) + main() From 88c61502d5e5183ed2cca692f859dd5ec1210411 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 14 Jan 2021 10:08:34 +0100 Subject: [PATCH 22/27] PY3: Convert tornado scripts to be setup.cfg compatible --- setup.cfg | 3 + .../Core/Tornado/scripts/tornado_start_CS.py | 57 +++++++------ .../Core/Tornado/scripts/tornado_start_all.py | 85 ++++++++++--------- 3 files changed, 80 insertions(+), 65 deletions(-) diff --git a/setup.cfg b/setup.cfg index 7b05ccc16c1..745ea3a9f88 100644 --- a/setup.cfg +++ b/setup.cfg @@ -109,6 +109,9 @@ console_scripts = dirac-service = DIRAC.Core.scripts.dirac_service:main dirac-setup-site = DIRAC.Core.scripts.dirac_setup_site:main dirac-version = DIRAC.Core.scripts.dirac_version:main + # Core.Tornado + tornado-start-CS = DIRAC.Core.Tornado.scripts.tornado_start_CS:main + tornado-start-all = DIRAC.Core.Tornado.scripts.tornado_start_all:main # DataManagementSystem dirac-admin-allow-se = DIRAC.DataManagementSystem.scripts.dirac_admin_allow_se:main dirac-admin-ban-se = DIRAC.DataManagementSystem.scripts.dirac_admin_ban_se:main diff --git a/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py index fb4e1811e26..164fddb8f1c 100644 --- a/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py +++ b/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py @@ -12,37 +12,46 @@ __RCSID__ = "$Id$" -# Must be define BEFORE any dirac import import os import sys -os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration -from DIRAC.ConfigurationSystem.private.Refresher import gRefresher -from DIRAC.Core.Utilities.DErrno import includeExtensionErrors -from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer -from DIRAC.FrameworkSystem.Client.Logger import gLogger -if gConfigurationData.isMaster(): - gRefresher.disable() +@DIRACScript() +def main(): + # Must be defined BEFORE any dirac import + os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" -localCfg = LocalConfiguration() -localCfg.addMandatoryEntry("/DIRAC/Setup") -localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") -localCfg.addDefaultEntry("LogLevel", "INFO") -localCfg.addDefaultEntry("LogColor", True) -resultDict = localCfg.loadUserData() -if not resultDict['OK']: - gLogger.initialize("Tornado-CS", "/") - gLogger.error("There were errors when loading configuration", resultDict['Message']) - sys.exit(1) + from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration + from DIRAC.ConfigurationSystem.private.Refresher import gRefresher + from DIRAC.Core.Utilities.DErrno import includeExtensionErrors + from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer + from DIRAC.FrameworkSystem.Client.Logger import gLogger -includeExtensionErrors() + if gConfigurationData.isMaster(): + gRefresher.disable() + localCfg = LocalConfiguration() + localCfg.addMandatoryEntry("/DIRAC/Setup") + localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") + localCfg.addDefaultEntry("LogLevel", "INFO") + localCfg.addDefaultEntry("LogColor", True) + resultDict = localCfg.loadUserData() + if not resultDict['OK']: + gLogger.initialize("Tornado-CS", "/") + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) -gLogger.initialize('Tornado-CS', "/") + includeExtensionErrors() -serverToLaunch = TornadoServer(services='Configuration/Server') -serverToLaunch.startTornado() + gLogger.initialize('Tornado-CS', "/") + + + serverToLaunch = TornadoServer(services='Configuration/Server') + serverToLaunch.startTornado() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py index 729165e08bf..56061542bc9 100644 --- a/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py +++ b/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py @@ -15,44 +15,47 @@ # Must be define BEFORE any dirac import import os import sys -os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" - - -from DIRAC import gConfig -from DIRAC.ConfigurationSystem.Client import PathFinder -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration -from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer -from DIRAC.Core.Utilities.DErrno import includeExtensionErrors -from DIRAC.FrameworkSystem.Client.Logger import gLogger - - -# We check if there is no configuration server started as master -# If you want to start a master CS you should use Configuration_Server.cfg and -# use tornado-start-CS.py -if gConfigurationData.isMaster() and gConfig.getValue( - '/Systems/Configuration/%s/Services/Server/Protocol' % - PathFinder.getSystemInstance('Configuration'), - 'dips').lower() == 'https': - gLogger.fatal("You can't run the CS and services in the same server!") - sys.exit(0) - -localCfg = LocalConfiguration() -localCfg.addMandatoryEntry("/DIRAC/Setup") -localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") -localCfg.addDefaultEntry("LogLevel", "INFO") -localCfg.addDefaultEntry("LogColor", True) -resultDict = localCfg.loadUserData() -if not resultDict['OK']: - gLogger.initialize("Tornado", "/") - gLogger.error("There were errors when loading configuration", resultDict['Message']) - sys.exit(1) - -includeExtensionErrors() - - -gLogger.initialize('Tornado', "/") - - -serverToLaunch = TornadoServer() -serverToLaunch.startTornado() +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" + + from DIRAC import gConfig + from DIRAC.ConfigurationSystem.Client import PathFinder + from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration + from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer + from DIRAC.Core.Utilities.DErrno import includeExtensionErrors + from DIRAC.FrameworkSystem.Client.Logger import gLogger + + # We check if there is no configuration server started as master + # If you want to start a master CS you should use Configuration_Server.cfg and + # use tornado-start-CS.py + key = '/Systems/Configuration/%s/Services/Server/Protocol' % PathFinder.getSystemInstance('Configuration') + if gConfigurationData.isMaster() and gConfig.getValue(key, 'dips').lower() == 'https': + gLogger.fatal("You can't run the CS and services in the same server!") + sys.exit(0) + + localCfg = LocalConfiguration() + localCfg.addMandatoryEntry("/DIRAC/Setup") + localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") + localCfg.addDefaultEntry("LogLevel", "INFO") + localCfg.addDefaultEntry("LogColor", True) + resultDict = localCfg.loadUserData() + if not resultDict['OK']: + gLogger.initialize("Tornado", "/") + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + + includeExtensionErrors() + + gLogger.initialize('Tornado', "/") + + serverToLaunch = TornadoServer() + serverToLaunch.startTornado() + + +if __name__ == "__main__": + main() From fa95a5c473dac74bc5b2b9964a9cfd353daf4cbd Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 14 Jan 2021 10:14:09 +0100 Subject: [PATCH 23/27] Fix formatting --- src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py | 2 -- src/DIRAC/Core/scripts/dirac_platform.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py index 164fddb8f1c..1e6d5fa7148 100644 --- a/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py +++ b/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py @@ -45,10 +45,8 @@ def main(): includeExtensionErrors() - gLogger.initialize('Tornado-CS', "/") - serverToLaunch = TornadoServer(services='Configuration/Server') serverToLaunch.startTornado() diff --git a/src/DIRAC/Core/scripts/dirac_platform.py b/src/DIRAC/Core/scripts/dirac_platform.py index 373f287dc1e..f134cb03d44 100755 --- a/src/DIRAC/Core/scripts/dirac_platform.py +++ b/src/DIRAC/Core/scripts/dirac_platform.py @@ -82,7 +82,6 @@ def libc_ver(executable=sys.executable, lib='', version='', chunksize=2048): return lib, '.'.join(map(str, version)) # Command line interface - def getPlatformString(): # Modified to return our desired platform string, R. Graciani platformTuple = (platform.system(), platform.machine()) @@ -127,13 +126,11 @@ def getPlatformString(): return platformString - def main(): print(getPlatformString()) else: from DIRAC.Core.Utilities.DIRACScript import DIRACScript - @DIRACScript() def main(): print(getPlatformString()) From f0392eae66fd3c6fb5ffee12eb7cbf21394287c1 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 14 Jan 2021 11:57:35 +0100 Subject: [PATCH 24/27] Restore dirac-agent.py dirac-executor.py and dirac-service.py from integration --- src/DIRAC/Core/scripts/dirac-agent.py | 46 +++++++++++++++++++ src/DIRAC/Core/scripts/dirac-executor.py | 56 ++++++++++++++++++++++++ src/DIRAC/Core/scripts/dirac-service.py | 53 ++++++++++++++++++++++ 3 files changed, 155 insertions(+) create mode 100755 src/DIRAC/Core/scripts/dirac-agent.py create mode 100755 src/DIRAC/Core/scripts/dirac-executor.py create mode 100755 src/DIRAC/Core/scripts/dirac-service.py diff --git a/src/DIRAC/Core/scripts/dirac-agent.py b/src/DIRAC/Core/scripts/dirac-agent.py new file mode 100755 index 00000000000..6bd5b3e43a2 --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac-agent.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-agent +# Author : Adria Casajus, Andrei Tsaregorodtsev, Stuart Paterson +######################################################################## +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +""" This is a script to launch DIRAC agents +""" + +import sys +from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration +from DIRAC import gLogger +from DIRAC.Core.Base.AgentReactor import AgentReactor +from DIRAC.Core.Utilities.DErrno import includeExtensionErrors + +localCfg = LocalConfiguration() + +positionalArgs = localCfg.getPositionalArguments() +if len(positionalArgs) == 0: + gLogger.fatal("You must specify which agent to run!") + sys.exit(1) + +agentName = positionalArgs[0] +localCfg.setConfigurationForAgent(agentName) +localCfg.addMandatoryEntry("/DIRAC/Setup") +localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") +localCfg.addDefaultEntry("LogLevel", "INFO") +localCfg.addDefaultEntry("LogColor", True) +resultDict = localCfg.loadUserData() +if not resultDict['OK']: + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + +includeExtensionErrors() + +agentReactor = AgentReactor(positionalArgs[0]) +result = agentReactor.loadAgentModules(positionalArgs) +if result['OK']: + agentReactor.go() +else: + gLogger.error("Error while loading agent module", result['Message']) + sys.exit(2) diff --git a/src/DIRAC/Core/scripts/dirac-executor.py b/src/DIRAC/Core/scripts/dirac-executor.py new file mode 100755 index 00000000000..76b578c5017 --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac-executor.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-executor +# Author : Adria Casajus +######################################################################## +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +""" This is a script to launch DIRAC executors +""" + +import sys +from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration +from DIRAC import gLogger +from DIRAC.Core.Base.ExecutorReactor import ExecutorReactor +from DIRAC.Core.Utilities.DErrno import includeExtensionErrors + +localCfg = LocalConfiguration() + +positionalArgs = localCfg.getPositionalArguments() +if len(positionalArgs) == 0: + gLogger.fatal("You must specify which executor to run!") + sys.exit(1) + +if len(positionalArgs) == 1 and positionalArgs[0].find("/") > -1: + mainName = positionalArgs[0] +else: + mainName = "Framework/MultiExecutor" + +localCfg.setConfigurationForExecutor(mainName) +localCfg.addMandatoryEntry("/DIRAC/Setup") +localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") +localCfg.addDefaultEntry("LogLevel", "INFO") +localCfg.addDefaultEntry("LogColor", True) +resultDict = localCfg.loadUserData() +if not resultDict['OK']: + gLogger.fatal("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + +includeExtensionErrors() +executorReactor = ExecutorReactor() + +result = executorReactor.loadModules(positionalArgs) +if not result['OK']: + gLogger.fatal("Error while loading executor", result['Message']) + sys.exit(1) + +result = executorReactor.go() +if not result['OK']: + gLogger.fatal(result['Message']) + sys.exit(1) + +gLogger.notice("Graceful exit. Bye!") +sys.exit(0) diff --git a/src/DIRAC/Core/scripts/dirac-service.py b/src/DIRAC/Core/scripts/dirac-service.py new file mode 100755 index 00000000000..49db27aa5fd --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac-service.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-service +# Author : Adria Casajus +######################################################################## + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import sys + +from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration +from DIRAC.FrameworkSystem.Client.Logger import gLogger +from DIRAC.Core.DISET.ServiceReactor import ServiceReactor +from DIRAC.Core.Utilities.DErrno import includeExtensionErrors + +__RCSID__ = "$Id$" + +localCfg = LocalConfiguration() + + +positionalArgs = localCfg.getPositionalArguments() +if len(positionalArgs) == 0: + gLogger.fatal("You must specify which server to run!") + sys.exit(1) + +serverName = positionalArgs[0] +localCfg.setConfigurationForServer(serverName) +localCfg.addMandatoryEntry("Port") +# localCfg.addMandatoryEntry( "HandlerPath" ) +localCfg.addMandatoryEntry("/DIRAC/Setup") +localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") +localCfg.addDefaultEntry("LogLevel", "INFO") +localCfg.addDefaultEntry("LogColor", True) +resultDict = localCfg.loadUserData() +if not resultDict['OK']: + gLogger.initialize(serverName, "/") + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + +includeExtensionErrors() + + +serverToLaunch = ServiceReactor() +result = serverToLaunch.initialize(positionalArgs) +if not result['OK']: + gLogger.error(result['Message']) + sys.exit(1) + +result = serverToLaunch.serve() +if not result['OK']: + gLogger.error(result['Message']) + sys.exit(1) From 35931a0abe235f15b0e3f99f3bc5b338023f05bb Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 14 Jan 2021 11:59:05 +0100 Subject: [PATCH 25/27] Add deprecation notice to dirac-agent.py dirac-executor.py and dirac-service.py --- src/DIRAC/Core/scripts/dirac-agent.py | 3 +++ src/DIRAC/Core/scripts/dirac-executor.py | 3 +++ src/DIRAC/Core/scripts/dirac-service.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/src/DIRAC/Core/scripts/dirac-agent.py b/src/DIRAC/Core/scripts/dirac-agent.py index 6bd5b3e43a2..d5817ec0b61 100755 --- a/src/DIRAC/Core/scripts/dirac-agent.py +++ b/src/DIRAC/Core/scripts/dirac-agent.py @@ -17,6 +17,9 @@ from DIRAC.Core.Base.AgentReactor import AgentReactor from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +print("NOTE:", __file__, "is deprecated and will be removed in v7r3, for details see", + "https://github.com/DIRACGrid/DIRAC/wiki/DIRAC-v7r2#rename-of-scripts") + localCfg = LocalConfiguration() positionalArgs = localCfg.getPositionalArguments() diff --git a/src/DIRAC/Core/scripts/dirac-executor.py b/src/DIRAC/Core/scripts/dirac-executor.py index 76b578c5017..bf03863132f 100755 --- a/src/DIRAC/Core/scripts/dirac-executor.py +++ b/src/DIRAC/Core/scripts/dirac-executor.py @@ -17,6 +17,9 @@ from DIRAC.Core.Base.ExecutorReactor import ExecutorReactor from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +print("NOTE:", __file__, "is deprecated and will be removed in v7r3, for details see", + "https://github.com/DIRACGrid/DIRAC/wiki/DIRAC-v7r2#rename-of-scripts") + localCfg = LocalConfiguration() positionalArgs = localCfg.getPositionalArguments() diff --git a/src/DIRAC/Core/scripts/dirac-service.py b/src/DIRAC/Core/scripts/dirac-service.py index 49db27aa5fd..39b54703879 100755 --- a/src/DIRAC/Core/scripts/dirac-service.py +++ b/src/DIRAC/Core/scripts/dirac-service.py @@ -16,6 +16,9 @@ __RCSID__ = "$Id$" +print("NOTE:", __file__, "is deprecated and will be removed in v7r3, for details see", + "https://github.com/DIRACGrid/DIRAC/wiki/DIRAC-v7r2#rename-of-scripts") + localCfg = LocalConfiguration() From 29f6fa842175bbaf7ff8fc329ec50e8d08c11fb4 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 14 Jan 2021 14:37:37 +0100 Subject: [PATCH 26/27] Improve console scripts documentation --- .../DevelopingCommands/dirac-my-great-script.py | 1 + .../AddingNewComponents/DevelopingCommands/index.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-my-great-script.py b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-my-great-script.py index ece39fb2217..d2633586853 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-my-great-script.py +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-my-great-script.py @@ -91,6 +91,7 @@ def parseSwitches(): return switches +# IMPORTANT: Make sure to add the console-scripts entry to setup.cfg as well! @DIRACScript() def main(): ''' diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst index ca2460579f9..d5cb6226026 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst @@ -53,7 +53,7 @@ which will set the interpreter directive to the python on the environment. if __name__ == "__main__": main() -**4.** Next the function must be registered as a ``console_scripts`` ``entrypoint`` in the ``setuptools`` metadata (`more details `_). This is done by adding a line to the ``console_scripts`` list like below, where the first string is the name for the script you want to create, the left hand side of ``:`` is the module that contains your function and the right hand side is the object you want to invoke (e.g. a function). +**4.** Next the function must be registered as a ``console_scripts`` ``entrypoint`` in the ``setuptools`` metadata (`more details `_). This is done by adding a line to the ``console_scripts`` list in ``setup.cfg`` like below, where the first string is the name for the script you want to create, the left hand side of ``:`` is the module that contains your function and the right hand side is the object you want to invoke (e.g. a function). .. code-block:: cfg From 3b627c791c3497675f4ce12f8a659968b195cf08 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 14 Jan 2021 17:30:05 +0100 Subject: [PATCH 27/27] Add setuptools_scm comment in pyproject.toml --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1cd983f7a52..70bf7d04371 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,4 +2,6 @@ requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] build-backend = "setuptools.build_meta" +# Enable setuptools_scm to compute the version number from the most recent tag +# https://github.com/pypa/setuptools_scm/#pyprojecttoml-usage [tool.setuptools_scm]