diff --git a/docs/diracdoctools/cmd/commandReference.py b/docs/diracdoctools/cmd/commandReference.py index 395c5e938f5..31204d01dc8 100644 --- a/docs/diracdoctools/cmd/commandReference.py +++ b/docs/diracdoctools/cmd/commandReference.py @@ -56,8 +56,8 @@ def getScripts(self): continue for mT in self.sectionDicts: - if any(pattern in scriptPath for pattern in mT[PATTERN]) and \ - not any(pattern in scriptPath for pattern in mT[EXCLUDE]): + if any(pattern in scriptPath.replace("_", "-") for pattern in mT[PATTERN]) and \ + not any(pattern in scriptPath.replace("_", "-") for pattern in mT[EXCLUDE]): mT[SCRIPTS].append(scriptPath) return @@ -93,6 +93,7 @@ def createFilesAndIndex(self, sectionDict): scriptName = os.path.basename(script) if scriptName.endswith('.py'): scriptName = scriptName[:-3] + scriptName = scriptName.replace("_", "-") prefix = sectionDict[PREFIX].lower() prefix = prefix + '_' if prefix else '' if self.createScriptDocFiles(script, sectionPath, scriptName, referencePrefix=prefix): @@ -120,6 +121,7 @@ def createFiles(self, sectionDict): scriptName = os.path.basename(script) if scriptName.endswith('.py'): scriptName = scriptName[:-3] + scriptName = scriptName.replace("_", "-") prefix = sectionDict[PREFIX].lower() prefix = prefix + '_' if prefix else '' if self.createScriptDocFiles(script, sectionPath, scriptName, referencePrefix=prefix) and \ @@ -138,7 +140,10 @@ def cleanExistingIndex(self, sectionDict): If an rst file exists for a command, we move it. An existing entry for a non existing rst file will create a warning when running sphinx. """ - existingCommands = {os.path.basename(com).replace('.py', '') for com in sectionDict[SCRIPTS] + sectionDict[MANUAL]} + existingCommands = { + os.path.basename(com).replace('.py', '').replace("_", "-") + for com in sectionDict[SCRIPTS] + sectionDict[MANUAL] + } sectionPath = os.path.join(self.config.docsPath, sectionDict[SECTION_PATH]) LOG.info('Checking %r for non-existent commands', sectionPath) # read the script index diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-my-great-script.py b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-my-great-script.py index eed6ab5f4b2..d2633586853 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-my-great-script.py +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-my-great-script.py @@ -17,11 +17,9 @@ __RCSID__ = '$Id$' from DIRAC import S_OK, S_ERROR, gLogger, exit as DIRACExit +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Base import Script -cliParams = None -switchDict = None - class Params(object): ''' @@ -93,10 +91,18 @@ def parseSwitches(): return switches +# IMPORTANT: Make sure to add the console-scripts entry to setup.cfg as well! +@DIRACScript() def main(): ''' This is the script main method, which will hold all the logic. ''' + # Script initialization + registerSwitches() + switchDict = parseSwitches() + + # Import the required DIRAC modules + from DIRAC.Interfaces.API.Dirac import Dirac # let's do something if not len(switchDict['servicesList']): @@ -104,18 +110,8 @@ def main(): DIRACExit(1) gLogger.notice('We are done') + DIRACExit(0) -if __name__ == "__main__": - - # Script initialization - registerSwitches() - switchDict = parseSwitches() - - # Import the required DIRAC modules - from DIRAC.Interfaces.API.Dirac import Dirac - # Run the script +if __name__ == "__main__": main() - - # Bye - DIRACExit(0) diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-ping-info.py b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-ping-info.py index dc7ec21f8c6..5128c8aff0e 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-ping-info.py +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac-ping-info.py @@ -12,10 +12,10 @@ from DIRAC import exit as DIRACExit from DIRAC import S_OK, S_ERROR from DIRAC.Core.Base import Script - -# Define a simple class to hold the script parameters +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +# Define a simple class to hold the script parameters class Params(object): def __init__(self): @@ -34,25 +34,33 @@ def setNumOfPingsToDo(self, value): return S_OK() -# Instantiate the params class -cliParams = Params() +@DIRACScript() +def main(): + # Instantiate the params class + cliParams = Params() + + # Register accepted switches and their callbacks + Script.registerSwitch("r", "showRaw", "show raw result from the query", cliParams.setRawResult) + Script.registerSwitch("p:", "numPings=", "Number of pings to do (by default 1)", cliParams.setNumOfPingsToDo) + + # Define a help message + Script.setUsageMessage('\n'.join([__doc__, + 'Usage:', + ' %s [option|cfgfile] +' % Script.scriptName, + ' Specifying a system is mandatory'])) + + # Parse the command line and initialize DIRAC + Script.parseCommandLine(ignoreErrors=False) -# Register accepted switches and their callbacks -Script.registerSwitch("r", "showRaw", "show raw result from the query", cliParams.setRawResult) -Script.registerSwitch("p:", "numPings=", "Number of pings to do (by default 1)", cliParams.setNumOfPingsToDo) + # Get the list of services + servicesList = Script.getPositionalArgs() -# Define a help message -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s [option|cfgfile] +' % Script.scriptName, - ' Specifying a system is mandatory'])) + # Check and process the command line switches and options + if not servicesList: + Script.showHelp(exitCode=1) -# Parse the command line and initialize DIRAC -Script.parseCommandLine(ignoreErrors=False) + # Do something! -# Get the list of services -servicesList = Script.getPositionalArgs() -# Check and process the command line switches and options -if not servicesList: - Script.showHelp(exitCode=1) +if __name__ == "__main__": + main() diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst index ca352e5bace..d5cb6226026 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.rst @@ -2,6 +2,10 @@ Developing Commands ====================================== +.. warning:: + This instructions here demonstrate how to support both the legacy (Python 2) and future (Python3) installations of DIRAC. + If only having Python 3 support is acceptable, the requirement for scripts to be in the the *scripts* directory of their parent system will be removed and the only requirement will be for the function to be decorated with the ``@DIRACScript()`` decorator. + Commands are one of the main interface tools for the users. Commands are also called *scripts* in DIRAC lingo. Where to place scripts @@ -31,9 +35,33 @@ The instructions below must be applied as close as possible although some variat which will set the interpreter directive to the python on the environment. -**2.** The next is the documentation line which is describing the command. This same documentation line will be used also the command help information available with the *-h* command switch. +**2.** The next is the documentation line which is describing the command. This same documentation line will be used also the command help information available with the *-h* command switch. + +**3.** The majority of the code should be contained with a function, often called ``main`` though this is not required. This function should be wrapped with the ``@DiracScript()`` decorator to allow the DIRAC plugin mechanism to override the script with the function from the highest priority extension. + +.. code-block:: python + + #Import the required DIRAC modules + from DIRAC.Core.Utilities.DIRACScript import DIRACScript + from DIRAC.Interfaces.API.DIRAC import DIRAC + from DIRAC import gLogger + + @DiracScript() + def main() + # Do stuff + + if __name__ == "__main__": + main() + +**4.** Next the function must be registered as a ``console_scripts`` ``entrypoint`` in the ``setuptools`` metadata (`more details `_). This is done by adding a line to the ``console_scripts`` list in ``setup.cfg`` like below, where the first string is the name for the script you want to create, the left hand side of ``:`` is the module that contains your function and the right hand side is the object you want to invoke (e.g. a function). -**3.** Users need to specify parameters to scripts to define what they want to do. To do so, they pass arguments when calling the script. The first thing any script has to do is define what options and arguments the script accepts. Once the valid arguments are defined, the script can parse the command line. An example follows which is a typical command description part +.. code-block:: cfg + + console_scripts = + dirac-info = DIRAC.Core.scripts.dirac_info:main + dirac-proxy-info = DIRAC.FrameworkSystem.scripts.dirac_proxy_info:main + +**5.** Users need to specify parameters to scripts to define what they want to do. To do so, they pass arguments when calling the script. The first thing any script has to do is define what options and arguments the script accepts. Once the valid arguments are defined, the script can parse the command line. An example follows which is a typical command description part .. literalinclude:: dirac-ping-info.py @@ -61,22 +89,7 @@ All the command line arguments that are not corresponding to the explicitly defi After defining the switches, the *parseCommandLine()* function has to be called. This method not only parses the command line options but also initializes DIRAC collecting all the configuration data. **It is absolutely important to call this function before importing any other DIRAC module**. The callbacks defined for the switches will be called when parsing the command line if necessary. *Even if the switch is not supposed to receive a parameter, the callback has to receive a value*. Switches without callbacks defined can be obtained with *getUnprocessedSwitches()* function. -**4.** Once the command line has been parsed and DIRAC is properly initialized, the rest of the required DIRAC modules can be imported and the script logic can take place: - -.. code-block:: python - - #Import the required DIRAC modules - from DIRAC.Interfaces.API.DIRAC import DIRAC - from DIRAC import gLogger - #Do stuff... depending on cliParams.raw, cliParams.pingsToDo and servicesList - - def executeCommandLogic() - # Do stuff - gLogger.notice('This is the result of the command') - - if __name__ == "__main__": - executeCommandLogic() - +**5.** Once the command line has been parsed and DIRAC is properly initialized, the rest of the required DIRAC modules can be imported and the script logic can take place. Having understood the logic of the script, there are few good practices that must be followed: @@ -84,8 +97,6 @@ Having understood the logic of the script, there are few good practices that mus * Encapsulate the command code into functions / classes so that it can be easily tested * Usage of *gLogger* instead of *print* is mandatory. The information in the normal command execution must be printed out in the NOTICE logging level. -* Use the *if __name__ == "__main__"* close for the actual command execution to avoid running the script - when it is imported. Example command ----------------- diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..70bf7d04371 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] +build-backend = "setuptools.build_meta" + +# Enable setuptools_scm to compute the version number from the most recent tag +# https://github.com/pypa/setuptools_scm/#pyprojecttoml-usage +[tool.setuptools_scm] diff --git a/setup.cfg b/setup.cfg index 7b75664a44d..745ea3a9f88 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,277 @@ -# This file should be used as the pep8/pycodestyle and autopep8 config +############################################################################### +# Setuptools +############################################################################### + +[metadata] +name = DIRAC +description = DIRAC is an interware, meaning a software framework for distributed computing. +long_description = file: README.rst +license = GPL-3.0-only +classifiers = + License :: OSI Approved :: GNU General Public License v3 (GPLv3) + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + +[options] +python_requires = >=3.8 +package_dir= + =src +packages = find: +# TODO: This should be treated as a legacy workaround and eventually removed +scripts = + src/DIRAC/Core/scripts/genAllCAs.sh + src/DIRAC/Core/scripts/dirac-cert-convert.sh + src/DIRAC/Core/scripts/install_site.sh + src/DIRAC/Core/scripts/genRevokedCerts.sh +install_requires = + boto3 + botocore + certifi + diraccfg + fts3-rest + future + gfal2-python + M2Crypto >=0.36 + pexpect + psutil + pyasn1 + pyasn1-modules + pyparsing + python-dateutil + pytz + requests + setuptools + six + sqlalchemy + subprocess32 + tornado ~=5.1.1 + +[options.packages.find] +where=src + +[options.extras_require] +server = + # arc6 doesn't provide a proper Python package with metadata + # (it just installs into site-packages) + # arc + CMRESHandler + elasticsearch + elasticsearch_dsl + GitPython + ldap3 + matplotlib + mysqlclient + numpy + pillow + python-irodsclient + python-json-logger + stomp.py + suds-jurko + tornado + tornado-m2crypto +testing = + flaky + hypothesis + mock + parameterized + pytest + +[options.entry_points] +dirac = + metadata = DIRAC:extension_metadata +# For details on adding additional scripts to vanilla DIRAC or external extensions see +# https://dirac.readthedocs.io/en/integration/DeveloperGuide/AddingNewComponents/DevelopingCommands/index.html +console_scripts = + # AccountingSystem + dirac-accounting-decode-fileid = DIRAC.AccountingSystem.scripts.dirac_accounting_decode_fileid:main + dirac-admin-accounting-cli = DIRAC.AccountingSystem.scripts.dirac_admin_accounting_cli:main + # ConfigurationSystem + dirac-admin-add-resources = DIRAC.ConfigurationSystem.scripts.dirac_admin_add_resources:main + dirac-admin-add-shifter = DIRAC.ConfigurationSystem.scripts.dirac_admin_add_shifter:main + dirac-admin-add-site = DIRAC.ConfigurationSystem.scripts.dirac_admin_add_site:main + dirac-admin-bdii-info = DIRAC.ConfigurationSystem.scripts.dirac_admin_bdii_info:main + dirac-admin-check-config-options = DIRAC.ConfigurationSystem.scripts.dirac_admin_check_config_options:main + dirac-admin-sort-cs-sites = DIRAC.ConfigurationSystem.scripts.dirac_admin_sort_cs_sites:main + dirac-admin-voms-sync = DIRAC.ConfigurationSystem.scripts.dirac_admin_voms_sync:main + dirac-configuration-cli = DIRAC.ConfigurationSystem.scripts.dirac_configuration_cli:main + dirac-configuration-dump-local-cache = DIRAC.ConfigurationSystem.scripts.dirac_configuration_dump_local_cache:main + dirac-configuration-shell = DIRAC.ConfigurationSystem.scripts.dirac_configuration_shell:main + # Core + dirac-agent = DIRAC.Core.scripts.dirac_agent:main + dirac-configure = DIRAC.Core.scripts.dirac_configure:main + dirac-executor = DIRAC.Core.scripts.dirac_executor:main + dirac-externals-requirements = DIRAC.Core.scripts.dirac_externals_requirements:main + dirac-info = DIRAC.Core.scripts.dirac_info:main + dirac-install-db = DIRAC.Core.scripts.dirac_install_db:main + dirac-install-web-portal = DIRAC.Core.scripts.dirac_install_web_portal:main + dirac-platform = DIRAC.Core.scripts.dirac_platform:main + dirac-service = DIRAC.Core.scripts.dirac_service:main + dirac-setup-site = DIRAC.Core.scripts.dirac_setup_site:main + dirac-version = DIRAC.Core.scripts.dirac_version:main + # Core.Tornado + tornado-start-CS = DIRAC.Core.Tornado.scripts.tornado_start_CS:main + tornado-start-all = DIRAC.Core.Tornado.scripts.tornado_start_all:main + # DataManagementSystem + dirac-admin-allow-se = DIRAC.DataManagementSystem.scripts.dirac_admin_allow_se:main + dirac-admin-ban-se = DIRAC.DataManagementSystem.scripts.dirac_admin_ban_se:main + dirac-admin-user-quota = DIRAC.DataManagementSystem.scripts.dirac_admin_user_quota:main + dirac-dms-add-file = DIRAC.DataManagementSystem.scripts.dirac_dms_add_file:main + dirac-dms-catalog-metadata = DIRAC.DataManagementSystem.scripts.dirac_dms_catalog_metadata:main + dirac-dms-change-replica-status = DIRAC.DataManagementSystem.scripts.dirac_dms_change_replica_status:main + dirac-dms-clean-directory = DIRAC.DataManagementSystem.scripts.dirac_dms_clean_directory:main + dirac-dms-create-archive-request = DIRAC.DataManagementSystem.scripts.dirac_dms_create_archive_request:main + dirac-dms-create-moving-request = DIRAC.DataManagementSystem.scripts.dirac_dms_create_moving_request:main + dirac-dms-create-removal-request = DIRAC.DataManagementSystem.scripts.dirac_dms_create_removal_request:main + dirac-dms-data-size = DIRAC.DataManagementSystem.scripts.dirac_dms_data_size:main + dirac-dms-directory-sync = DIRAC.DataManagementSystem.scripts.dirac_dms_directory_sync:main + dirac-dms-filecatalog-cli = DIRAC.DataManagementSystem.scripts.dirac_dms_filecatalog_cli:main + dirac-dms-find-lfns = DIRAC.DataManagementSystem.scripts.dirac_dms_find_lfns:main + dirac-dms-move-replica-request = DIRAC.DataManagementSystem.scripts.dirac_dms_move_replica_request:main + dirac-dms-protocol-matrix = DIRAC.DataManagementSystem.scripts.dirac_dms_protocol_matrix:main + dirac-dms-put-and-register-request = DIRAC.DataManagementSystem.scripts.dirac_dms_put_and_register_request:main + dirac-dms-remove-catalog-files = DIRAC.DataManagementSystem.scripts.dirac_dms_remove_catalog_files:main + dirac-dms-remove-catalog-replicas = DIRAC.DataManagementSystem.scripts.dirac_dms_remove_catalog_replicas:main + dirac-dms-remove-files = DIRAC.DataManagementSystem.scripts.dirac_dms_remove_files:main + dirac-dms-remove-replicas = DIRAC.DataManagementSystem.scripts.dirac_dms_remove_replicas:main + dirac-dms-replica-metadata = DIRAC.DataManagementSystem.scripts.dirac_dms_replica_metadata:main + dirac-dms-replicate-and-register-request = DIRAC.DataManagementSystem.scripts.dirac_dms_replicate_and_register_request:main + dirac-dms-resolve-guid = DIRAC.DataManagementSystem.scripts.dirac_dms_resolve_guid:main + dirac-dms-set-replica-status = DIRAC.DataManagementSystem.scripts.dirac_dms_set_replica_status:main + dirac-dms-show-se-status = DIRAC.DataManagementSystem.scripts.dirac_dms_show_se_status:main + dirac-dms-user-lfns = DIRAC.DataManagementSystem.scripts.dirac_dms_user_lfns:main + dirac-dms-user-quota = DIRAC.DataManagementSystem.scripts.dirac_dms_user_quota:main + # FrameworkSystem + dirac-admin-get-CAs = DIRAC.FrameworkSystem.scripts.dirac_admin_get_CAs:main + dirac-admin-get-proxy = DIRAC.FrameworkSystem.scripts.dirac_admin_get_proxy:main + dirac-admin-proxy-upload = DIRAC.FrameworkSystem.scripts.dirac_admin_proxy_upload:main + dirac-admin-sysadmin-cli = DIRAC.FrameworkSystem.scripts.dirac_admin_sysadmin_cli:main + dirac-admin-update-instance = DIRAC.FrameworkSystem.scripts.dirac_admin_update_instance:main + dirac-admin-update-pilot = DIRAC.FrameworkSystem.scripts.dirac_admin_update_pilot:main + dirac-admin-users-with-proxy = DIRAC.FrameworkSystem.scripts.dirac_admin_users_with_proxy:main + dirac-install-component = DIRAC.FrameworkSystem.scripts.dirac_install_component:main + dirac-install-tornado-service = DIRAC.FrameworkSystem.scripts.dirac_install_tornado_service:main + dirac-monitoring-get-components-status = DIRAC.FrameworkSystem.scripts.dirac_monitoring_get_components_status:main + dirac-myproxy-upload = DIRAC.FrameworkSystem.scripts.dirac_myproxy_upload:main + dirac-populate-component-db = DIRAC.FrameworkSystem.scripts.dirac_populate_component_db:main + dirac-proxy-destroy = DIRAC.FrameworkSystem.scripts.dirac_proxy_destroy:main + dirac-proxy-get-uploaded-info = DIRAC.FrameworkSystem.scripts.dirac_proxy_get_uploaded_info:main + dirac-proxy-info = DIRAC.FrameworkSystem.scripts.dirac_proxy_info:main + dirac-proxy-init = DIRAC.FrameworkSystem.scripts.dirac_proxy_init:main + dirac-restart-component = DIRAC.FrameworkSystem.scripts.dirac_restart_component:main + dirac-start-component = DIRAC.FrameworkSystem.scripts.dirac_start_component:main + dirac-status-component = DIRAC.FrameworkSystem.scripts.dirac_status_component:main + dirac-stop-component = DIRAC.FrameworkSystem.scripts.dirac_stop_component:main + dirac-sys-sendmail = DIRAC.FrameworkSystem.scripts.dirac_sys_sendmail:main + dirac-uninstall-component = DIRAC.FrameworkSystem.scripts.dirac_uninstall_component:main + # Interfaces + dirac-admin-add-group = DIRAC.Interfaces.scripts.dirac_admin_add_group:main + dirac-admin-add-host = DIRAC.Interfaces.scripts.dirac_admin_add_host:main + dirac-admin-add-user = DIRAC.Interfaces.scripts.dirac_admin_add_user:main + dirac-admin-allow-site = DIRAC.Interfaces.scripts.dirac_admin_allow_site:main + dirac-admin-ban-site = DIRAC.Interfaces.scripts.dirac_admin_ban_site:main + dirac-admin-ce-info = DIRAC.Interfaces.scripts.dirac_admin_ce_info:main + dirac-admin-delete-user = DIRAC.Interfaces.scripts.dirac_admin_delete_user:main + dirac-admin-get-banned-sites = DIRAC.Interfaces.scripts.dirac_admin_get_banned_sites:main + dirac-admin-get-job-pilot-output = DIRAC.Interfaces.scripts.dirac_admin_get_job_pilot_output:main + dirac-admin-get-job-pilots = DIRAC.Interfaces.scripts.dirac_admin_get_job_pilots:main + dirac-admin-get-pilot-info = DIRAC.Interfaces.scripts.dirac_admin_get_pilot_info:main + dirac-admin-get-pilot-logging-info = DIRAC.Interfaces.scripts.dirac_admin_get_pilot_logging_info:main + dirac-admin-get-pilot-output = DIRAC.Interfaces.scripts.dirac_admin_get_pilot_output:main + dirac-admin-get-site-mask = DIRAC.Interfaces.scripts.dirac_admin_get_site_mask:main + dirac-admin-list-hosts = DIRAC.Interfaces.scripts.dirac_admin_list_hosts:main + dirac-admin-list-users = DIRAC.Interfaces.scripts.dirac_admin_list_users:main + dirac-admin-modify-user = DIRAC.Interfaces.scripts.dirac_admin_modify_user:main + dirac-admin-pilot-summary = DIRAC.Interfaces.scripts.dirac_admin_pilot_summary:main + dirac-admin-reset-job = DIRAC.Interfaces.scripts.dirac_admin_reset_job:main + dirac-admin-service-ports = DIRAC.Interfaces.scripts.dirac_admin_service_ports:main + dirac-admin-set-site-protocols = DIRAC.Interfaces.scripts.dirac_admin_set_site_protocols:main + dirac-admin-site-info = DIRAC.Interfaces.scripts.dirac_admin_site_info:main + dirac-admin-site-mask-logging = DIRAC.Interfaces.scripts.dirac_admin_site_mask_logging:main + dirac-admin-sync-users-from-file = DIRAC.Interfaces.scripts.dirac_admin_sync_users_from_file:main + dirac-dms-get-file = DIRAC.Interfaces.scripts.dirac_dms_get_file:main + dirac-dms-lfn-accessURL = DIRAC.Interfaces.scripts.dirac_dms_lfn_accessURL:main + dirac-dms-lfn-metadata = DIRAC.Interfaces.scripts.dirac_dms_lfn_metadata:main + dirac-dms-lfn-replicas = DIRAC.Interfaces.scripts.dirac_dms_lfn_replicas:main + dirac-dms-pfn-accessURL = DIRAC.Interfaces.scripts.dirac_dms_pfn_accessURL:main + dirac-dms-pfn-metadata = DIRAC.Interfaces.scripts.dirac_dms_pfn_metadata:main + dirac-dms-replicate-lfn = DIRAC.Interfaces.scripts.dirac_dms_replicate_lfn:main + dirac-framework-ping-service = DIRAC.Interfaces.scripts.dirac_framework_ping_service:main + dirac-framework-self-ping = DIRAC.Interfaces.scripts.dirac_framework_self_ping:main + dirac-repo-monitor = DIRAC.Interfaces.scripts.dirac_repo_monitor:main + dirac-utils-file-adler = DIRAC.Interfaces.scripts.dirac_utils_file_adler:main + dirac-utils-file-md5 = DIRAC.Interfaces.scripts.dirac_utils_file_md5:main + dirac-wms-get-normalized-queue-length = DIRAC.Interfaces.scripts.dirac_wms_get_normalized_queue_length:main + dirac-wms-get-queue-normalization = DIRAC.Interfaces.scripts.dirac_wms_get_queue_normalization:main + dirac-wms-job-attributes = DIRAC.Interfaces.scripts.dirac_wms_job_attributes:main + dirac-wms-job-delete = DIRAC.Interfaces.scripts.dirac_wms_job_delete:main + dirac-wms-job-get-input = DIRAC.Interfaces.scripts.dirac_wms_job_get_input:main + dirac-wms-job-get-jdl = DIRAC.Interfaces.scripts.dirac_wms_job_get_jdl:main + dirac-wms-job-get-output = DIRAC.Interfaces.scripts.dirac_wms_job_get_output:main + dirac-wms-job-get-output-data = DIRAC.Interfaces.scripts.dirac_wms_job_get_output_data:main + dirac-wms-job-kill = DIRAC.Interfaces.scripts.dirac_wms_job_kill:main + dirac-wms-job-logging-info = DIRAC.Interfaces.scripts.dirac_wms_job_logging_info:main + dirac-wms-job-parameters = DIRAC.Interfaces.scripts.dirac_wms_job_parameters:main + dirac-wms-job-peek = DIRAC.Interfaces.scripts.dirac_wms_job_peek:main + dirac-wms-job-reschedule = DIRAC.Interfaces.scripts.dirac_wms_job_reschedule:main + dirac-wms-job-status = DIRAC.Interfaces.scripts.dirac_wms_job_status:main + dirac-wms-job-submit = DIRAC.Interfaces.scripts.dirac_wms_job_submit:main + dirac-wms-jobs-select-output-search = DIRAC.Interfaces.scripts.dirac_wms_jobs_select_output_search:main + dirac-wms-select-jobs = DIRAC.Interfaces.scripts.dirac_wms_select_jobs:main + # ProductionSystem + dirac-prod-add-trans = DIRAC.ProductionSystem.scripts.dirac_prod_add_trans:main + dirac-prod-clean = DIRAC.ProductionSystem.scripts.dirac_prod_clean:main + dirac-prod-delete = DIRAC.ProductionSystem.scripts.dirac_prod_delete:main + dirac-prod-get = DIRAC.ProductionSystem.scripts.dirac_prod_get:main + dirac-prod-get-all = DIRAC.ProductionSystem.scripts.dirac_prod_get_all:main + dirac-prod-get-description = DIRAC.ProductionSystem.scripts.dirac_prod_get_description:main + dirac-prod-get-trans = DIRAC.ProductionSystem.scripts.dirac_prod_get_trans:main + dirac-prod-start = DIRAC.ProductionSystem.scripts.dirac_prod_start:main + dirac-prod-stop = DIRAC.ProductionSystem.scripts.dirac_prod_stop:main + # RequestManagementSystem + dirac-rms-list-req-cache = DIRAC.RequestManagementSystem.scripts.dirac_rms_list_req_cache:main + dirac-rms-reqdb-summary = DIRAC.RequestManagementSystem.scripts.dirac_rms_reqdb_summary:main + dirac-rms-request = DIRAC.RequestManagementSystem.scripts.dirac_rms_request:main + # ResourceStatusSystem + dirac-rss-list-status = DIRAC.ResourceStatusSystem.scripts.dirac_rss_list_status:main + dirac-rss-query-db = DIRAC.ResourceStatusSystem.scripts.dirac_rss_query_db:main + dirac-rss-query-dtcache = DIRAC.ResourceStatusSystem.scripts.dirac_rss_query_dtcache:main + dirac-rss-set-status = DIRAC.ResourceStatusSystem.scripts.dirac_rss_set_status:main + dirac-rss-set-token = DIRAC.ResourceStatusSystem.scripts.dirac_rss_set_token:main + dirac-rss-sync = DIRAC.ResourceStatusSystem.scripts.dirac_rss_sync:main + # StorageManagementSystem + dirac-stager-monitor-file = DIRAC.StorageManagementSystem.scripts.dirac_stager_monitor_file:main + dirac-stager-monitor-jobs = DIRAC.StorageManagementSystem.scripts.dirac_stager_monitor_jobs:main + dirac-stager-monitor-request = DIRAC.StorageManagementSystem.scripts.dirac_stager_monitor_request:main + dirac-stager-monitor-requests = DIRAC.StorageManagementSystem.scripts.dirac_stager_monitor_requests:main + dirac-stager-show-stats = DIRAC.StorageManagementSystem.scripts.dirac_stager_show_stats:main + dirac-stager-stage-files = DIRAC.StorageManagementSystem.scripts.dirac_stager_stage_files:main + # TransformationSystem + dirac-production-runjoblocal = DIRAC.TransformationSystem.scripts.dirac_production_runjoblocal:main + dirac-transformation-add-files = DIRAC.TransformationSystem.scripts.dirac_transformation_add_files:main + dirac-transformation-archive = DIRAC.TransformationSystem.scripts.dirac_transformation_archive:main + dirac-transformation-clean = DIRAC.TransformationSystem.scripts.dirac_transformation_clean:main + dirac-transformation-cli = DIRAC.TransformationSystem.scripts.dirac_transformation_cli:main + dirac-transformation-get-files = DIRAC.TransformationSystem.scripts.dirac_transformation_get_files:main + dirac-transformation-recover-data = DIRAC.TransformationSystem.scripts.dirac_transformation_recover_data:main + dirac-transformation-remove-output = DIRAC.TransformationSystem.scripts.dirac_transformation_remove_output:main + dirac-transformation-verify-outputdata = DIRAC.TransformationSystem.scripts.dirac_transformation_verify_outputdata:main + # WorkloadManagementSystem + dirac-admin-kill-pilot = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_kill_pilot:main + dirac-admin-pilot-logging-info = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_pilot_logging_info:main + dirac-admin-show-task-queues = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_show_task_queues:main + dirac-admin-sync-pilot = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_sync_pilot:main + dirac-jobexec = DIRAC.WorkloadManagementSystem.scripts.dirac_jobexec:main + dirac-wms-cpu-normalization = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_cpu_normalization:main + dirac-wms-get-queue-cpu-time = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_get_queue_cpu_time:main + dirac-wms-get-wn = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_get_wn:main + dirac-wms-get-wn-parameters = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_get_wn_parameters:main + dirac-wms-match = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_match:main + dirac-wms-pilot-job-info = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_pilot_job_info:main + +############################################################################### +# Linting +############################################################################### # pep8 binary will still look into this section [pep8] diff --git a/setup.py b/setup.py index 577c0c0181a..dcded7c1fbf 100755 --- a/setup.py +++ b/setup.py @@ -1,45 +1,8 @@ -""" Basic setuptools script for DIRAC. - Does not contain any dependency -""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os -import glob -# Actual setuptools -from setuptools import setup, find_packages +from setuptools import setup -# Find the base dir where the setup.py lies -base_dir = os.path.abspath(os.path.dirname(__file__)) - -# Take all the packages but the scripts and tests -allPackages = find_packages(where=base_dir, exclude=["*test*", "*scripts*", "*docs*"]) - -# Because we want to have a 'DIRAC' base module and that the setup.py -# is lying inside it, we need to define a mapping -# < module name : directory > -# e.g. DIRAC.DataManagementSystem is base_dir/DataManagementSystem - -package_dir = dict(("DIRAC.%s" % p, os.path.join(base_dir, p.replace('.', '/'))) for p in allPackages) - -# We also rename the packages so that they contain DIRAC -allPackages = ['DIRAC.%s' % p for p in allPackages] - -# Artificially create the 'DIRAC' package -# at the root -allPackages.insert(0, 'DIRAC') -package_dir['DIRAC'] = base_dir - -# The scripts to be distributed -scripts = glob.glob('%s/*/scripts/*.py' % base_dir) - -setup( - name="DIRAC", - version="7.2.pre28", - url="https://github.com/DIRACGRID/DIRAC", - license="GPLv3", - package_dir=package_dir, - packages=allPackages, - scripts=scripts, -) +# This is required to allow editable pip installs while using the declarative configuration (setup.cfg) +setup() diff --git a/src/DIRAC/AccountingSystem/scripts/__init__.py b/src/DIRAC/AccountingSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/AccountingSystem/scripts/dirac-accounting-decode-fileid.py b/src/DIRAC/AccountingSystem/scripts/dirac-accounting-decode-fileid.py deleted file mode 100755 index 0dd71abdacf..00000000000 --- a/src/DIRAC/AccountingSystem/scripts/dirac-accounting-decode-fileid.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-accounting-decode-fileid -# Author : Adria Casajus -######################################################################## -""" - Decode Accounting plot URLs -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -import pprint -import sys -from six.moves.urllib import parse as urlparse -import cgi -from DIRAC import gLogger -from DIRAC.Core.Base import Script -from DIRAC.Core.Utilities.Plotting.FileCoding import extractRequestFromFileId - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... URL ...' % Script.scriptName, - 'Arguments:', - ' URL: encoded URL of a DIRAC Accounting plot'])) -Script.parseCommandLine() - -fileIds = Script.getPositionalArgs() - -for fileId in fileIds: - # Try to find if it's a url - parseRes = urlparse.urlparse(fileId) - if parseRes.query: - queryRes = cgi.parse_qs(parseRes.query) - if 'file' in queryRes: - fileId = queryRes['file'][0] - # Decode - result = extractRequestFromFileId(fileId) - if not result['OK']: - gLogger.error("Could not decode fileId", "'%s', error was %s" % (fileId, result['Message'])) - sys.exit(1) - gLogger.notice("Decode for '%s' is:\n%s" % (fileId, pprint.pformat(result['Value']))) - -sys.exit(0) diff --git a/src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py b/src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py new file mode 100755 index 00000000000..3c749e2ce47 --- /dev/null +++ b/src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-accounting-decode-fileid +# Author : Adria Casajus +######################################################################## +""" + Decode Accounting plot URLs +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +import pprint +import sys +from six.moves.urllib import parse as urlparse +import cgi +from DIRAC import gLogger +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.Core.Utilities.Plotting.FileCoding import extractRequestFromFileId + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... URL ...' % Script.scriptName, + 'Arguments:', + ' URL: encoded URL of a DIRAC Accounting plot'])) + Script.parseCommandLine() + + fileIds = Script.getPositionalArgs() + + for fileId in fileIds: + # Try to find if it's a url + parseRes = urlparse.urlparse(fileId) + if parseRes.query: + queryRes = cgi.parse_qs(parseRes.query) + if 'file' in queryRes: + fileId = queryRes['file'][0] + # Decode + result = extractRequestFromFileId(fileId) + if not result['OK']: + gLogger.error("Could not decode fileId", "'%s', error was %s" % (fileId, result['Message'])) + sys.exit(1) + gLogger.notice("Decode for '%s' is:\n%s" % (fileId, pprint.pformat(result['Value']))) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/AccountingSystem/scripts/dirac-admin-accounting-cli.py b/src/DIRAC/AccountingSystem/scripts/dirac_admin_accounting_cli.py similarity index 55% rename from src/DIRAC/AccountingSystem/scripts/dirac-admin-accounting-cli.py rename to src/DIRAC/AccountingSystem/scripts/dirac_admin_accounting_cli.py index 838fe5d5c67..1847340c011 100755 --- a/src/DIRAC/AccountingSystem/scripts/dirac-admin-accounting-cli.py +++ b/src/DIRAC/AccountingSystem/scripts/dirac_admin_accounting_cli.py @@ -13,15 +13,22 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.localCfg.addDefaultEntry("LogLevel", "info") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % Script.scriptName, ])) -Script.parseCommandLine() -from DIRAC.AccountingSystem.Client.AccountingCLI import AccountingCLI +@DIRACScript() +def main(): + Script.localCfg.addDefaultEntry("LogLevel", "info") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % Script.scriptName, ])) + Script.parseCommandLine() + + from DIRAC.AccountingSystem.Client.AccountingCLI import AccountingCLI -if __name__ == "__main__": acli = AccountingCLI() acli.start() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/__init__.py b/src/DIRAC/ConfigurationSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-sort-cs-sites.py b/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-sort-cs-sites.py deleted file mode 100755 index a9faf2e3028..00000000000 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-sort-cs-sites.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# $HeadURL$ -# File : dirac-admin-sort-cs-sites -# Author : Matvey Sapunov -######################################################################## -""" - Sort site names at CS in "/Resources" section. Sort can be alphabetic or by country postfix in a site name. - Alphabetic sort is default (i.e. LCG.IHEP.cn, LCG.IHEP.su, LCG.IN2P3.fr) - - Options: - -C --country Sort site names by country postfix (i.e. LCG.IHEP.cn, LCG.IN2P3.fr, LCG.IHEP.su) - -R --reverse Reverse the sort order - - Argument: - Name of the subsection in the CS '/Resources/Sites/' section to be sorted (i.e. LCG, DIRAC) - - Example: dirac-admin-sort-cs-sites -C DIRAC - Sort sites in subsection /Resources/Sites/DIRAC by country postfix -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -from DIRAC import gLogger, exit as DIRACExit -from DIRAC.Core.Base import Script -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getPropertiesForGroup -from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI -from DIRAC.Core.Utilities.Time import dateTime, toString - -global SORTBYNAME, REVERSE -SORTBYNAME = True -REVERSE = False - - -def sortBy(arg): - global SORTBYNAME - SORTBYNAME = False - - -def isReverse(arg): - global REVERSE - REVERSE = True - - -def country(arg): - cb = arg.split(".") - if not len(cb) == 3: - gLogger.error("%s is not in GRID.NAME.COUNTRY format ") - return False - return cb[2] - - -Script.registerSwitch( - "C", - "country", - "Sort site names by country postfix (i.e. LCG.IHEP.cn, LCG.IN2P3.fr, LCG.IHEP.su)", - sortBy) -Script.registerSwitch("R", "reverse", "Reverse the sort order", isReverse) - -Script.setUsageMessage( - "\n".join( - [ - __doc__.split("\n")[1], - "Usage:", - " %s [option|cfgfile]
" % - Script.scriptName, - "Optional arguments:", - " Section: Name of the subsection in '/Resources/Sites/' for sort (i.e. LCG DIRAC)", - "Example:", - " dirac-admin-sort-cs-sites -C CLOUDS DIRAC", - " sort site names by country postfix in '/Resources/Sites/CLOUDS' and '/Resources/Sites/DIRAC' subsection", - ""])) - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -result = getProxyInfo() -if not result["OK"]: - gLogger.error("Failed to get proxy information", result["Message"]) - DIRACExit(2) -proxy = result["Value"] -if proxy["secondsLeft"] < 1: - gLogger.error("Your proxy has expired, please create new one") - DIRACExit(2) -group = proxy["group"] -if "CSAdministrator" not in getPropertiesForGroup(group): - gLogger.error("You must be CSAdministrator user to execute this script") - gLogger.notice("Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'") - DIRACExit(2) - -cs = CSAPI() -result = cs.getCurrentCFG() -if not result["OK"]: - gLogger.error("Failed to get copy of CS", result["Message"]) - DIRACExit(2) -cfg = result["Value"] - -if not cfg.isSection("Resources"): - gLogger.error("Section '/Resources' is absent in CS") - DIRACExit(2) - -if not cfg.isSection("Resources/Sites"): - gLogger.error("Subsection '/Resources/Sites' is absent in CS") - DIRACExit(2) - - -if args and len(args) > 0: - resultList = args[:] -else: - resultList = cfg["Resources"]["Sites"].listSections() - -hasRun = False -isDirty = False -for i in resultList: - if not cfg.isSection("Resources/Sites/%s" % i): - gLogger.error("Subsection /Resources/Sites/%s does not exists" % i) - continue - hasRun = True - if SORTBYNAME: - dirty = cfg["Resources"]["Sites"][i].sortAlphabetically(ascending=not REVERSE) - else: - dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country, reverse=REVERSE) - if dirty: - isDirty = True - -if not hasRun: - gLogger.notice("Failed to find suitable subsections with site names to sort") - DIRACExit(0) - -if not isDirty: - gLogger.notice("Nothing to do, site names are already sorted") - DIRACExit(0) - -timestamp = toString(dateTime()) -stamp = "Site names are sorted by %s script at %s" % (Script.scriptName, timestamp) -cs.setOptionComment("/Resources/Sites", stamp) - -result = cs.commit() -if not result["OK"]: - gLogger.error("Failed to commit changes to CS", result["Message"]) - DIRACExit(2) -gLogger.notice("Site names are sorted and committed to CS") -DIRACExit(0) diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-voms-sync.py b/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-voms-sync.py deleted file mode 100755 index a00df5d0ba5..00000000000 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-voms-sync.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-voms-sync -# Author : Andrei Tsaregorodtsev -######################################################################## -""" - Synchronize VOMS user data with the DIRAC Registry -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -from DIRAC import gLogger, exit as DIRACExit, S_OK -from DIRAC.Core.Base import Script -from DIRAC.ConfigurationSystem.Client.VOMS2CSSynchronizer import VOMS2CSSynchronizer -from DIRAC.Core.Utilities.Proxy import executeWithUserProxy -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOOption - - -dryRun = False - - -def setDryRun(value): - global dryRun - dryRun = True - return S_OK() - - -voName = None - - -def setVO(value): - global voName - voName = value - return S_OK() - - -Script.registerSwitch("V:", "vo=", "VO name", setVO) -Script.registerSwitch("D", "dryRun", "Dry run", setDryRun) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % Script.scriptName] - )) - -Script.parseCommandLine(ignoreErrors=True) - - -@executeWithUserProxy -def syncCSWithVOMS(vomsSync): - return vomsSync.syncCSWithVOMS() - - -def getVOAdmin(voName): - voAdminUser = getVOOption(voName, "VOAdmin") - voAdminGroup = getVOOption(voName, "VOAdminGroup", getVOOption(voName, "DefaultGroup")) - return voAdminUser, voAdminGroup - - -voAdminUser, voAdminGroup = getVOAdmin(voName) - -vomsSync = VOMS2CSSynchronizer(voName) -result = syncCSWithVOMS(vomsSync, # pylint: disable=unexpected-keyword-arg - proxyUserName=voAdminUser, - proxyUserGroup=voAdminGroup) -if not result['OK']: - gLogger.error("Failed to synchronize user data") - DIRACExit(-1) - -resultDict = result['Value'] -newUsers = resultDict.get("NewUsers", []) -modUsers = resultDict.get("ModifiedUsers", []) -delUsers = resultDict.get("DeletedUsers", []) -susUsers = resultDict.get("SuspendedUsers", []) -gLogger.notice("\nUser results: new %d, modified %d, deleted %d, new/suspended %d" % - (len(newUsers), len(modUsers), len(delUsers), len(susUsers))) - -for msg in resultDict["AdminMessages"]["Info"]: - gLogger.notice(msg) - -csapi = resultDict.get("CSAPI") -if csapi and csapi.csModified: - if dryRun: - gLogger.notice("There are changes to Registry ready to commit, skipped because of dry run") - else: - yn = raw_input("There are changes to Registry ready to commit, do you want to proceed ? [Y|n]:") - if yn == '' or yn[0].lower() == 'y': - result = csapi.commitChanges() - if not result['OK']: - gLogger.error("Could not commit configuration changes", result['Message']) - else: - gLogger.notice("Registry changes committed for VO %s" % voName) - else: - gLogger.notice("Registry changes are not committed") -else: - gLogger.notice("No changes to Registry for VO %s" % voName) - -result = vomsSync.getVOUserReport() -if not result['OK']: - gLogger.error('Failed to generate user data report') - DIRACExit(-1) - -gLogger.notice("\n" + result['Value']) diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-dump-local-cache.py b/src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-dump-local-cache.py deleted file mode 100755 index 75fde14489c..00000000000 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-dump-local-cache.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-configuration-cli -# Author : Adria Casajus -######################################################################## -""" - Dump DIRAC Configuration data -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import sys -import DIRAC -from DIRAC.Core.Base import Script - -Script.localCfg.addDefaultEntry("LogLevel", "fatal") - -fileName = "" - - -def setFilename(args): - global fileName - fileName = args - return DIRAC.S_OK() - - -raw = False - - -def setRaw(args): - global raw - raw = True - return DIRAC.S_OK() - - -Script.registerSwitch("f:", "file=", "Dump Configuration data into ", setFilename) -Script.registerSwitch("r", "raw", "Do not make any modification to the data", setRaw) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % Script.scriptName, ])) -Script.parseCommandLine() - -from DIRAC import gConfig, gLogger -result = gConfig.dumpCFGAsLocalCache(fileName, raw) -if not result['OK']: - print("Error: %s" % result['Message']) - sys.exit(1) - -if not fileName: - print(result['Value']) - -sys.exit(0) diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-resources.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py similarity index 98% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-resources.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py index b8282ff6931..5f7fe0107ac 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-resources.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py @@ -16,6 +16,7 @@ import shlex from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import gLogger, exit as DIRACExit from DIRAC.ConfigurationSystem.Client.Utilities import getGridCEs, getSiteUpdates from DIRAC.Core.Utilities.Subprocess import systemCall @@ -264,15 +265,12 @@ def handler(signum, frame): DIRACExit(-1) -if __name__ == "__main__": - +@DIRACScript() +def main(): signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGINT, handler) - vo = '' - dry = False - doCEs = False - ceBdiiDict = None + global vo, dry, doCEs, ceBdiiDict processScriptSwitches() @@ -280,7 +278,6 @@ def handler(signum, frame): gLogger.error('No VO specified') DIRACExit(-1) - diracVO = vo vo = getVOOption(vo, 'VOMSName', vo) if doCEs: @@ -293,3 +290,12 @@ def handler(signum, frame): yn = yn.strip() if yn == '' or yn.lower().startswith('y'): updateSites() + + +if __name__ == "__main__": + vo = '' + dry = False + doCEs = False + ceBdiiDict = None + + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-shifter.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_shifter.py similarity index 95% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-shifter.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_shifter.py index 161d2130bba..59fa8c6d731 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-shifter.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_shifter.py @@ -11,11 +11,13 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC import exit as DIRACExit, gLogger -if __name__ == "__main__": +@DIRACScript() +def main(): Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... ShifterRole UserName DIRACGroup ...' % Script.scriptName, @@ -44,3 +46,7 @@ gLogger.error("Could not add shifter", ": " + res['Message']) DIRACExit(1) gLogger.notice("Added shifter %s as user %s with group %s" % (shifterRole, userName, diracGroup)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-site.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py similarity index 97% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-site.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py index f3a4b94cf4f..d6e7ed85fc5 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-add-site.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py @@ -14,13 +14,14 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import exit as DIRACExit, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getDIRACSiteName from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI -if __name__ == "__main__": - +@DIRACScript() +def main(): Script.setUsageMessage( '\n'.join( [ @@ -99,3 +100,7 @@ if not res['OK']: gLogger.error("Failure committing to CS", res['Message']) DIRACExit(3) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-bdii-info.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_bdii_info.py similarity index 94% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-admin-bdii-info.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_admin_bdii_info.py index 2619ffd5e5c..cb65a3d167c 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-bdii-info.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_bdii_info.py @@ -14,6 +14,7 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup @@ -131,10 +132,8 @@ def showInfo(result, info): print("}") -# ............................................................................... - -if __name__ == "__main__": - +@DIRACScript() +def main(): # Script initialization registerSwitches() # registerUsageMessage() @@ -144,5 +143,6 @@ def showInfo(result, info): DIRAC.exit(0) -################################################################################ -# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-check-config-options.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py similarity index 98% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-admin-check-config-options.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py index 087fd876853..53b9f5fc011 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac-admin-check-config-options.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py @@ -22,6 +22,7 @@ from diraccfg import CFG from DIRAC import gLogger, S_ERROR, S_OK, gConfig from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Utilities.List import fromChar LOG = gLogger @@ -202,5 +203,10 @@ def run(self): return S_OK() -if __name__ == "__main__": +@DIRACScript() +def main(): CheckConfig().run() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py new file mode 100755 index 00000000000..113c766fc48 --- /dev/null +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +######################################################################## +# $HeadURL$ +# File : dirac-admin-sort-cs-sites +# Author : Matvey Sapunov +######################################################################## +""" + Sort site names at CS in "/Resources" section. Sort can be alphabetic or by country postfix in a site name. + Alphabetic sort is default (i.e. LCG.IHEP.cn, LCG.IHEP.su, LCG.IN2P3.fr) + + Options: + -C --country Sort site names by country postfix (i.e. LCG.IHEP.cn, LCG.IN2P3.fr, LCG.IHEP.su) + -R --reverse Reverse the sort order + + Argument: + Name of the subsection in the CS '/Resources/Sites/' section to be sorted (i.e. LCG, DIRAC) + + Example: dirac-admin-sort-cs-sites -C DIRAC + Sort sites in subsection /Resources/Sites/DIRAC by country postfix +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +from DIRAC import gLogger, exit as DIRACExit +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.Core.Security.ProxyInfo import getProxyInfo +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getPropertiesForGroup +from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI +from DIRAC.Core.Utilities.Time import dateTime, toString + +global SORTBYNAME, REVERSE +SORTBYNAME = True +REVERSE = False + + +def sortBy(arg): + global SORTBYNAME + SORTBYNAME = False + + +def isReverse(arg): + global REVERSE + REVERSE = True + + +def country(arg): + cb = arg.split(".") + if not len(cb) == 3: + gLogger.error("%s is not in GRID.NAME.COUNTRY format ") + return False + return cb[2] + + +@DIRACScript() +def main(): + Script.registerSwitch( + "C", + "country", + "Sort site names by country postfix (i.e. LCG.IHEP.cn, LCG.IN2P3.fr, LCG.IHEP.su)", + sortBy) + Script.registerSwitch("R", "reverse", "Reverse the sort order", isReverse) + + Script.setUsageMessage("\n".join([ + __doc__.split("\n")[1], + "Usage:", + " %s [option|cfgfile]
" % + Script.scriptName, + "Optional arguments:", + " Section: Name of the subsection in '/Resources/Sites/' for sort (i.e. LCG DIRAC)", + "Example:", + " dirac-admin-sort-cs-sites -C CLOUDS DIRAC", + " sort site names by country postfix in '/Resources/Sites/CLOUDS' and '/Resources/Sites/DIRAC' subsection", + "" + ])) + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + result = getProxyInfo() + if not result["OK"]: + gLogger.error("Failed to get proxy information", result["Message"]) + DIRACExit(2) + proxy = result["Value"] + if proxy["secondsLeft"] < 1: + gLogger.error("Your proxy has expired, please create new one") + DIRACExit(2) + group = proxy["group"] + if "CSAdministrator" not in getPropertiesForGroup(group): + gLogger.error("You must be CSAdministrator user to execute this script") + gLogger.notice("Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'") + DIRACExit(2) + + cs = CSAPI() + result = cs.getCurrentCFG() + if not result["OK"]: + gLogger.error("Failed to get copy of CS", result["Message"]) + DIRACExit(2) + cfg = result["Value"] + + if not cfg.isSection("Resources"): + gLogger.error("Section '/Resources' is absent in CS") + DIRACExit(2) + + if not cfg.isSection("Resources/Sites"): + gLogger.error("Subsection '/Resources/Sites' is absent in CS") + DIRACExit(2) + + if args and len(args) > 0: + resultList = args[:] + else: + resultList = cfg["Resources"]["Sites"].listSections() + + hasRun = False + isDirty = False + for i in resultList: + if not cfg.isSection("Resources/Sites/%s" % i): + gLogger.error("Subsection /Resources/Sites/%s does not exists" % i) + continue + hasRun = True + if SORTBYNAME: + dirty = cfg["Resources"]["Sites"][i].sortAlphabetically(ascending=not REVERSE) + else: + dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country, reverse=REVERSE) + if dirty: + isDirty = True + + if not hasRun: + gLogger.notice("Failed to find suitable subsections with site names to sort") + DIRACExit(0) + + if not isDirty: + gLogger.notice("Nothing to do, site names are already sorted") + DIRACExit(0) + + timestamp = toString(dateTime()) + stamp = "Site names are sorted by %s script at %s" % (Script.scriptName, timestamp) + cs.setOptionComment("/Resources/Sites", stamp) + + result = cs.commit() + if not result["OK"]: + gLogger.error("Failed to commit changes to CS", result["Message"]) + DIRACExit(2) + gLogger.notice("Site names are sorted and committed to CS") + DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py new file mode 100755 index 00000000000..c7b2aeeadf6 --- /dev/null +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-voms-sync +# Author : Andrei Tsaregorodtsev +######################################################################## +""" + Synchronize VOMS user data with the DIRAC Registry +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from DIRAC import gLogger, exit as DIRACExit, S_OK +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.ConfigurationSystem.Client.VOMS2CSSynchronizer import VOMS2CSSynchronizer +from DIRAC.Core.Utilities.Proxy import executeWithUserProxy +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOOption + + +dryRun = False +voName = None + + +def setDryRun(value): + global dryRun + dryRun = True + return S_OK() + + +def setVO(value): + global voName + voName = value + return S_OK() + + +@DIRACScript() +def main(): + Script.registerSwitch("V:", "vo=", "VO name", setVO) + Script.registerSwitch("D", "dryRun", "Dry run", setDryRun) + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % Script.scriptName + ])) + + Script.parseCommandLine(ignoreErrors=True) + + @executeWithUserProxy + def syncCSWithVOMS(vomsSync): + return vomsSync.syncCSWithVOMS() + + def getVOAdmin(voName): + voAdminUser = getVOOption(voName, "VOAdmin") + voAdminGroup = getVOOption(voName, "VOAdminGroup", getVOOption(voName, "DefaultGroup")) + return voAdminUser, voAdminGroup + + voAdminUser, voAdminGroup = getVOAdmin(voName) + + vomsSync = VOMS2CSSynchronizer(voName) + result = syncCSWithVOMS(vomsSync, # pylint: disable=unexpected-keyword-arg + proxyUserName=voAdminUser, + proxyUserGroup=voAdminGroup) + if not result['OK']: + gLogger.error("Failed to synchronize user data") + DIRACExit(-1) + + resultDict = result['Value'] + newUsers = resultDict.get("NewUsers", []) + modUsers = resultDict.get("ModifiedUsers", []) + delUsers = resultDict.get("DeletedUsers", []) + susUsers = resultDict.get("SuspendedUsers", []) + gLogger.notice("\nUser results: new %d, modified %d, deleted %d, new/suspended %d" % + (len(newUsers), len(modUsers), len(delUsers), len(susUsers))) + + for msg in resultDict["AdminMessages"]["Info"]: + gLogger.notice(msg) + + csapi = resultDict.get("CSAPI") + if csapi and csapi.csModified: + if dryRun: + gLogger.notice("There are changes to Registry ready to commit, skipped because of dry run") + else: + yn = raw_input("There are changes to Registry ready to commit, do you want to proceed ? [Y|n]:") + if yn == '' or yn[0].lower() == 'y': + result = csapi.commitChanges() + if not result['OK']: + gLogger.error("Could not commit configuration changes", result['Message']) + else: + gLogger.notice("Registry changes committed for VO %s" % voName) + else: + gLogger.notice("Registry changes are not committed") + else: + gLogger.notice("No changes to Registry for VO %s" % voName) + + result = vomsSync.getVOUserReport() + if not result['OK']: + gLogger.error('Failed to generate user data report') + DIRACExit(-1) + + gLogger.notice("\n" + result['Value']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-cli.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_cli.py similarity index 58% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-cli.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_cli.py index d26a8be2ae7..6641f2e44a2 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-cli.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_cli.py @@ -12,12 +12,22 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.ConfigurationSystem.Client.CSCLI import CSCLI -Script.localCfg.addDefaultEntry("LogLevel", "fatal") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % Script.scriptName, ])) -Script.parseCommandLine() -CSCLI().start() +@DIRACScript() +def main(): + Script.localCfg.addDefaultEntry("LogLevel", "fatal") + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % Script.scriptName, + ])) + Script.parseCommandLine() + + CSCLI().start() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py new file mode 100755 index 00000000000..33e6f80caf2 --- /dev/null +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-configuration-cli +# Author : Adria Casajus +######################################################################## +""" + Dump DIRAC Configuration data +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import sys +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.localCfg.addDefaultEntry("LogLevel", "fatal") + + fileName = "" + + def setFilename(args): + global fileName + fileName = args + return DIRAC.S_OK() + + raw = False + + def setRaw(args): + global raw + raw = True + return DIRAC.S_OK() + + Script.registerSwitch("f:", "file=", "Dump Configuration data into ", setFilename) + Script.registerSwitch("r", "raw", "Do not make any modification to the data", setRaw) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % Script.scriptName, ])) + Script.parseCommandLine() + + from DIRAC import gConfig, gLogger + result = gConfig.dumpCFGAsLocalCache(fileName, raw) + if not result['OK']: + print("Error: %s" % result['Message']) + sys.exit(1) + + if not fileName: + print(result['Value']) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-shell.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py similarity index 70% rename from src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-shell.py rename to src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py index 27b65491393..25223b11b4d 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac-configuration-shell.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_shell.py @@ -9,20 +9,20 @@ import sys from DIRAC.Core.Base import Script - -Script.parseCommandLine() - -from DIRAC.ConfigurationSystem.Client.CSShellCLI import CSShellCLI +from DIRAC.Core.Utilities.DIRACScript import DIRACScript # Invariants: # * root does not end with "/" or root is "/" # * root starts with "/" +@DIRACScript() def main(): + Script.parseCommandLine() + from DIRAC.ConfigurationSystem.Client.CSShellCLI import CSShellCLI shell = CSShellCLI() shell.cmdloop() if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/src/DIRAC/Core/Tornado/scripts/tornado-start-CS.py b/src/DIRAC/Core/Tornado/scripts/tornado-start-CS.py deleted file mode 100644 index fb4e1811e26..00000000000 --- a/src/DIRAC/Core/Tornado/scripts/tornado-start-CS.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : tornado-start-CS -# Author : Louis MARTIN -######################################################################## -# Just run this script to start Tornado and CS service -# Use dirac.cfg (or other cfg given in the command line) to change port - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -# Must be define BEFORE any dirac import -import os -import sys -os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" - -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration -from DIRAC.ConfigurationSystem.private.Refresher import gRefresher -from DIRAC.Core.Utilities.DErrno import includeExtensionErrors -from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer -from DIRAC.FrameworkSystem.Client.Logger import gLogger - -if gConfigurationData.isMaster(): - gRefresher.disable() - -localCfg = LocalConfiguration() -localCfg.addMandatoryEntry("/DIRAC/Setup") -localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") -localCfg.addDefaultEntry("LogLevel", "INFO") -localCfg.addDefaultEntry("LogColor", True) -resultDict = localCfg.loadUserData() -if not resultDict['OK']: - gLogger.initialize("Tornado-CS", "/") - gLogger.error("There were errors when loading configuration", resultDict['Message']) - sys.exit(1) - -includeExtensionErrors() - - -gLogger.initialize('Tornado-CS', "/") - - -serverToLaunch = TornadoServer(services='Configuration/Server') -serverToLaunch.startTornado() diff --git a/src/DIRAC/Core/Tornado/scripts/tornado-start-all.py b/src/DIRAC/Core/Tornado/scripts/tornado-start-all.py deleted file mode 100644 index 729165e08bf..00000000000 --- a/src/DIRAC/Core/Tornado/scripts/tornado-start-all.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : tornado-start-all -# Author : Louis MARTIN -######################################################################## -# Just run this script to start Tornado and all services -# Use CS to change port - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -# Must be define BEFORE any dirac import -import os -import sys -os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" - - -from DIRAC import gConfig -from DIRAC.ConfigurationSystem.Client import PathFinder -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration -from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer -from DIRAC.Core.Utilities.DErrno import includeExtensionErrors -from DIRAC.FrameworkSystem.Client.Logger import gLogger - - -# We check if there is no configuration server started as master -# If you want to start a master CS you should use Configuration_Server.cfg and -# use tornado-start-CS.py -if gConfigurationData.isMaster() and gConfig.getValue( - '/Systems/Configuration/%s/Services/Server/Protocol' % - PathFinder.getSystemInstance('Configuration'), - 'dips').lower() == 'https': - gLogger.fatal("You can't run the CS and services in the same server!") - sys.exit(0) - -localCfg = LocalConfiguration() -localCfg.addMandatoryEntry("/DIRAC/Setup") -localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") -localCfg.addDefaultEntry("LogLevel", "INFO") -localCfg.addDefaultEntry("LogColor", True) -resultDict = localCfg.loadUserData() -if not resultDict['OK']: - gLogger.initialize("Tornado", "/") - gLogger.error("There were errors when loading configuration", resultDict['Message']) - sys.exit(1) - -includeExtensionErrors() - - -gLogger.initialize('Tornado', "/") - - -serverToLaunch = TornadoServer() -serverToLaunch.startTornado() diff --git a/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py new file mode 100644 index 00000000000..1e6d5fa7148 --- /dev/null +++ b/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +######################################################################## +# File : tornado-start-CS +# Author : Louis MARTIN +######################################################################## +# Just run this script to start Tornado and CS service +# Use dirac.cfg (or other cfg given in the command line) to change port + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import os +import sys +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + # Must be defined BEFORE any dirac import + os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" + + from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration + from DIRAC.ConfigurationSystem.private.Refresher import gRefresher + from DIRAC.Core.Utilities.DErrno import includeExtensionErrors + from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer + from DIRAC.FrameworkSystem.Client.Logger import gLogger + + if gConfigurationData.isMaster(): + gRefresher.disable() + + localCfg = LocalConfiguration() + localCfg.addMandatoryEntry("/DIRAC/Setup") + localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") + localCfg.addDefaultEntry("LogLevel", "INFO") + localCfg.addDefaultEntry("LogColor", True) + resultDict = localCfg.loadUserData() + if not resultDict['OK']: + gLogger.initialize("Tornado-CS", "/") + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + + includeExtensionErrors() + + gLogger.initialize('Tornado-CS', "/") + + serverToLaunch = TornadoServer(services='Configuration/Server') + serverToLaunch.startTornado() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py new file mode 100644 index 00000000000..56061542bc9 --- /dev/null +++ b/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +######################################################################## +# File : tornado-start-all +# Author : Louis MARTIN +######################################################################## +# Just run this script to start Tornado and all services +# Use CS to change port + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +# Must be define BEFORE any dirac import +import os +import sys +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" + + from DIRAC import gConfig + from DIRAC.ConfigurationSystem.Client import PathFinder + from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration + from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer + from DIRAC.Core.Utilities.DErrno import includeExtensionErrors + from DIRAC.FrameworkSystem.Client.Logger import gLogger + + # We check if there is no configuration server started as master + # If you want to start a master CS you should use Configuration_Server.cfg and + # use tornado-start-CS.py + key = '/Systems/Configuration/%s/Services/Server/Protocol' % PathFinder.getSystemInstance('Configuration') + if gConfigurationData.isMaster() and gConfig.getValue(key, 'dips').lower() == 'https': + gLogger.fatal("You can't run the CS and services in the same server!") + sys.exit(0) + + localCfg = LocalConfiguration() + localCfg.addMandatoryEntry("/DIRAC/Setup") + localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") + localCfg.addDefaultEntry("LogLevel", "INFO") + localCfg.addDefaultEntry("LogColor", True) + resultDict = localCfg.loadUserData() + if not resultDict['OK']: + gLogger.initialize("Tornado", "/") + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + + includeExtensionErrors() + + gLogger.initialize('Tornado', "/") + + serverToLaunch = TornadoServer() + serverToLaunch.startTornado() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/Utilities/DIRACScript.py b/src/DIRAC/Core/Utilities/DIRACScript.py new file mode 100644 index 00000000000..87c7618bcd0 --- /dev/null +++ b/src/DIRAC/Core/Utilities/DIRACScript.py @@ -0,0 +1,108 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from collections import defaultdict +import functools + +import six + + +class DIRACScript(object): + """Decorator for providing command line executables + + All console-scripts entrypoints in DIRAC and downstream extensions should be + wrapped in this decorator to allow extensions to override any entry_point. + """ + def __init__(self): + """ c'tor + """ + pass + + def __call__(self, func=None): + """Set the wrapped function or call the script + + This function is either called with a decorator or directly to call the + underlying function. When running with Python 2 the raw function will always + be called however in Python 3 the priorities will be applied from the + dirac.extension_metadata entry_point. + """ + # If func is provided then the decorator is being applied to a function + if func is not None: + self._func = func + return functools.wraps(func)(self) + + # Setuptools based installations aren't supported with Python 2 + if six.PY2: + return self._func() # pylint: disable=not-callable + + # This is only available in Python 3.8+ so it has to be here for now + from importlib import metadata # pylint: disable=no-name-in-module + + # Iterate through all known entry_points looking for DIRACScripts + matches = defaultdict(list) + function_name = None + for entrypoint in metadata.entry_points()['console_scripts']: + if not entrypoint.name.startswith("dirac-"): + continue + entrypointFunc = entrypoint.load() + if not isinstance(entrypointFunc, DIRACScript): + raise ImportError( + "Invalid dirac- console_scripts entry_point: " + repr(entrypoint) + "\n" + + "All dirac- console_scripts should be wrapped in the DiracScript " + + "decorator to ensure extension overlays are applied correctly." + ) + matches[entrypoint.name].append(entrypoint) + # If the function is self then we've found the currently called function + if entrypointFunc is self: + function_name = entrypoint.name + + if function_name is None: + # TODO: This should an error once the integration tests modified to use pip install + return self._func() # pylint: disable=not-callable + # raise NotImplementedError("Something is very wrong") + + # Call the entry_point from the extension with the highest priority + rankedExtensions = _extensionsByPriority() + entrypoint = max( + matches[function_name], + key=lambda e: rankedExtensions.index(_entrypointToExtension(e)), + ) + + return entrypoint.load()._func() + + +def _entrypointToExtension(entrypoint): + """"Get the extension name from an EntryPoint object""" + # In Python 3.9 this can be "entrypoint.module" + module = entrypoint.pattern.match(entrypoint.value).groupdict()["module"] + extensionName = module.split(".")[0] + return extensionName + + +def _extensionsByPriority(): + """Discover extensions using the setuptools metadata + + TODO: This should move into a function which can also be called to fill the CS + """ + # This is only available in Python 3.8+ so it has to be here for now + from importlib import metadata # pylint: disable=no-name-in-module + + priorties = defaultdict(list) + for entrypoint in metadata.entry_points()['dirac']: + extensionName = _entrypointToExtension(entrypoint) + extension_metadata = entrypoint.load()() + priorties[extension_metadata["priority"]].append(extensionName) + + extensions = [] + for priority, extensionNames in sorted(priorties.items()): + if len(extensionNames) != 1: + print( + "WARNING: Found multiple extensions with priority", + "{} ({})".format(priority, extensionNames), + ) + # If multiple are passed, sort the extensions so things are deterministic at least + extensions.extend(sorted(extensionNames)) + return extensions diff --git a/src/DIRAC/Core/scripts/__init__.py b/src/DIRAC/Core/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/Core/scripts/dirac-agent.py b/src/DIRAC/Core/scripts/dirac-agent.py index 6bd5b3e43a2..d5817ec0b61 100755 --- a/src/DIRAC/Core/scripts/dirac-agent.py +++ b/src/DIRAC/Core/scripts/dirac-agent.py @@ -17,6 +17,9 @@ from DIRAC.Core.Base.AgentReactor import AgentReactor from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +print("NOTE:", __file__, "is deprecated and will be removed in v7r3, for details see", + "https://github.com/DIRACGrid/DIRAC/wiki/DIRAC-v7r2#rename-of-scripts") + localCfg = LocalConfiguration() positionalArgs = localCfg.getPositionalArguments() diff --git a/src/DIRAC/Core/scripts/dirac-configure.py b/src/DIRAC/Core/scripts/dirac-configure.py deleted file mode 100755 index 6127d6c066d..00000000000 --- a/src/DIRAC/Core/scripts/dirac-configure.py +++ /dev/null @@ -1,562 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-configure -# Author : Ricardo Graciani -######################################################################## -""" - Main script to write dirac.cfg for a new DIRAC installation and initial download of CAs and CRLs - if necessary. - - To be used by VO specific scripts to configure new DIRAC installations - - There are 2 mandatory arguments: - - -S --Setup= To define the DIRAC setup for the current installation - -C --ConfigurationServer=|-W --Gateway To define the reference Configuration Servers/Gateway - for the current installation - - others are optional - - -I --IncludeAllServers To include all Configuration Servers - (by default only those in -C option are included) - -n --SiteName= To define the DIRAC Site Name for the installation - -N --CEName= To determine the DIRAC Site Name from the CE Name - -V --VO= To define the VO for the installation - -U --UseServerCertificate To use Server Certificate for all clients - -H --SkipCAChecks To skip check of CAs for all clients - -D --SkipCADownload To skip download of CAs - -M --SkipVOMSDownload To skip download of VOMS info - -v --UseVersionsDir Use versions directory - (This option will properly define RootPath and InstancePath) - -A --Architecture= To define /LocalSite/Architecture= - -L --LocalSE= To define /LocalSite/LocalSE= - -F --ForceUpdate Forces the update of cfg file (i.e. dirac.cfg), - even if it does already exists (use with care) - -O --Output define output configuration file - - Other arguments will take proper defaults if not defined. - - Additionally all options can all be passed inside a .cfg file passed as argument. - The following options are recognized:: - - Setup - ConfigurationServer - IncludeAllServers - Gateway - SiteName - CEName - VirtualOrganization - UseServerCertificate - SkipCAChecks - SkipCADownload - UseVersionsDir - Architecture - LocalSE - LogLevel - - As in any other script command line option take precedence over .cfg files passed as arguments. - The combination of both is written into the installed dirac.cfg. - - Notice: It will not overwrite exiting info in current dirac.cfg if it exists. - - Example: dirac-configure -d - -S LHCb-Development - -C 'dips://lhcbprod.pic.es:9135/Configuration/Server' - -W 'dips://lhcbprod.pic.es:9135' - --SkipCAChecks - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import sys -import os - -import DIRAC -from DIRAC.Core.Utilities.File import mkDir -from DIRAC.Core.Base import Script -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.ConfigurationSystem.Client.Helpers import cfgInstallPath, cfgPath, Registry -from DIRAC.Core.Utilities.SiteSEMapping import getSEsForSite -from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient - - -__RCSID__ = "$Id$" - - -logLevel = None -setup = None -configurationServer = None -includeAllServers = False -gatewayServer = None -siteName = None -useServerCert = False -skipCAChecks = False -skipCADownload = False -useVersionsDir = False -architecture = None -localSE = None -ceName = None -vo = None -update = False -outputFile = '' -skipVOMSDownload = False -extensions = '' - - -def setGateway(optionValue): - global gatewayServer - gatewayServer = optionValue - setServer(gatewayServer + '/Configuration/Server') - DIRAC.gConfig.setOptionValue(cfgInstallPath('Gateway'), gatewayServer) - return DIRAC.S_OK() - - -def setOutput(optionValue): - global outputFile - outputFile = optionValue - return DIRAC.S_OK() - - -def setServer(optionValue): - global configurationServer - configurationServer = optionValue - DIRAC.gConfig.setOptionValue('/DIRAC/Configuration/Servers', configurationServer) - DIRAC.gConfig.setOptionValue(cfgInstallPath('ConfigurationServer'), configurationServer) - return DIRAC.S_OK() - - -def setAllServers(optionValue): - global includeAllServers - includeAllServers = True - - -def setSetup(optionValue): - global setup - setup = optionValue - DIRAC.gConfig.setOptionValue('/DIRAC/Setup', setup) - DIRAC.gConfig.setOptionValue(cfgInstallPath('Setup'), setup) - return DIRAC.S_OK() - - -def setSiteName(optionValue): - global siteName - siteName = optionValue - Script.localCfg.addDefaultEntry('/LocalSite/Site', siteName) - DIRAC.__siteName = False - DIRAC.gConfig.setOptionValue(cfgInstallPath('SiteName'), siteName) - return DIRAC.S_OK() - - -def setCEName(optionValue): - global ceName - ceName = optionValue - DIRAC.gConfig.setOptionValue(cfgInstallPath('CEName'), ceName) - return DIRAC.S_OK() - - -def setServerCert(optionValue): - global useServerCert - useServerCert = True - DIRAC.gConfig.setOptionValue(cfgInstallPath('UseServerCertificate'), useServerCert) - return DIRAC.S_OK() - - -def setSkipCAChecks(optionValue): - global skipCAChecks - skipCAChecks = True - DIRAC.gConfig.setOptionValue(cfgInstallPath('SkipCAChecks'), skipCAChecks) - return DIRAC.S_OK() - - -def setSkipCADownload(optionValue): - global skipCADownload - skipCADownload = True - DIRAC.gConfig.setOptionValue(cfgInstallPath('SkipCADownload'), skipCADownload) - return DIRAC.S_OK() - - -def setSkipVOMSDownload(optionValue): - global skipVOMSDownload - skipVOMSDownload = True - DIRAC.gConfig.setOptionValue(cfgInstallPath('SkipVOMSDownload'), skipVOMSDownload) - return DIRAC.S_OK() - - -def setUseVersionsDir(optionValue): - global useVersionsDir - useVersionsDir = True - DIRAC.gConfig.setOptionValue(cfgInstallPath('UseVersionsDir'), useVersionsDir) - return DIRAC.S_OK() - - -def setArchitecture(optionValue): - global architecture - architecture = optionValue - Script.localCfg.addDefaultEntry('/LocalSite/Architecture', architecture) - DIRAC.gConfig.setOptionValue(cfgInstallPath('Architecture'), architecture) - return DIRAC.S_OK() - - -def setLocalSE(optionValue): - global localSE - localSE = optionValue - Script.localCfg.addDefaultEntry('/LocalSite/LocalSE', localSE) - DIRAC.gConfig.setOptionValue(cfgInstallPath('LocalSE'), localSE) - return DIRAC.S_OK() - - -def setVO(optionValue): - global vo - vo = optionValue - Script.localCfg.addDefaultEntry('/DIRAC/VirtualOrganization', vo) - DIRAC.gConfig.setOptionValue(cfgInstallPath('VirtualOrganization'), vo) - return DIRAC.S_OK() - - -def forceUpdate(optionValue): - global update - update = True - return DIRAC.S_OK() - - -def setExtensions(optionValue): - global extensions - extensions = optionValue - DIRAC.gConfig.setOptionValue('/DIRAC/Extensions', extensions) - DIRAC.gConfig.setOptionValue(cfgInstallPath('Extensions'), extensions) - return DIRAC.S_OK() - - -Script.disableCS() - -Script.registerSwitch("S:", "Setup=", "Set as DIRAC setup", setSetup) -Script.registerSwitch("e:", "Extensions=", "Set as DIRAC extensions", setExtensions) -Script.registerSwitch("C:", "ConfigurationServer=", "Set as DIRAC configuration server", setServer) -Script.registerSwitch("I", "IncludeAllServers", "include all Configuration Servers", setAllServers) -Script.registerSwitch("n:", "SiteName=", "Set as DIRAC Site Name", setSiteName) -Script.registerSwitch("N:", "CEName=", "Determiner from ", setCEName) -Script.registerSwitch("V:", "VO=", "Set the VO name", setVO) - -Script.registerSwitch("W:", "gateway=", "Configure as DIRAC Gateway for the site", setGateway) - -Script.registerSwitch("U", "UseServerCertificate", "Configure to use Server Certificate", setServerCert) -Script.registerSwitch("H", "SkipCAChecks", "Configure to skip check of CAs", setSkipCAChecks) -Script.registerSwitch("D", "SkipCADownload", "Configure to skip download of CAs", setSkipCADownload) -Script.registerSwitch("M", "SkipVOMSDownload", "Configure to skip download of VOMS info", setSkipVOMSDownload) - -Script.registerSwitch("v", "UseVersionsDir", "Use versions directory", setUseVersionsDir) - -Script.registerSwitch("A:", "Architecture=", "Configure /Architecture=", setArchitecture) -Script.registerSwitch("L:", "LocalSE=", "Configure LocalSite/LocalSE=", setLocalSE) - -Script.registerSwitch( - "F", - "ForceUpdate", - "Force Update of cfg file (i.e. dirac.cfg) (otherwise nothing happens if dirac.cfg already exists)", - forceUpdate) - -Script.registerSwitch("O:", "output=", "output configuration file", setOutput) - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - '\nUsage:', - ' %s [option|cfgfile] ...\n' % Script.scriptName])) - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getExtraCLICFGFiles() - -if not logLevel: - logLevel = DIRAC.gConfig.getValue(cfgInstallPath('LogLevel'), '') - if logLevel: - DIRAC.gLogger.setLevel(logLevel) -else: - DIRAC.gConfig.setOptionValue(cfgInstallPath('LogLevel'), logLevel) - -if not gatewayServer: - newGatewayServer = DIRAC.gConfig.getValue(cfgInstallPath('Gateway'), '') - if newGatewayServer: - setGateway(newGatewayServer) - -if not configurationServer: - newConfigurationServer = DIRAC.gConfig.getValue(cfgInstallPath('ConfigurationServer'), '') - if newConfigurationServer: - setServer(newConfigurationServer) - -if not includeAllServers: - newIncludeAllServer = DIRAC.gConfig.getValue(cfgInstallPath('IncludeAllServers'), False) - if newIncludeAllServer: - setAllServers(True) - -if not setup: - newSetup = DIRAC.gConfig.getValue(cfgInstallPath('Setup'), '') - if newSetup: - setSetup(newSetup) - -if not siteName: - newSiteName = DIRAC.gConfig.getValue(cfgInstallPath('SiteName'), '') - if newSiteName: - setSiteName(newSiteName) - -if not ceName: - newCEName = DIRAC.gConfig.getValue(cfgInstallPath('CEName'), '') - if newCEName: - setCEName(newCEName) - -if not useServerCert: - newUserServerCert = DIRAC.gConfig.getValue(cfgInstallPath('UseServerCertificate'), False) - if newUserServerCert: - setServerCert(newUserServerCert) - -if not skipCAChecks: - newSkipCAChecks = DIRAC.gConfig.getValue(cfgInstallPath('SkipCAChecks'), False) - if newSkipCAChecks: - setSkipCAChecks(newSkipCAChecks) - -if not skipCADownload: - newSkipCADownload = DIRAC.gConfig.getValue(cfgInstallPath('SkipCADownload'), False) - if newSkipCADownload: - setSkipCADownload(newSkipCADownload) - -if not useVersionsDir: - newUseVersionsDir = DIRAC.gConfig.getValue(cfgInstallPath('UseVersionsDir'), False) - if newUseVersionsDir: - setUseVersionsDir(newUseVersionsDir) - # Set proper Defaults in configuration (even if they will be properly overwrite by gComponentInstaller - instancePath = os.path.dirname(os.path.dirname(DIRAC.rootPath)) - rootPath = os.path.join(instancePath, 'pro') - DIRAC.gConfig.setOptionValue(cfgInstallPath('InstancePath'), instancePath) - DIRAC.gConfig.setOptionValue(cfgInstallPath('RootPath'), rootPath) - -if not architecture: - newArchitecture = DIRAC.gConfig.getValue(cfgInstallPath('Architecture'), '') - if newArchitecture: - setArchitecture(newArchitecture) - -if not vo: - newVO = DIRAC.gConfig.getValue(cfgInstallPath('VirtualOrganization'), '') - if newVO: - setVO(newVO) - -if not extensions: - newExtensions = DIRAC.gConfig.getValue(cfgInstallPath('Extensions'), '') - if newExtensions: - setExtensions(newExtensions) - -DIRAC.gLogger.notice('Executing: %s ' % (' '.join(sys.argv))) -DIRAC.gLogger.notice('Checking DIRAC installation at "%s"' % DIRAC.rootPath) - -if update: - if outputFile: - DIRAC.gLogger.notice('Will update the output file %s' % outputFile) - else: - DIRAC.gLogger.notice('Will update %s' % DIRAC.gConfig.diracConfigFilePath) - -if setup: - DIRAC.gLogger.verbose('/DIRAC/Setup =', setup) -if vo: - DIRAC.gLogger.verbose('/DIRAC/VirtualOrganization =', vo) -if configurationServer: - DIRAC.gLogger.verbose('/DIRAC/Configuration/Servers =', configurationServer) - -if siteName: - DIRAC.gLogger.verbose('/LocalSite/Site =', siteName) -if architecture: - DIRAC.gLogger.verbose('/LocalSite/Architecture =', architecture) -if localSE: - DIRAC.gLogger.verbose('/LocalSite/localSE =', localSE) - -if not useServerCert: - DIRAC.gLogger.verbose('/DIRAC/Security/UseServerCertificate =', 'no') - # Being sure it was not there before - Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') - Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'no') -else: - DIRAC.gLogger.verbose('/DIRAC/Security/UseServerCertificate =', 'yes') - # Being sure it was not there before - Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') - Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'yes') - -host = DIRAC.gConfig.getValue(cfgInstallPath("Host"), "") -if host: - DIRAC.gConfig.setOptionValue(cfgPath("DIRAC", "Hostname"), host) - -if skipCAChecks: - DIRAC.gLogger.verbose('/DIRAC/Security/SkipCAChecks =', 'yes') - # Being sure it was not there before - Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') - Script.localCfg.addDefaultEntry('/DIRAC/Security/SkipCAChecks', 'yes') -else: - # Necessary to allow initial download of CA's - if not skipCADownload: - DIRAC.gConfig.setOptionValue('/DIRAC/Security/SkipCAChecks', 'yes') -if not skipCADownload: - Script.enableCS() - try: - dirName = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'certificates') - mkDir(dirName) - except BaseException: - DIRAC.gLogger.exception() - DIRAC.gLogger.fatal('Fail to create directory:', dirName) - DIRAC.exit(-1) - try: - bdc = BundleDeliveryClient() - result = bdc.syncCAs() - if result['OK']: - result = bdc.syncCRLs() - except Exception as e: - DIRAC.gLogger.error('Failed to sync CAs and CRLs: %s' % str(e)) - - if not skipCAChecks: - Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') - -if ceName or siteName: - # This is used in the pilot context, we should have a proxy, or a certificate, and access to CS - if useServerCert: - # Being sure it was not there before - Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') - Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'yes') - Script.enableCS() - # Get the site resource section - gridSections = DIRAC.gConfig.getSections('/Resources/Sites/') - if not gridSections['OK']: - DIRAC.gLogger.warn('Could not get grid sections list') - grids = [] - else: - grids = gridSections['Value'] - # try to get siteName from ceName or Local SE from siteName using Remote Configuration - for grid in grids: - siteSections = DIRAC.gConfig.getSections('/Resources/Sites/%s/' % grid) - if not siteSections['OK']: - DIRAC.gLogger.warn('Could not get %s site list' % grid) - sites = [] - else: - sites = siteSections['Value'] - - if not siteName: - if ceName: - for site in sites: - res = DIRAC.gConfig.getSections('/Resources/Sites/%s/%s/CEs/' % (grid, site), []) - if not res['OK']: - DIRAC.gLogger.warn('Could not get %s CEs list' % site) - if ceName in res['Value']: - siteName = site - break - if siteName: - DIRAC.gLogger.notice('Setting /LocalSite/Site = %s' % siteName) - Script.localCfg.addDefaultEntry('/LocalSite/Site', siteName) - DIRAC.__siteName = False - if ceName: - DIRAC.gLogger.notice('Setting /LocalSite/GridCE = %s' % ceName) - Script.localCfg.addDefaultEntry('/LocalSite/GridCE', ceName) - - if not localSE and siteName in sites: - localSE = getSEsForSite(siteName) - if localSE['OK'] and localSE['Value']: - localSE = ','.join(localSE['Value']) - DIRAC.gLogger.notice('Setting /LocalSite/LocalSE =', localSE) - Script.localCfg.addDefaultEntry('/LocalSite/LocalSE', localSE) - break - -if gatewayServer: - DIRAC.gLogger.verbose('/DIRAC/Gateways/%s =' % DIRAC.siteName(), gatewayServer) - Script.localCfg.addDefaultEntry('/DIRAC/Gateways/%s' % DIRAC.siteName(), gatewayServer) - -# Create the local cfg if it is not yet there -if not outputFile: - outputFile = DIRAC.gConfig.diracConfigFilePath -outputFile = os.path.abspath(outputFile) -if not os.path.exists(outputFile): - configDir = os.path.dirname(outputFile) - mkDir(configDir) - update = True - DIRAC.gConfig.dumpLocalCFGToFile(outputFile) - -if includeAllServers: - # We need user proxy or server certificate to continue in order to get all the CS URLs - if not useServerCert: - Script.enableCS() - result = getProxyInfo() - if not result['OK']: - DIRAC.gLogger.notice('Configuration is not completed because no user proxy is available') - DIRAC.gLogger.notice('Create one using dirac-proxy-init and execute again with -F option') - sys.exit(1) - else: - Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') - # When using Server Certs CA's will be checked, the flag only disables initial download - # this will be replaced by the use of SkipCADownload - Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'yes') - Script.enableCS() - - DIRAC.gConfig.setOptionValue('/DIRAC/Configuration/Servers', ','.join(DIRAC.gConfig.getServersList())) - DIRAC.gLogger.verbose('/DIRAC/Configuration/Servers =', ','.join(DIRAC.gConfig.getServersList())) - -if useServerCert: - # always removing before dumping - Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') - Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') - Script.localCfg.deleteOption('/DIRAC/Security/SkipVOMSDownload') - -if update: - DIRAC.gConfig.dumpLocalCFGToFile(outputFile) - - -# ## LAST PART: do the vomsdir/vomses magic - -# This has to be done for all VOs in the installation - -if skipVOMSDownload: - # We stop here - sys.exit(0) - - -result = Registry.getVOMSServerInfo() -if not result['OK']: - sys.exit(1) - -error = '' -vomsDict = result['Value'] -for vo in vomsDict: - voName = vomsDict[vo]['VOMSName'] - vomsDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'vomsdir', voName) - vomsesDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'vomses') - for path in (vomsDirPath, vomsesDirPath): - mkDir(path) - vomsesLines = [] - for vomsHost in vomsDict[vo].get('Servers', {}): - hostFilePath = os.path.join(vomsDirPath, "%s.lsc" % vomsHost) - try: - DN = vomsDict[vo]['Servers'][vomsHost]['DN'] - CA = vomsDict[vo]['Servers'][vomsHost]['CA'] - port = vomsDict[vo]['Servers'][vomsHost]['Port'] - if not DN or not CA or not port: - DIRAC.gLogger.error('DN = %s' % DN) - DIRAC.gLogger.error('CA = %s' % CA) - DIRAC.gLogger.error('Port = %s' % port) - DIRAC.gLogger.error('Missing Parameter for %s' % vomsHost) - continue - with open(hostFilePath, "wt") as fd: - fd.write("%s\n%s\n" % (DN, CA)) - vomsesLines.append('"%s" "%s" "%s" "%s" "%s" "24"' % (voName, vomsHost, port, DN, voName)) - DIRAC.gLogger.notice("Created vomsdir file %s" % hostFilePath) - except Exception: - DIRAC.gLogger.exception("Could not generate vomsdir file for host", vomsHost) - error = "Could not generate vomsdir file for VO %s, host %s" % (voName, vomsHost) - try: - vomsesFilePath = os.path.join(vomsesDirPath, voName) - with open(vomsesFilePath, "wt") as fd: - fd.write("%s\n" % "\n".join(vomsesLines)) - DIRAC.gLogger.notice("Created vomses file %s" % vomsesFilePath) - except Exception: - DIRAC.gLogger.exception("Could not generate vomses file") - error = "Could not generate vomses file for VO %s" % voName - -if useServerCert: - Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') - # When using Server Certs CA's will be checked, the flag only disables initial download - # this will be replaced by the use of SkipCADownload - Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') - -if error: - sys.exit(1) - -sys.exit(0) diff --git a/src/DIRAC/Core/scripts/dirac-executor.py b/src/DIRAC/Core/scripts/dirac-executor.py index 76b578c5017..bf03863132f 100755 --- a/src/DIRAC/Core/scripts/dirac-executor.py +++ b/src/DIRAC/Core/scripts/dirac-executor.py @@ -17,6 +17,9 @@ from DIRAC.Core.Base.ExecutorReactor import ExecutorReactor from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +print("NOTE:", __file__, "is deprecated and will be removed in v7r3, for details see", + "https://github.com/DIRACGrid/DIRAC/wiki/DIRAC-v7r2#rename-of-scripts") + localCfg = LocalConfiguration() positionalArgs = localCfg.getPositionalArguments() diff --git a/src/DIRAC/Core/scripts/dirac-externals-requirements.py b/src/DIRAC/Core/scripts/dirac-externals-requirements.py deleted file mode 100755 index 71753e17d7e..00000000000 --- a/src/DIRAC/Core/scripts/dirac-externals-requirements.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-externals-requirements -# Author : Adri/Federico/Andrei -######################################################################## -""" If RequiredExternals section is found in releases.cfg of any extension, - then some python packages to install with pip may be found. This script - will install the requested modules. - - The command is called from the dirac-install general installation command. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import sys -# TODO: This should be modernised to use subprocess(32) -try: - import commands -except ImportError: - # Python 3's subprocess module contains a compatibility layer - import subprocess as commands - -from diraccfg import CFG -from DIRAC.Core.Base import Script -Script.disableCS() - -from DIRAC import gLogger, rootPath, S_OK - -__RCSID__ = "$Id$" - -# Default installation type -instType = "server" - - -def setInstallType(val): - global instType - instType = val - return S_OK() - - -Script.registerSwitch("t:", "type=", "Installation type. 'server' by default.", setInstallType) -Script.parseCommandLine(ignoreErrors=True) - - -def pipInstall(package, switches=""): - # The right pip should be in the PATH, which is the case after sourcing the DIRAC bashrc - cmd = "pip install --trusted-host pypi.python.org %s %s" % (switches, package) - gLogger.notice("Executing %s" % cmd) - return commands.getstatusoutput(cmd) - - -# Collect all the requested python modules to install -reqDict = {} - - -for entry in os.listdir(rootPath): - if len(entry) < 5 or entry.find("DIRAC") != len(entry) - 5: - continue - reqFile = os.path.join(rootPath, entry, "releases.cfg") - try: - with open(reqFile, "r") as extfd: - reqCFG = CFG().loadFromBuffer(extfd.read()) - except BaseException: - gLogger.verbose("%s not found" % reqFile) - continue - reqList = reqCFG.getOption("/RequiredExternals/%s" % instType.capitalize(), []) - if not reqList: - gLogger.verbose("%s does not have requirements for %s installation" % (entry, instType)) - continue - for req in reqList: - reqName = False - reqCond = "" - for cond in ("==", ">="): - iP = cond.find(req) - if iP > 0: - reqName = req[:iP] - reqCond = req[iP:] - break - if not reqName: - reqName = req - if reqName not in reqDict: - reqDict[reqName] = (reqCond, entry) - else: - gLogger.notice("Skipping %s, it's already requested by %s" % (reqName, reqDict[reqName][1])) - -if not reqDict: - gLogger.notice("No extra python module requested to be installed") - sys.exit(0) - -for reqName in reqDict: - package = "%s%s" % (reqName, reqDict[reqName][0]) - gLogger.notice("Requesting installation of %s" % package) - status, output = pipInstall(package) - if status != 0: - gLogger.error(output) - else: - gLogger.notice("Successfully installed %s" % package) diff --git a/src/DIRAC/Core/scripts/dirac-info.py b/src/DIRAC/Core/scripts/dirac-info.py deleted file mode 100755 index 94d36a82b74..00000000000 --- a/src/DIRAC/Core/scripts/dirac-info.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-info -# Author : Andrei Tsaregorodtsev -######################################################################## -""" - Report info about local DIRAC installation -""" - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -import os - -import DIRAC -from DIRAC import gConfig -from DIRAC.Core.Base import Script -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup -from DIRAC.Core.Utilities.PrettyPrint import printTable - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Site' % Script.scriptName, ])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -records = [] - -records.append(('Setup', gConfig.getValue('/DIRAC/Setup', 'Unknown'))) -records.append(('ConfigurationServer', gConfig.getValue('/DIRAC/Configuration/Servers', []))) -records.append(('Installation path', DIRAC.rootPath)) - -if os.path.exists(os.path.join(DIRAC.rootPath, DIRAC.getPlatform(), 'bin', 'mysql')): - records.append(('Installation type', 'server')) -else: - records.append(('Installation type', 'client')) - -records.append(('Platform', DIRAC.getPlatform())) - -ret = getProxyInfo(disableVOMS=True) -if ret['OK']: - if 'group' in ret['Value']: - vo = getVOForGroup(ret['Value']['group']) - else: - vo = getVOForGroup('') - if not vo: - vo = "None" - records.append(('VirtualOrganization', vo)) - if 'identity' in ret['Value']: - records.append(('User DN', ret['Value']['identity'])) - if 'secondsLeft' in ret['Value']: - records.append(('Proxy validity, secs', {'Value': str(ret['Value']['secondsLeft']), 'Just': 'L'})) - -if gConfig.getValue('/DIRAC/Security/UseServerCertificate', True): - records.append(('Use Server Certificate', 'Yes')) -else: - records.append(('Use Server Certificate', 'No')) -if gConfig.getValue('/DIRAC/Security/SkipCAChecks', False): - records.append(('Skip CA Checks', 'Yes')) -else: - records.append(('Skip CA Checks', 'No')) - -records.append(('DIRAC version', DIRAC.version)) - -fields = ['Option', 'Value'] - -print() -printTable(fields, records, numbering=False) -print() diff --git a/src/DIRAC/Core/scripts/dirac-install-db.py b/src/DIRAC/Core/scripts/dirac-install-db.py deleted file mode 100755 index 1b7a1b62061..00000000000 --- a/src/DIRAC/Core/scripts/dirac-install-db.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -""" -Create a new DB in the MySQL server -""" -# Script initialization and parseCommandLine -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from DIRAC.Core.Base import Script -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgFile] ... DB ...' % Script.scriptName, - 'Arguments:', - ' DB: Name of the Database (mandatory)'])) -Script.parseCommandLine() -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) - -# Script imports -from DIRAC import gConfig -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller -from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities - -__RCSID__ = "$Id$" - -gComponentInstaller.exitOnError = True -gComponentInstaller.getMySQLPasswords() -for db in args: - result = gComponentInstaller.installDatabase(db) - if not result['OK']: - print("ERROR: failed to correctly install %s" % db, result['Message']) - else: - extension, system = result['Value'] - gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, db, overwrite=True) - - if db != 'InstalledComponentsDB': - result = MonitoringUtilities.monitorInstallation('DB', system, db) - if not result['OK']: - print("ERROR: failed to register installation in database: %s" % result['Message']) diff --git a/src/DIRAC/Core/scripts/dirac-install-web-portal.py b/src/DIRAC/Core/scripts/dirac-install-web-portal.py deleted file mode 100755 index 77cbf31161d..00000000000 --- a/src/DIRAC/Core/scripts/dirac-install-web-portal.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-install-web-portal -# Author : Ricardo Graciani -######################################################################## -""" -Do the initial installation of a DIRAC Web portal -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" -# -from DIRAC.Core.Base import Script -Script.disableCS() -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % Script.scriptName])) - -Script.parseCommandLine() - -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller - -gComponentInstaller.exitOnError = True - - -gComponentInstaller.setupPortal() diff --git a/src/DIRAC/Core/scripts/dirac-service.py b/src/DIRAC/Core/scripts/dirac-service.py index 49db27aa5fd..39b54703879 100755 --- a/src/DIRAC/Core/scripts/dirac-service.py +++ b/src/DIRAC/Core/scripts/dirac-service.py @@ -16,6 +16,9 @@ __RCSID__ = "$Id$" +print("NOTE:", __file__, "is deprecated and will be removed in v7r3, for details see", + "https://github.com/DIRACGrid/DIRAC/wiki/DIRAC-v7r2#rename-of-scripts") + localCfg = LocalConfiguration() diff --git a/src/DIRAC/Core/scripts/dirac-setup-site.py b/src/DIRAC/Core/scripts/dirac-setup-site.py deleted file mode 100755 index 7945086a73d..00000000000 --- a/src/DIRAC/Core/scripts/dirac-setup-site.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-setup-site -# Author : Ricardo Graciani -######################################################################## -""" -Initial installation and configuration of a new DIRAC server (DBs, Services, Agents, Web Portal,...) -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -from DIRAC import S_OK -from DIRAC.Core.Base import Script - - -class Params(object): - - def __init__(self): - self.exitOnError = False - - def setExitOnError(self, value): - self.exitOnError = True - return S_OK() - - -cliParams = Params() - -Script.disableCS() -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option] ... [cfgfile]' % Script.scriptName, - 'Arguments:', - ' cfgfile: DIRAC Cfg with description of the configuration (optional)'])) - -Script.registerSwitch( - "e", - "exitOnError", - "flag to exit on error of any component installation", - cliParams.setExitOnError) - -Script.addDefaultOptionValue('/DIRAC/Security/UseServerCertificate', 'yes') -Script.addDefaultOptionValue('LogLevel', 'INFO') -Script.parseCommandLine() -args = Script.getExtraCLICFGFiles() -# -if len(args) > 1: - Script.showHelp(exitCode=1) -# -cfg = None -if len(args): - cfg = args[0] -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller -# -gComponentInstaller.exitOnError = cliParams.exitOnError -# -result = gComponentInstaller.setupSite(Script.localCfg, cfg) -if not result['OK']: - print("ERROR:", result['Message']) - exit(-1) -# -result = gComponentInstaller.getStartupComponentStatus([]) -if not result['OK']: - print('ERROR:', result['Message']) - exit(-1) - -print("\nStatus of installed components:\n") -result = gComponentInstaller.printStartupStatus(result['Value']) -if not result['OK']: - print('ERROR:', result['Message']) - exit(-1) diff --git a/src/DIRAC/Core/scripts/dirac_agent.py b/src/DIRAC/Core/scripts/dirac_agent.py new file mode 100755 index 00000000000..e449967ed6c --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac_agent.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-agent +# Author : Adria Casajus, Andrei Tsaregorodtsev, Stuart Paterson +######################################################################## +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +""" This is a script to launch DIRAC agents +""" + +import sys +from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration +from DIRAC import gLogger +from DIRAC.Core.Base.AgentReactor import AgentReactor +from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + localCfg = LocalConfiguration() + + positionalArgs = localCfg.getPositionalArguments() + if len(positionalArgs) == 0: + gLogger.fatal("You must specify which agent to run!") + sys.exit(1) + + agentName = positionalArgs[0] + localCfg.setConfigurationForAgent(agentName) + localCfg.addMandatoryEntry("/DIRAC/Setup") + localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") + localCfg.addDefaultEntry("LogLevel", "INFO") + localCfg.addDefaultEntry("LogColor", True) + resultDict = localCfg.loadUserData() + if not resultDict['OK']: + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + + includeExtensionErrors() + + agentReactor = AgentReactor(positionalArgs[0]) + result = agentReactor.loadAgentModules(positionalArgs) + if result['OK']: + agentReactor.go() + else: + gLogger.error("Error while loading agent module", result['Message']) + sys.exit(2) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_configure.py b/src/DIRAC/Core/scripts/dirac_configure.py new file mode 100755 index 00000000000..e87350e4590 --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac_configure.py @@ -0,0 +1,586 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-configure +# Author : Ricardo Graciani +######################################################################## +""" + Main script to write dirac.cfg for a new DIRAC installation and initial download of CAs and CRLs + if necessary. + + To be used by VO specific scripts to configure new DIRAC installations + + There are 2 mandatory arguments: + + -S --Setup= To define the DIRAC setup for the current installation + -C --ConfigurationServer=|-W --Gateway To define the reference Configuration Servers/Gateway + for the current installation + + others are optional + + -I --IncludeAllServers To include all Configuration Servers + (by default only those in -C option are included) + -n --SiteName= To define the DIRAC Site Name for the installation + -N --CEName= To determine the DIRAC Site Name from the CE Name + -V --VO= To define the VO for the installation + -U --UseServerCertificate To use Server Certificate for all clients + -H --SkipCAChecks To skip check of CAs for all clients + -D --SkipCADownload To skip download of CAs + -M --SkipVOMSDownload To skip download of VOMS info + -v --UseVersionsDir Use versions directory + (This option will properly define RootPath and InstancePath) + -A --Architecture= To define /LocalSite/Architecture= + -L --LocalSE= To define /LocalSite/LocalSE= + -F --ForceUpdate Forces the update of cfg file (i.e. dirac.cfg), + even if it does already exists (use with care) + -O --Output define output configuration file + + Other arguments will take proper defaults if not defined. + + Additionally all options can all be passed inside a .cfg file passed as argument. + The following options are recognized:: + + Setup + ConfigurationServer + IncludeAllServers + Gateway + SiteName + CEName + VirtualOrganization + UseServerCertificate + SkipCAChecks + SkipCADownload + UseVersionsDir + Architecture + LocalSE + LogLevel + + As in any other script command line option take precedence over .cfg files passed as arguments. + The combination of both is written into the installed dirac.cfg. + + Notice: It will not overwrite exiting info in current dirac.cfg if it exists. + + Example: dirac-configure -d + -S LHCb-Development + -C 'dips://lhcbprod.pic.es:9135/Configuration/Server' + -W 'dips://lhcbprod.pic.es:9135' + --SkipCAChecks + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import sys +import os + +import DIRAC +from DIRAC.Core.Utilities.File import mkDir +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.Core.Security.ProxyInfo import getProxyInfo +from DIRAC.ConfigurationSystem.Client.Helpers import cfgInstallPath, cfgPath, Registry +from DIRAC.Core.Utilities.SiteSEMapping import getSEsForSite +from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient + + +__RCSID__ = "$Id$" + + +logLevel = None +setup = None +configurationServer = None +includeAllServers = False +gatewayServer = None +siteName = None +useServerCert = False +skipCAChecks = False +skipCADownload = False +useVersionsDir = False +architecture = None +localSE = None +ceName = None +vo = None +update = False +outputFile = '' +skipVOMSDownload = False +extensions = '' + + +def setGateway(optionValue): + global gatewayServer + gatewayServer = optionValue + setServer(gatewayServer + '/Configuration/Server') + DIRAC.gConfig.setOptionValue(cfgInstallPath('Gateway'), gatewayServer) + return DIRAC.S_OK() + + +def setOutput(optionValue): + global outputFile + outputFile = optionValue + return DIRAC.S_OK() + + +def setServer(optionValue): + global configurationServer + configurationServer = optionValue + DIRAC.gConfig.setOptionValue('/DIRAC/Configuration/Servers', configurationServer) + DIRAC.gConfig.setOptionValue(cfgInstallPath('ConfigurationServer'), configurationServer) + return DIRAC.S_OK() + + +def setAllServers(optionValue): + global includeAllServers + includeAllServers = True + + +def setSetup(optionValue): + global setup + setup = optionValue + DIRAC.gConfig.setOptionValue('/DIRAC/Setup', setup) + DIRAC.gConfig.setOptionValue(cfgInstallPath('Setup'), setup) + return DIRAC.S_OK() + + +def setSiteName(optionValue): + global siteName + siteName = optionValue + Script.localCfg.addDefaultEntry('/LocalSite/Site', siteName) + DIRAC.__siteName = False + DIRAC.gConfig.setOptionValue(cfgInstallPath('SiteName'), siteName) + return DIRAC.S_OK() + + +def setCEName(optionValue): + global ceName + ceName = optionValue + DIRAC.gConfig.setOptionValue(cfgInstallPath('CEName'), ceName) + return DIRAC.S_OK() + + +def setServerCert(optionValue): + global useServerCert + useServerCert = True + DIRAC.gConfig.setOptionValue(cfgInstallPath('UseServerCertificate'), useServerCert) + return DIRAC.S_OK() + + +def setSkipCAChecks(optionValue): + global skipCAChecks + skipCAChecks = True + DIRAC.gConfig.setOptionValue(cfgInstallPath('SkipCAChecks'), skipCAChecks) + return DIRAC.S_OK() + + +def setSkipCADownload(optionValue): + global skipCADownload + skipCADownload = True + DIRAC.gConfig.setOptionValue(cfgInstallPath('SkipCADownload'), skipCADownload) + return DIRAC.S_OK() + + +def setSkipVOMSDownload(optionValue): + global skipVOMSDownload + skipVOMSDownload = True + DIRAC.gConfig.setOptionValue(cfgInstallPath('SkipVOMSDownload'), skipVOMSDownload) + return DIRAC.S_OK() + + +def setUseVersionsDir(optionValue): + global useVersionsDir + useVersionsDir = True + DIRAC.gConfig.setOptionValue(cfgInstallPath('UseVersionsDir'), useVersionsDir) + return DIRAC.S_OK() + + +def setArchitecture(optionValue): + global architecture + architecture = optionValue + Script.localCfg.addDefaultEntry('/LocalSite/Architecture', architecture) + DIRAC.gConfig.setOptionValue(cfgInstallPath('Architecture'), architecture) + return DIRAC.S_OK() + + +def setLocalSE(optionValue): + global localSE + localSE = optionValue + Script.localCfg.addDefaultEntry('/LocalSite/LocalSE', localSE) + DIRAC.gConfig.setOptionValue(cfgInstallPath('LocalSE'), localSE) + return DIRAC.S_OK() + + +def setVO(optionValue): + global vo + vo = optionValue + Script.localCfg.addDefaultEntry('/DIRAC/VirtualOrganization', vo) + DIRAC.gConfig.setOptionValue(cfgInstallPath('VirtualOrganization'), vo) + return DIRAC.S_OK() + + +def forceUpdate(optionValue): + global update + update = True + return DIRAC.S_OK() + + +def setExtensions(optionValue): + global extensions + extensions = optionValue + DIRAC.gConfig.setOptionValue('/DIRAC/Extensions', extensions) + DIRAC.gConfig.setOptionValue(cfgInstallPath('Extensions'), extensions) + return DIRAC.S_OK() + + +@DIRACScript() +def main(): + global logLevel + global setup + global configurationServer + global includeAllServers + global gatewayServer + global siteName + global useServerCert + global skipCAChecks + global skipCADownload + global useVersionsDir + global architecture + global localSE + global ceName + global vo + global update + global outputFile + global skipVOMSDownload + global extensions + + Script.disableCS() + + Script.registerSwitch("S:", "Setup=", "Set as DIRAC setup", setSetup) + Script.registerSwitch("e:", "Extensions=", "Set as DIRAC extensions", setExtensions) + Script.registerSwitch("C:", "ConfigurationServer=", "Set as DIRAC configuration server", setServer) + Script.registerSwitch("I", "IncludeAllServers", "include all Configuration Servers", setAllServers) + Script.registerSwitch("n:", "SiteName=", "Set as DIRAC Site Name", setSiteName) + Script.registerSwitch("N:", "CEName=", "Determiner from ", setCEName) + Script.registerSwitch("V:", "VO=", "Set the VO name", setVO) + + Script.registerSwitch("W:", "gateway=", "Configure as DIRAC Gateway for the site", setGateway) + + Script.registerSwitch("U", "UseServerCertificate", "Configure to use Server Certificate", setServerCert) + Script.registerSwitch("H", "SkipCAChecks", "Configure to skip check of CAs", setSkipCAChecks) + Script.registerSwitch("D", "SkipCADownload", "Configure to skip download of CAs", setSkipCADownload) + Script.registerSwitch("M", "SkipVOMSDownload", "Configure to skip download of VOMS info", setSkipVOMSDownload) + + Script.registerSwitch("v", "UseVersionsDir", "Use versions directory", setUseVersionsDir) + + Script.registerSwitch("A:", "Architecture=", "Configure /Architecture=", setArchitecture) + Script.registerSwitch("L:", "LocalSE=", "Configure LocalSite/LocalSE=", setLocalSE) + + Script.registerSwitch( + "F", + "ForceUpdate", + "Force Update of cfg file (i.e. dirac.cfg) (otherwise nothing happens if dirac.cfg already exists)", + forceUpdate) + + Script.registerSwitch("O:", "output=", "output configuration file", setOutput) + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + '\nUsage:', + ' %s [option|cfgfile] ...\n' % Script.scriptName])) + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getExtraCLICFGFiles() + + if not logLevel: + logLevel = DIRAC.gConfig.getValue(cfgInstallPath('LogLevel'), '') + if logLevel: + DIRAC.gLogger.setLevel(logLevel) + else: + DIRAC.gConfig.setOptionValue(cfgInstallPath('LogLevel'), logLevel) + + if not gatewayServer: + newGatewayServer = DIRAC.gConfig.getValue(cfgInstallPath('Gateway'), '') + if newGatewayServer: + setGateway(newGatewayServer) + + if not configurationServer: + newConfigurationServer = DIRAC.gConfig.getValue(cfgInstallPath('ConfigurationServer'), '') + if newConfigurationServer: + setServer(newConfigurationServer) + + if not includeAllServers: + newIncludeAllServer = DIRAC.gConfig.getValue(cfgInstallPath('IncludeAllServers'), False) + if newIncludeAllServer: + setAllServers(True) + + if not setup: + newSetup = DIRAC.gConfig.getValue(cfgInstallPath('Setup'), '') + if newSetup: + setSetup(newSetup) + + if not siteName: + newSiteName = DIRAC.gConfig.getValue(cfgInstallPath('SiteName'), '') + if newSiteName: + setSiteName(newSiteName) + + if not ceName: + newCEName = DIRAC.gConfig.getValue(cfgInstallPath('CEName'), '') + if newCEName: + setCEName(newCEName) + + if not useServerCert: + newUserServerCert = DIRAC.gConfig.getValue(cfgInstallPath('UseServerCertificate'), False) + if newUserServerCert: + setServerCert(newUserServerCert) + + if not skipCAChecks: + newSkipCAChecks = DIRAC.gConfig.getValue(cfgInstallPath('SkipCAChecks'), False) + if newSkipCAChecks: + setSkipCAChecks(newSkipCAChecks) + + if not skipCADownload: + newSkipCADownload = DIRAC.gConfig.getValue(cfgInstallPath('SkipCADownload'), False) + if newSkipCADownload: + setSkipCADownload(newSkipCADownload) + + if not useVersionsDir: + newUseVersionsDir = DIRAC.gConfig.getValue(cfgInstallPath('UseVersionsDir'), False) + if newUseVersionsDir: + setUseVersionsDir(newUseVersionsDir) + # Set proper Defaults in configuration (even if they will be properly overwrite by gComponentInstaller + instancePath = os.path.dirname(os.path.dirname(DIRAC.rootPath)) + rootPath = os.path.join(instancePath, 'pro') + DIRAC.gConfig.setOptionValue(cfgInstallPath('InstancePath'), instancePath) + DIRAC.gConfig.setOptionValue(cfgInstallPath('RootPath'), rootPath) + + if not architecture: + newArchitecture = DIRAC.gConfig.getValue(cfgInstallPath('Architecture'), '') + if newArchitecture: + setArchitecture(newArchitecture) + + if not vo: + newVO = DIRAC.gConfig.getValue(cfgInstallPath('VirtualOrganization'), '') + if newVO: + setVO(newVO) + + if not extensions: + newExtensions = DIRAC.gConfig.getValue(cfgInstallPath('Extensions'), '') + if newExtensions: + setExtensions(newExtensions) + + DIRAC.gLogger.notice('Executing: %s ' % (' '.join(sys.argv))) + DIRAC.gLogger.notice('Checking DIRAC installation at "%s"' % DIRAC.rootPath) + + if update: + if outputFile: + DIRAC.gLogger.notice('Will update the output file %s' % outputFile) + else: + DIRAC.gLogger.notice('Will update %s' % DIRAC.gConfig.diracConfigFilePath) + + if setup: + DIRAC.gLogger.verbose('/DIRAC/Setup =', setup) + if vo: + DIRAC.gLogger.verbose('/DIRAC/VirtualOrganization =', vo) + if configurationServer: + DIRAC.gLogger.verbose('/DIRAC/Configuration/Servers =', configurationServer) + + if siteName: + DIRAC.gLogger.verbose('/LocalSite/Site =', siteName) + if architecture: + DIRAC.gLogger.verbose('/LocalSite/Architecture =', architecture) + if localSE: + DIRAC.gLogger.verbose('/LocalSite/localSE =', localSE) + + if not useServerCert: + DIRAC.gLogger.verbose('/DIRAC/Security/UseServerCertificate =', 'no') + # Being sure it was not there before + Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') + Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'no') + else: + DIRAC.gLogger.verbose('/DIRAC/Security/UseServerCertificate =', 'yes') + # Being sure it was not there before + Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') + Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'yes') + + host = DIRAC.gConfig.getValue(cfgInstallPath("Host"), "") + if host: + DIRAC.gConfig.setOptionValue(cfgPath("DIRAC", "Hostname"), host) + + if skipCAChecks: + DIRAC.gLogger.verbose('/DIRAC/Security/SkipCAChecks =', 'yes') + # Being sure it was not there before + Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') + Script.localCfg.addDefaultEntry('/DIRAC/Security/SkipCAChecks', 'yes') + else: + # Necessary to allow initial download of CA's + if not skipCADownload: + DIRAC.gConfig.setOptionValue('/DIRAC/Security/SkipCAChecks', 'yes') + if not skipCADownload: + Script.enableCS() + try: + dirName = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'certificates') + mkDir(dirName) + except BaseException: + DIRAC.gLogger.exception() + DIRAC.gLogger.fatal('Fail to create directory:', dirName) + DIRAC.exit(-1) + try: + bdc = BundleDeliveryClient() + result = bdc.syncCAs() + if result['OK']: + result = bdc.syncCRLs() + except Exception as e: + DIRAC.gLogger.error('Failed to sync CAs and CRLs: %s' % str(e)) + + if not skipCAChecks: + Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') + + if ceName or siteName: + # This is used in the pilot context, we should have a proxy, or a certificate, and access to CS + if useServerCert: + # Being sure it was not there before + Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') + Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'yes') + Script.enableCS() + # Get the site resource section + gridSections = DIRAC.gConfig.getSections('/Resources/Sites/') + if not gridSections['OK']: + DIRAC.gLogger.warn('Could not get grid sections list') + grids = [] + else: + grids = gridSections['Value'] + # try to get siteName from ceName or Local SE from siteName using Remote Configuration + for grid in grids: + siteSections = DIRAC.gConfig.getSections('/Resources/Sites/%s/' % grid) + if not siteSections['OK']: + DIRAC.gLogger.warn('Could not get %s site list' % grid) + sites = [] + else: + sites = siteSections['Value'] + + if not siteName: + if ceName: + for site in sites: + res = DIRAC.gConfig.getSections('/Resources/Sites/%s/%s/CEs/' % (grid, site), []) + if not res['OK']: + DIRAC.gLogger.warn('Could not get %s CEs list' % site) + if ceName in res['Value']: + siteName = site + break + if siteName: + DIRAC.gLogger.notice('Setting /LocalSite/Site = %s' % siteName) + Script.localCfg.addDefaultEntry('/LocalSite/Site', siteName) + DIRAC.__siteName = False + if ceName: + DIRAC.gLogger.notice('Setting /LocalSite/GridCE = %s' % ceName) + Script.localCfg.addDefaultEntry('/LocalSite/GridCE', ceName) + + if not localSE and siteName in sites: + localSE = getSEsForSite(siteName) + if localSE['OK'] and localSE['Value']: + localSE = ','.join(localSE['Value']) + DIRAC.gLogger.notice('Setting /LocalSite/LocalSE =', localSE) + Script.localCfg.addDefaultEntry('/LocalSite/LocalSE', localSE) + break + + if gatewayServer: + DIRAC.gLogger.verbose('/DIRAC/Gateways/%s =' % DIRAC.siteName(), gatewayServer) + Script.localCfg.addDefaultEntry('/DIRAC/Gateways/%s' % DIRAC.siteName(), gatewayServer) + + # Create the local cfg if it is not yet there + if not outputFile: + outputFile = DIRAC.gConfig.diracConfigFilePath + outputFile = os.path.abspath(outputFile) + if not os.path.exists(outputFile): + configDir = os.path.dirname(outputFile) + mkDir(configDir) + update = True + DIRAC.gConfig.dumpLocalCFGToFile(outputFile) + + if includeAllServers: + # We need user proxy or server certificate to continue in order to get all the CS URLs + if not useServerCert: + Script.enableCS() + result = getProxyInfo() + if not result['OK']: + DIRAC.gLogger.notice('Configuration is not completed because no user proxy is available') + DIRAC.gLogger.notice('Create one using dirac-proxy-init and execute again with -F option') + sys.exit(1) + else: + Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') + # When using Server Certs CA's will be checked, the flag only disables initial download + # this will be replaced by the use of SkipCADownload + Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate', 'yes') + Script.enableCS() + + DIRAC.gConfig.setOptionValue('/DIRAC/Configuration/Servers', ','.join(DIRAC.gConfig.getServersList())) + DIRAC.gLogger.verbose('/DIRAC/Configuration/Servers =', ','.join(DIRAC.gConfig.getServersList())) + + if useServerCert: + # always removing before dumping + Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') + Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') + Script.localCfg.deleteOption('/DIRAC/Security/SkipVOMSDownload') + + if update: + DIRAC.gConfig.dumpLocalCFGToFile(outputFile) + + # ## LAST PART: do the vomsdir/vomses magic + + # This has to be done for all VOs in the installation + + if skipVOMSDownload: + # We stop here + sys.exit(0) + + result = Registry.getVOMSServerInfo() + if not result['OK']: + sys.exit(1) + + error = '' + vomsDict = result['Value'] + for vo in vomsDict: + voName = vomsDict[vo]['VOMSName'] + vomsDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'vomsdir', voName) + vomsesDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security', 'vomses') + for path in (vomsDirPath, vomsesDirPath): + mkDir(path) + vomsesLines = [] + for vomsHost in vomsDict[vo].get('Servers', {}): + hostFilePath = os.path.join(vomsDirPath, "%s.lsc" % vomsHost) + try: + DN = vomsDict[vo]['Servers'][vomsHost]['DN'] + CA = vomsDict[vo]['Servers'][vomsHost]['CA'] + port = vomsDict[vo]['Servers'][vomsHost]['Port'] + if not DN or not CA or not port: + DIRAC.gLogger.error('DN = %s' % DN) + DIRAC.gLogger.error('CA = %s' % CA) + DIRAC.gLogger.error('Port = %s' % port) + DIRAC.gLogger.error('Missing Parameter for %s' % vomsHost) + continue + with open(hostFilePath, "wt") as fd: + fd.write("%s\n%s\n" % (DN, CA)) + vomsesLines.append('"%s" "%s" "%s" "%s" "%s" "24"' % (voName, vomsHost, port, DN, voName)) + DIRAC.gLogger.notice("Created vomsdir file %s" % hostFilePath) + except Exception: + DIRAC.gLogger.exception("Could not generate vomsdir file for host", vomsHost) + error = "Could not generate vomsdir file for VO %s, host %s" % (voName, vomsHost) + try: + vomsesFilePath = os.path.join(vomsesDirPath, voName) + with open(vomsesFilePath, "wt") as fd: + fd.write("%s\n" % "\n".join(vomsesLines)) + DIRAC.gLogger.notice("Created vomses file %s" % vomsesFilePath) + except Exception: + DIRAC.gLogger.exception("Could not generate vomses file") + error = "Could not generate vomses file for VO %s" % voName + + if useServerCert: + Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate') + # When using Server Certs CA's will be checked, the flag only disables initial download + # this will be replaced by the use of SkipCADownload + Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks') + + if error: + sys.exit(1) + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac-deploy-scripts.py b/src/DIRAC/Core/scripts/dirac_deploy_scripts.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-deploy-scripts.py rename to src/DIRAC/Core/scripts/dirac_deploy_scripts.py diff --git a/src/DIRAC/Core/scripts/dirac_executor.py b/src/DIRAC/Core/scripts/dirac_executor.py new file mode 100755 index 00000000000..d9a2970b24f --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac_executor.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-executor +# Author : Adria Casajus +######################################################################## +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +""" This is a script to launch DIRAC executors +""" + +import sys +from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration +from DIRAC import gLogger +from DIRAC.Core.Base.ExecutorReactor import ExecutorReactor +from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + localCfg = LocalConfiguration() + + positionalArgs = localCfg.getPositionalArguments() + if len(positionalArgs) == 0: + gLogger.fatal("You must specify which executor to run!") + sys.exit(1) + + if len(positionalArgs) == 1 and positionalArgs[0].find("/") > -1: + mainName = positionalArgs[0] + else: + mainName = "Framework/MultiExecutor" + + localCfg.setConfigurationForExecutor(mainName) + localCfg.addMandatoryEntry("/DIRAC/Setup") + localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") + localCfg.addDefaultEntry("LogLevel", "INFO") + localCfg.addDefaultEntry("LogColor", True) + resultDict = localCfg.loadUserData() + if not resultDict['OK']: + gLogger.fatal("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + + includeExtensionErrors() + executorReactor = ExecutorReactor() + + result = executorReactor.loadModules(positionalArgs) + if not result['OK']: + gLogger.fatal("Error while loading executor", result['Message']) + sys.exit(1) + + result = executorReactor.go() + if not result['OK']: + gLogger.fatal(result['Message']) + sys.exit(1) + + gLogger.notice("Graceful exit. Bye!") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_externals_requirements.py b/src/DIRAC/Core/scripts/dirac_externals_requirements.py new file mode 100755 index 00000000000..9817f819d29 --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac_externals_requirements.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-externals-requirements +# Author : Adri/Federico/Andrei +######################################################################## +""" If RequiredExternals section is found in releases.cfg of any extension, + then some python packages to install with pip may be found. This script + will install the requested modules. + + The command is called from the dirac-install general installation command. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +# TODO: This should be modernised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatibility layer + import subprocess as commands + +from diraccfg import CFG +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC import gLogger, rootPath, S_OK + +__RCSID__ = "$Id$" + +# Default installation type +instType = "server" + + +def setInstallType(val): + global instType + instType = val + return S_OK() + + +def pipInstall(package, switches=""): + # The right pip should be in the PATH, which is the case after sourcing the DIRAC bashrc + cmd = "pip install --trusted-host pypi.python.org %s %s" % (switches, package) + gLogger.notice("Executing %s" % cmd) + return commands.getstatusoutput(cmd) + + +@DIRACScript() +def main(): + Script.disableCS() + + Script.registerSwitch("t:", "type=", "Installation type. 'server' by default.", setInstallType) + Script.parseCommandLine(ignoreErrors=True) + + # Collect all the requested python modules to install + reqDict = {} + for entry in os.listdir(rootPath): + if len(entry) < 5 or entry.find("DIRAC") != len(entry) - 5: + continue + reqFile = os.path.join(rootPath, entry, "releases.cfg") + try: + with open(reqFile, "r") as extfd: + reqCFG = CFG().loadFromBuffer(extfd.read()) + except BaseException: + gLogger.verbose("%s not found" % reqFile) + continue + reqList = reqCFG.getOption("/RequiredExternals/%s" % instType.capitalize(), []) + if not reqList: + gLogger.verbose("%s does not have requirements for %s installation" % (entry, instType)) + continue + for req in reqList: + reqName = False + reqCond = "" + for cond in ("==", ">="): + iP = cond.find(req) + if iP > 0: + reqName = req[:iP] + reqCond = req[iP:] + break + if not reqName: + reqName = req + if reqName not in reqDict: + reqDict[reqName] = (reqCond, entry) + else: + gLogger.notice("Skipping %s, it's already requested by %s" % (reqName, reqDict[reqName][1])) + + if not reqDict: + gLogger.notice("No extra python module requested to be installed") + sys.exit(0) + + for reqName in reqDict: + package = "%s%s" % (reqName, reqDict[reqName][0]) + gLogger.notice("Requesting installation of %s" % package) + status, output = pipInstall(package) + if status != 0: + gLogger.error(output) + else: + gLogger.notice("Successfully installed %s" % package) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_info.py b/src/DIRAC/Core/scripts/dirac_info.py new file mode 100755 index 00000000000..745ba252afc --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac_info.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-info +# Author : Andrei Tsaregorodtsev +######################################################################## +""" + Report info about local DIRAC installation +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + import os + + import DIRAC + from DIRAC import gConfig + from DIRAC.Core.Base import Script + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup + from DIRAC.Core.Utilities.PrettyPrint import printTable + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Site' % Script.scriptName, ])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + records = [] + + records.append(('Setup', gConfig.getValue('/DIRAC/Setup', 'Unknown'))) + records.append(('ConfigurationServer', gConfig.getValue('/DIRAC/Configuration/Servers', []))) + records.append(('Installation path', DIRAC.rootPath)) + + if os.path.exists(os.path.join(DIRAC.rootPath, DIRAC.getPlatform(), 'bin', 'mysql')): + records.append(('Installation type', 'server')) + else: + records.append(('Installation type', 'client')) + + records.append(('Platform', DIRAC.getPlatform())) + + ret = getProxyInfo(disableVOMS=True) + if ret['OK']: + if 'group' in ret['Value']: + vo = getVOForGroup(ret['Value']['group']) + else: + vo = getVOForGroup('') + if not vo: + vo = "None" + records.append(('VirtualOrganization', vo)) + if 'identity' in ret['Value']: + records.append(('User DN', ret['Value']['identity'])) + if 'secondsLeft' in ret['Value']: + records.append(('Proxy validity, secs', {'Value': str(ret['Value']['secondsLeft']), 'Just': 'L'})) + + if gConfig.getValue('/DIRAC/Security/UseServerCertificate', True): + records.append(('Use Server Certificate', 'Yes')) + else: + records.append(('Use Server Certificate', 'No')) + if gConfig.getValue('/DIRAC/Security/SkipCAChecks', False): + records.append(('Skip CA Checks', 'Yes')) + else: + records.append(('Skip CA Checks', 'No')) + + records.append(('DIRAC version', DIRAC.version)) + + fields = ['Option', 'Value'] + + print() + printTable(fields, records, numbering=False) + print() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_install_db.py b/src/DIRAC/Core/scripts/dirac_install_db.py new file mode 100755 index 00000000000..0778e809c4f --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac_install_db.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +""" +Create a new DB in the MySQL server +""" +# Script initialization and parseCommandLine +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgFile] ... DB ...' % Script.scriptName, + 'Arguments:', + ' DB: Name of the Database (mandatory)'])) + Script.parseCommandLine() + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) + + # Script imports + from DIRAC import gConfig + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities + + __RCSID__ = "$Id$" + + gComponentInstaller.exitOnError = True + gComponentInstaller.getMySQLPasswords() + for db in args: + result = gComponentInstaller.installDatabase(db) + if not result['OK']: + print("ERROR: failed to correctly install %s" % db, result['Message']) + else: + extension, system = result['Value'] + gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, db, overwrite=True) + + if db != 'InstalledComponentsDB': + result = MonitoringUtilities.monitorInstallation('DB', system, db) + if not result['OK']: + print("ERROR: failed to register installation in database: %s" % result['Message']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac-install-extension.py b/src/DIRAC/Core/scripts/dirac_install_extension.py similarity index 100% rename from src/DIRAC/Core/scripts/dirac-install-extension.py rename to src/DIRAC/Core/scripts/dirac_install_extension.py diff --git a/src/DIRAC/Core/scripts/dirac_install_web_portal.py b/src/DIRAC/Core/scripts/dirac_install_web_portal.py new file mode 100755 index 00000000000..4e8d7f81af8 --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac_install_web_portal.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-install-web-portal +# Author : Ricardo Graciani +######################################################################## +""" +Do the initial installation of a DIRAC Web portal +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" +# +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.disableCS() + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % Script.scriptName])) + + Script.parseCommandLine() + + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + + gComponentInstaller.exitOnError = True + gComponentInstaller.setupPortal() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac-platform.py b/src/DIRAC/Core/scripts/dirac_platform.py similarity index 96% rename from src/DIRAC/Core/scripts/dirac-platform.py rename to src/DIRAC/Core/scripts/dirac_platform.py index fa0b9f1deef..f134cb03d44 100755 --- a/src/DIRAC/Core/scripts/dirac-platform.py +++ b/src/DIRAC/Core/scripts/dirac_platform.py @@ -82,7 +82,6 @@ def libc_ver(executable=sys.executable, lib='', version='', chunksize=2048): return lib, '.'.join(map(str, version)) # Command line interface - def getPlatformString(): # Modified to return our desired platform string, R. Graciani platformTuple = (platform.system(), platform.machine()) @@ -127,5 +126,15 @@ def getPlatformString(): return platformString + def main(): + print(getPlatformString()) +else: + from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + @DIRACScript() + def main(): + print(getPlatformString()) + + if __name__ == "__main__": - print(getPlatformString()) + main() diff --git a/src/DIRAC/Core/scripts/dirac_service.py b/src/DIRAC/Core/scripts/dirac_service.py new file mode 100755 index 00000000000..7626ce51622 --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac_service.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-service +# Author : Adria Casajus +######################################################################## + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import sys + +from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration +from DIRAC.FrameworkSystem.Client.Logger import gLogger +from DIRAC.Core.DISET.ServiceReactor import ServiceReactor +from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + +__RCSID__ = "$Id$" + + +@DIRACScript() +def main(): + localCfg = LocalConfiguration() + + positionalArgs = localCfg.getPositionalArguments() + if len(positionalArgs) == 0: + gLogger.fatal("You must specify which server to run!") + sys.exit(1) + + serverName = positionalArgs[0] + localCfg.setConfigurationForServer(serverName) + localCfg.addMandatoryEntry("Port") + # localCfg.addMandatoryEntry( "HandlerPath" ) + localCfg.addMandatoryEntry("/DIRAC/Setup") + localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") + localCfg.addDefaultEntry("LogLevel", "INFO") + localCfg.addDefaultEntry("LogColor", True) + resultDict = localCfg.loadUserData() + if not resultDict['OK']: + gLogger.initialize(serverName, "/") + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + + includeExtensionErrors() + + serverToLaunch = ServiceReactor() + result = serverToLaunch.initialize(positionalArgs) + if not result['OK']: + gLogger.error(result['Message']) + sys.exit(1) + + result = serverToLaunch.serve() + if not result['OK']: + gLogger.error(result['Message']) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_setup_site.py b/src/DIRAC/Core/scripts/dirac_setup_site.py new file mode 100755 index 00000000000..a99d454ff84 --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac_setup_site.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-setup-site +# Author : Ricardo Graciani +######################################################################## +""" +Initial installation and configuration of a new DIRAC server (DBs, Services, Agents, Web Portal,...) +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +from DIRAC import S_OK +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +class Params(object): + + def __init__(self): + self.exitOnError = False + + def setExitOnError(self, value): + self.exitOnError = True + return S_OK() + + +@DIRACScript() +def main(): + cliParams = Params() + + Script.disableCS() + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option] ... [cfgfile]' % Script.scriptName, + 'Arguments:', + ' cfgfile: DIRAC Cfg with description of the configuration (optional)'])) + + Script.registerSwitch( + "e", + "exitOnError", + "flag to exit on error of any component installation", + cliParams.setExitOnError) + + Script.addDefaultOptionValue('/DIRAC/Security/UseServerCertificate', 'yes') + Script.addDefaultOptionValue('LogLevel', 'INFO') + Script.parseCommandLine() + args = Script.getExtraCLICFGFiles() + + if len(args) > 1: + Script.showHelp(exitCode=1) + + cfg = None + if len(args): + cfg = args[0] + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + + gComponentInstaller.exitOnError = cliParams.exitOnError + + result = gComponentInstaller.setupSite(Script.localCfg, cfg) + if not result['OK']: + print("ERROR:", result['Message']) + exit(-1) + + result = gComponentInstaller.getStartupComponentStatus([]) + if not result['OK']: + print('ERROR:', result['Message']) + exit(-1) + + print("\nStatus of installed components:\n") + result = gComponentInstaller.printStartupStatus(result['Value']) + if not result['OK']: + print('ERROR:', result['Message']) + exit(-1) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac-version.py b/src/DIRAC/Core/scripts/dirac_version.py similarity index 73% rename from src/DIRAC/Core/scripts/dirac-version.py rename to src/DIRAC/Core/scripts/dirac_version.py index 09dce990d0c..6cd1072a1b4 100755 --- a/src/DIRAC/Core/scripts/dirac-version.py +++ b/src/DIRAC/Core/scripts/dirac_version.py @@ -11,4 +11,13 @@ from __future__ import division import DIRAC -print(DIRAC.version) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + print(DIRAC.version) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py b/src/DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py index 7ec8396c465..dd1c9f038b9 100644 --- a/src/DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py +++ b/src/DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py @@ -84,7 +84,7 @@ def get_filename(self, path, dirname): if __name__ == "__main__": - from AbstractFileSystem import UnixLikeFileSystem # pylint: disable=import-error + from DIRAC.DataManagementSystem.Client.CmdDirCompletion.AbstractFileSystem import UnixLikeFileSystem ulfs = UnixLikeFileSystem() dc = DirectoryCompletion(ulfs) diff --git a/src/DIRAC/DataManagementSystem/Service/StorageElementProxyHandler.py b/src/DIRAC/DataManagementSystem/Service/StorageElementProxyHandler.py index 0527be85de6..2fbfb556325 100644 --- a/src/DIRAC/DataManagementSystem/Service/StorageElementProxyHandler.py +++ b/src/DIRAC/DataManagementSystem/Service/StorageElementProxyHandler.py @@ -13,7 +13,7 @@ import os import shutil -import SocketServer +from six.moves import socketserver import threading import socket import random @@ -75,7 +75,7 @@ def initializeStorageElementProxyHandler(serviceInfo): return S_OK() -class ThreadedSocketServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): +class ThreadedSocketServer(socketserver.ThreadingMixIn, socketserver.TCPServer): """ bag dummy class to hold ThreadingMixIn and TCPServer """ pass diff --git a/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py b/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py index 850d10607b0..9bcf1a06be4 100644 --- a/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py +++ b/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py @@ -14,10 +14,11 @@ __RCS__ = "$Id$" import os -import BaseHTTPServer import shutil import random +from six.moves import BaseHTTPServer + from DIRAC.Core.Utilities.DictCache import DictCache diff --git a/src/DIRAC/DataManagementSystem/scripts/__init__.py b/src/DIRAC/DataManagementSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-admin-allow-se.py b/src/DIRAC/DataManagementSystem/scripts/dirac-admin-allow-se.py deleted file mode 100755 index 1baafd4d3f0..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-admin-allow-se.py +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env python -""" Enable using one or more Storage Elements -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -read = False -write = False -check = False -remove = False -site = '' -mute = False - -Script.setUsageMessage(""" -Enable using one or more Storage Elements - -Usage: - %s SE1 [SE2 ...] -""" % Script.scriptName) - -Script.registerSwitch("r", "AllowRead", " Allow only reading from the storage element") -Script.registerSwitch("w", "AllowWrite", " Allow only writing to the storage element") -Script.registerSwitch("k", "AllowCheck", " Allow only check access to the storage element") -Script.registerSwitch("v", "AllowRemove", " Allow only remove access to the storage element") -Script.registerSwitch("a", "All", " Allow all access to the storage element") -Script.registerSwitch("m", "Mute", " Do not send email") -Script.registerSwitch("S:", "Site=", " Allow all SEs associated to site") - -Script.parseCommandLine(ignoreErrors=True) - -ses = Script.getPositionalArgs() -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() in ("r", "allowread"): - read = True - if switch[0].lower() in ("w", "allowwrite"): - write = True - if switch[0].lower() in ("k", "allowcheck"): - check = True - if switch[0].lower() in ("v", "allowremove"): - remove = True - if switch[0].lower() in ("a", "all"): - read = True - write = True - check = True - remove = True - if switch[0].lower() in ("m", "mute"): - mute = True - if switch[0].lower() in ("s", "site"): - site = switch[1] - -# imports -from DIRAC import gConfig, gLogger -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus - -if not (read or write or check or remove): - # No switch was specified, means we need all of them - gLogger.notice("No option given, all accesses will be allowed if they were not") - read = True - write = True - check = True - remove = True - -ses = resolveSEGroup(ses) -diracAdmin = DiracAdmin() -errorList = [] -setup = gConfig.getValue('/DIRAC/Setup', '') -if not setup: - print('ERROR: Could not contact Configuration Service') - DIRAC.exit(2) - -res = getProxyInfo() -if not res['OK']: - gLogger.error('Failed to get proxy information', res['Message']) - DIRAC.exit(2) - -userName = res['Value'].get('username') -if not userName: - gLogger.error('Failed to get username for proxy') - DIRAC.exit(2) - -if site: - res = getSites() - if not res['OK']: - gLogger.error(res['Message']) - DIRAC.exit(-1) - if site not in res['Value']: - gLogger.error('The provided site (%s) is not known.' % site) - DIRAC.exit(-1) - ses.extend(res['Value']['SE'].replace(' ', '').split(',')) -if not ses: - gLogger.error('There were no SEs provided') - DIRAC.exit() - -STATUS_TYPES = ["ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess"] -ALLOWED_STATUSES = ["Unknown", "InActive", "Banned", "Probing", "Degraded"] - -statusAllowedDict = {} -for statusType in STATUS_TYPES: - statusAllowedDict[statusType] = [] - -statusFlagDict = {} -statusFlagDict['ReadAccess'] = read -statusFlagDict['WriteAccess'] = write -statusFlagDict['CheckAccess'] = check -statusFlagDict['RemoveAccess'] = remove - -resourceStatus = ResourceStatus() - -res = resourceStatus.getElementStatus(ses, "StorageElement") -if not res['OK']: - gLogger.error('Storage Element %s does not exist' % ses) - DIRAC.exit(-1) - -reason = 'Forced with dirac-admin-allow-se by %s' % userName - -for se, seOptions in res['Value'].items(): - # InActive is used on the CS model, Banned is the equivalent in RSS - for statusType in STATUS_TYPES: - if statusFlagDict[statusType]: - if seOptions.get(statusType) == "Active": - gLogger.notice('%s status of %s is already Active' % (statusType, se)) - continue - if statusType in seOptions: - if not seOptions[statusType] in ALLOWED_STATUSES: - gLogger.notice('%s option for %s is %s, instead of %s' % - (statusType, se, seOptions['ReadAccess'], ALLOWED_STATUSES)) - gLogger.notice('Try specifying the command switches') - else: - resR = resourceStatus.setElementStatus(se, "StorageElement", statusType, 'Active', reason, userName) - if not resR['OK']: - gLogger.fatal("Failed to update %s %s to Active, exit -" % (se, statusType), resR['Message']) - DIRAC.exit(-1) - else: - gLogger.notice("Successfully updated %s %s to Active" % (se, statusType)) - statusAllowedDict[statusType].append(se) - -totalAllowed = 0 -totalAllowedSEs = [] -for statusType in STATUS_TYPES: - totalAllowed += len(statusAllowedDict[statusType]) - totalAllowedSEs += statusAllowedDict[statusType] -totalAllowedSEs = list(set(totalAllowedSEs)) - -if not totalAllowed: - gLogger.info("No storage elements were allowed") - DIRAC.exit(-1) - -if mute: - gLogger.notice('Email is muted by script switch') - DIRAC.exit(0) - -subject = '%s storage elements allowed for use' % len(totalAllowedSEs) -addressPath = 'EMail/Production' -address = Operations().getValue(addressPath, '') - - -body = '' -if read: - body = "%s\n\nThe following storage elements were allowed for reading:" % body - for se in statusAllowedDict['ReadAccess']: - body = "%s\n%s" % (body, se) -if write: - body = "%s\n\nThe following storage elements were allowed for writing:" % body - for se in statusAllowedDict['WriteAccess']: - body = "%s\n%s" % (body, se) -if check: - body = "%s\n\nThe following storage elements were allowed for checking:" % body - for se in statusAllowedDict['CheckAccess']: - body = "%s\n%s" % (body, se) -if remove: - body = "%s\n\nThe following storage elements were allowed for removing:" % body - for se in statusAllowedDict['RemoveAccess']: - body = "%s\n%s" % (body, se) - -if not address: - gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) - DIRAC.exit(0) - -res = diracAdmin.sendMail(address, subject, body) -gLogger.notice('Notifying %s' % address) -if res['OK']: - gLogger.notice(res['Value']) -else: - gLogger.notice(res['Message']) - -DIRAC.exit(0) - -################################################################################ -# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-admin-ban-se.py b/src/DIRAC/DataManagementSystem/scripts/dirac-admin-ban-se.py deleted file mode 100755 index 04519a5cd21..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-admin-ban-se.py +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env python - -""" Ban one or more Storage Elements for usage -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -read = True -write = True -check = True -remove = True -site = '' -mute = False - -Script.setUsageMessage(""" -Ban one or more Storage Elements for usage - -Usage: - %s SE1 [SE2 ...] -""" % Script.scriptName) - -Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element") -Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element") -Script.registerSwitch("k", "BanCheck", " Ban check access to the storage element") -Script.registerSwitch("v", "BanRemove", " Ban remove access to the storage element") -Script.registerSwitch("a", "All", " Ban all access to the storage element") -Script.registerSwitch("m", "Mute", " Do not send email") -Script.registerSwitch( - "S:", - "Site=", - " Ban all SEs associate to site (note that if writing is allowed, check is always allowed)") -Script.parseCommandLine(ignoreErrors=True) - -ses = Script.getPositionalArgs() -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() in ("r", "banread"): - write = False - check = False - remove = False - if switch[0].lower() in ("w", "banwrite"): - read = False - check = False - remove = False - if switch[0].lower() in ("k", "bancheck"): - read = False - write = False - remove = False - if switch[0].lower() in ("v", "banremove"): - read = False - write = False - check = False - if switch[0].lower() in ("a", "all"): - pass - if switch[0].lower() in ("m", "mute"): - mute = True - if switch[0].lower() in ("s", "site"): - site = switch[1] - -# from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI -from DIRAC import gConfig, gLogger -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus -from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup - -ses = resolveSEGroup(ses) -diracAdmin = DiracAdmin() -errorList = [] -setup = gConfig.getValue('/DIRAC/Setup', '') -if not setup: - print('ERROR: Could not contact Configuration Service') - DIRAC.exit(2) - -res = getProxyInfo() -if not res['OK']: - gLogger.error('Failed to get proxy information', res['Message']) - DIRAC.exit(2) - -userName = res['Value'].get('username') -if not userName: - gLogger.error('Failed to get username for proxy') - DIRAC.exit(2) - -if site: - res = getSites() - if not res['OK']: - gLogger.error(res['Message']) - DIRAC.exit(-1) - if site not in res['Value']: - gLogger.error('The provided site (%s) is not known.' % site) - DIRAC.exit(-1) - ses.extend(res['Value']['SE'].replace(' ', '').split(',')) - -if not ses: - gLogger.error('There were no SEs provided') - DIRAC.exit(-1) - -readBanned = [] -writeBanned = [] -checkBanned = [] -removeBanned = [] - -resourceStatus = ResourceStatus() - -res = resourceStatus.getElementStatus(ses, "StorageElement") -if not res['OK']: - gLogger.error("Storage Element %s does not exist" % ses) - DIRAC.exit(-1) - -reason = 'Forced with dirac-admin-ban-se by %s' % userName - -for se, seOptions in res['Value'].items(): - - resW = resC = resR = {'OK': False} - - # Eventually, we will get rid of the notion of InActive, as we always write Banned. - if read and 'ReadAccess' in seOptions: - - if not seOptions['ReadAccess'] in ['Active', 'Degraded', 'Probing']: - gLogger.notice('Read option for %s is %s, instead of %s' % - (se, seOptions['ReadAccess'], ['Active', 'Degraded', 'Probing'])) - gLogger.notice('Try specifying the command switches') - else: - - resR = resourceStatus.setElementStatus(se, 'StorageElement', 'ReadAccess', 'Banned', reason, userName) - # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) - if not resR['OK']: - gLogger.error('Failed to update %s read access to Banned' % se) - else: - gLogger.notice('Successfully updated %s read access to Banned' % se) - readBanned.append(se) - - # Eventually, we will get rid of the notion of InActive, as we always write Banned. - if write and 'WriteAccess' in seOptions: - - if not seOptions['WriteAccess'] in ['Active', 'Degraded', 'Probing']: - gLogger.notice('Write option for %s is %s, instead of %s' % - (se, seOptions['WriteAccess'], ['Active', 'Degraded', 'Probing'])) - gLogger.notice('Try specifying the command switches') - else: - - resW = resourceStatus.setElementStatus(se, 'StorageElement', 'WriteAccess', 'Banned', reason, userName) - # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) - if not resW['OK']: - gLogger.error("Failed to update %s write access to Banned" % se) - else: - gLogger.notice("Successfully updated %s write access to Banned" % se) - writeBanned.append(se) - - # Eventually, we will get rid of the notion of InActive, as we always write Banned. - if check and 'CheckAccess' in seOptions: - - if not seOptions['CheckAccess'] in ['Active', 'Degraded', 'Probing']: - gLogger.notice('Check option for %s is %s, instead of %s' % - (se, seOptions['CheckAccess'], ['Active', 'Degraded', 'Probing'])) - gLogger.notice('Try specifying the command switches') - else: - - resC = resourceStatus.setElementStatus(se, 'StorageElement', 'CheckAccess', 'Banned', reason, userName) - # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) - if not resC['OK']: - gLogger.error("Failed to update %s check access to Banned" % se) - else: - gLogger.notice("Successfully updated %s check access to Banned" % se) - checkBanned.append(se) - - # Eventually, we will get rid of the notion of InActive, as we always write Banned. - if remove and 'RemoveAccess' in seOptions: - - if not seOptions['RemoveAccess'] in ['Active', 'Degraded', 'Probing']: - gLogger.notice('Remove option for %s is %s, instead of %s' % - (se, seOptions['RemoveAccess'], ['Active', 'Degraded', 'Probing'])) - gLogger.notice('Try specifying the command switches') - else: - - resC = resourceStatus.setElementStatus(se, 'StorageElement', 'RemoveAccess', 'Banned', reason, userName) - # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) - if not resC['OK']: - gLogger.error("Failed to update %s remove access to Banned" % se) - else: - gLogger.notice("Successfully updated %s remove access to Banned" % se) - removeBanned.append(se) - - if not(resR['OK'] or resW['OK'] or resC['OK']): - DIRAC.exit(-1) - -if not (writeBanned or readBanned or checkBanned or removeBanned): - gLogger.notice("No storage elements were banned") - DIRAC.exit(-1) - -if mute: - gLogger.notice('Email is muted by script switch') - DIRAC.exit(0) - -subject = '%s storage elements banned for use' % len(writeBanned + readBanned + checkBanned + removeBanned) -addressPath = 'EMail/Production' -address = Operations().getValue(addressPath, '') - -body = '' -if read: - body = "%s\n\nThe following storage elements were banned for reading:" % body - for se in readBanned: - body = "%s\n%s" % (body, se) -if write: - body = "%s\n\nThe following storage elements were banned for writing:" % body - for se in writeBanned: - body = "%s\n%s" % (body, se) -if check: - body = "%s\n\nThe following storage elements were banned for check access:" % body - for se in checkBanned: - body = "%s\n%s" % (body, se) -if remove: - body = "%s\n\nThe following storage elements were banned for remove access:" % body - for se in removeBanned: - body = "%s\n%s" % (body, se) - -if not address: - gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) - DIRAC.exit(0) - -res = diracAdmin.sendMail(address, subject, body) -gLogger.notice('Notifying %s' % address) -if res['OK']: - gLogger.notice(res['Value']) -else: - gLogger.notice(res['Message']) -DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-admin-user-quota.py b/src/DIRAC/DataManagementSystem/scripts/dirac-admin-user-quota.py deleted file mode 100755 index cb7977e868b..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-admin-user-quota.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# $HeadURL$ -######################################################################## -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage(""" -Show storage quotas for specified users or for all registered users if nobody is specified - -Usage: - %s [user1 ...] -""" % Script.scriptName) - -Script.parseCommandLine() -users = Script.getPositionalArgs() - -from DIRAC import gLogger, gConfig -from DIRAC.Core.Security.ProxyInfo import getProxyInfo - -if not users: - res = gConfig.getSections('/Registry/Users') - if not res['OK']: - gLogger.error("Failed to retrieve user list from CS", res['Message']) - DIRAC.exit(2) - users = res['Value'] - -gLogger.notice("-" * 30) -gLogger.notice("%s|%s" % ('Username'.ljust(15), 'Quota (GB)'.rjust(15))) -gLogger.notice("-" * 30) -for user in sorted(users): - quota = gConfig.getValue('/Registry/Users/%s/Quota' % user, 0) - if not quota: - quota = gConfig.getValue('/Registry/DefaultStorageQuota') - gLogger.notice("%s|%s" % (user.ljust(15), str(quota).rjust(15))) -gLogger.notice("-" * 30) -DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-add-file.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-add-file.py deleted file mode 100755 index cb4c6fedd57..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-add-file.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-dms-add-file -# Author : Stuart Paterson -######################################################################## -""" - Upload a file to the grid storage and register it in the File Catalog -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script -from DIRAC import S_OK -import os -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - '\nUsage:', - ' %s [option|cfgfile] ... LFN Path SE [GUID]' % Script.scriptName, - '\nArguments:', - ' LFN: Logical File Name', - ' Path: Local path to the file', - ' SE: DIRAC Storage Element', - ' GUID: GUID to use in the registration (optional)', - '', - '**OR**', - '', - 'Usage:', - ' %s [option|cfgfile] ... LocalFile' % Script.scriptName, - '\nArguments:', - ' LocalFile: Path to local file containing all the above, i.e.:', - ' lfn1 localfile1 SE [GUID1]', - ' lfn2 localfile2 SE [GUID2]']) - ) -overwrite = False - - -def setOverwrite(arg): - global overwrite - overwrite = True - return S_OK() - - -Script.registerSwitch("f", "force", "Force overwrite of existing file", setOverwrite) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 1 or len(args) > 4: - Script.showHelp(exitCode=1) - - -def getDict(item_list): - """ - From the input list, populate the dictionary - """ - lfn_dict = {} - lfn_dict['lfn'] = item_list[0].replace('LFN:', '').replace('lfn:', '') - lfn_dict['localfile'] = item_list[1] - lfn_dict['SE'] = item_list[2] - guid = None - if len(item_list) > 3: - guid = item_list[3] - lfn_dict['guid'] = guid - return lfn_dict - - -from DIRAC.DataManagementSystem.Client.DataManager import DataManager -from DIRAC import gLogger -import DIRAC -exitCode = 0 - -lfns = [] -if len(args) == 1: - inputFileName = args[0] - if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - for line in inputFile: - line = line.rstrip() - items = line.split() - items[0] = items[0].replace('LFN:', '').replace('lfn:', '') - lfns.append(getDict(items)) - inputFile.close() - else: - gLogger.error("Error: LFN list '%s' missing." % inputFileName) - exitCode = 4 -else: - lfns.append(getDict(args)) - -dm = DataManager() -for lfn in lfns: - if not os.path.exists(lfn['localfile']): - gLogger.error("File %s must exist locally" % lfn['localfile']) - exitCode = 1 - continue - if not os.path.isfile(lfn['localfile']): - gLogger.error("%s is not a file" % lfn['localfile']) - exitCode = 2 - continue - - gLogger.notice("\nUploading %s" % lfn['lfn']) - res = dm.putAndRegister(lfn['lfn'], lfn['localfile'], lfn['SE'], lfn['guid'], overwrite=overwrite) - if not res['OK']: - exitCode = 3 - gLogger.error('Error: failed to upload %s to %s' % (lfn['lfn'], lfn['SE'])) - continue - else: - gLogger.notice('Successfully uploaded file to %s' % lfn['SE']) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-catalog-metadata.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-catalog-metadata.py deleted file mode 100755 index 7fc6a429271..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-catalog-metadata.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -from DIRAC import exit as DIRACExit -from DIRAC.Core.Base import Script - -Script.setUsageMessage(""" -Get metadata for the given file specified by its Logical File Name or for a list of files -contained in the specifed file - -Usage: - %s [Catalog] -""" % Script.scriptName) - -Script.parseCommandLine() - -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog - -import os -args = Script.getPositionalArgs() - -if not len(args) >= 1: - Script.showHelp(exitCode=1) -else: - inputFileName = args[0] - catalogs = [] - if len(args) == 2: - catalogs = [args[1]] - - -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - lfns = string.splitlines() - inputFile.close() -else: - lfns = [inputFileName] - -res = FileCatalog(catalogs=catalogs).getFileMetadata(lfns) -if not res['OK']: - print("ERROR:", res['Message']) - DIRACExit(-1) - -print( - '%s %s %s %s %s' % - ('FileName'.ljust(100), - 'Size'.ljust(10), - 'GUID'.ljust(40), - 'Status'.ljust(8), - 'Checksum'.ljust(10))) -for lfn in sorted(res['Value']['Successful'].keys()): - metadata = res['Value']['Successful'][lfn] - checksum = '' - if 'Checksum' in metadata: - checksum = str(metadata['Checksum']) - size = '' - if 'Size' in metadata: - size = str(metadata['Size']) - guid = '' - if 'GUID' in metadata: - guid = str(metadata['GUID']) - status = '' - if 'Status' in metadata: - status = str(metadata['Status']) - print('%s %s %s %s %s' % (lfn.ljust(100), size.ljust(10), guid.ljust(40), status.ljust(8), checksum.ljust(10))) - -for lfn in sorted(res['Value']['Failed'].keys()): - message = res['Value']['Failed'][lfn] - print(lfn, message) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-change-replica-status.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-change-replica-status.py deleted file mode 100755 index 8cc76bcfa25..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-change-replica-status.py +++ /dev/null @@ -1,60 +0,0 @@ -#! /usr/bin/env python -######################################################################## -# $HeadURL$ -######################################################################## -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -from DIRAC import exit as DIRACExit -from DIRAC.Core.Base import Script - -Script.setUsageMessage(""" -Change status of replica of a given file or a list of files at a given Storage Element - -Usage: - %s -""" % Script.scriptName) - -Script.parseCommandLine() - -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -catalog = FileCatalog() -import os -args = Script.getPositionalArgs() -if not len(args) == 3: - Script.showHelp(exitCode=1) -else: - inputFileName = args[0] - se = args[1] - newStatus = args[2] - -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - lfns = string.splitlines() - inputFile.close() -else: - lfns = [inputFileName] - -res = catalog.getReplicas(lfns, True) -if not res['OK']: - print(res['Message']) - DIRACExit(-1) -replicas = res['Value']['Successful'] - -lfnDict = {} -for lfn in lfns: - lfnDict[lfn] = {} - lfnDict[lfn]['SE'] = se - lfnDict[lfn]['Status'] = newStatus - lfnDict[lfn]['PFN'] = replicas[lfn][se] - -res = catalog.setReplicaStatus(lfnDict) -if not res['OK']: - print("ERROR:", res['Message']) -if res['Value']['Failed']: - print("Failed to update %d replica status" % len(res['Value']['Failed'])) -if res['Value']['Successful']: - print("Successfully updated %d replica status" % len(res['Value']['Successful'])) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-clean-directory.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-clean-directory.py deleted file mode 100755 index 63e041ff8ea..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-clean-directory.py +++ /dev/null @@ -1,52 +0,0 @@ -#! /usr/bin/env python - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -import os - -from DIRAC import exit as DIRACExit, gLogger -from DIRAC.Core.Base import Script - -Script.setUsageMessage(""" -Clean the given directory or a list of directories by removing it and all the -contained files and subdirectories from the physical storage and from the -file catalogs. - -Usage: - %s -""" % Script.scriptName) - -Script.parseCommandLine() - -args = Script.getPositionalArgs() -if len(args) != 1: - Script.showHelp(exitCode=1) - -inputFileName = args[0] - -if os.path.exists(inputFileName): - lfns = [lfn.strip().split()[0] for lfn in sorted(open(inputFileName, 'r').read().splitlines())] -else: - lfns = [inputFileName] - -from DIRAC.DataManagementSystem.Client.DataManager import DataManager -dm = DataManager() -retVal = 0 -for lfn in [lfn for lfn in lfns if lfn]: - gLogger.notice("Cleaning directory %r ... " % lfn) - result = dm.cleanLogicalDirectory(lfn) - if not result['OK']: - gLogger.error('Failed to clean directory', result['Message']) - retVal = -1 - else: - if not result['Value']['Failed']: - gLogger.notice('OK') - else: - for folder, message in result['Value']['Failed'].items(): - gLogger.error('Failed to clean folder', "%r: %s" % (folder, message)) - retVal = -1 - - DIRACExit(retVal) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-removal-request.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-removal-request.py deleted file mode 100755 index f85b8e8f6f2..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-removal-request.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python -""" Create a DIRAC RemoveReplica|RemoveFile request to be executed by the RMS -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "ea64b42 (2012-07-29 16:45:05 +0200) ricardo " - -import os -from hashlib import md5 -import time -from DIRAC.Core.Base import Script -from DIRAC.Core.Utilities.List import breakListIntoChunks - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[0], - __doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... SE LFN ...' % Script.scriptName, - 'Arguments:', - ' SE: StorageElement|All', - ' LFN: LFN or file containing a List of LFNs'])) - -Script.parseCommandLine(ignoreErrors=False) - -args = Script.getPositionalArgs() -if len(args) < 2: - Script.showHelp() - -targetSE = args.pop(0) - -lfns = [] -for inputFileName in args: - if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - inputFile.close() - lfns.extend([lfn.strip() for lfn in string.splitlines()]) - else: - lfns.append(inputFileName) - -from DIRAC.Resources.Storage.StorageElement import StorageElement -import DIRAC -# Check is provided SE is OK -if targetSE != 'All': - se = StorageElement(targetSE) - if not se.valid: - print(se.errorReason) - print() - Script.showHelp() - -from DIRAC.RequestManagementSystem.Client.Request import Request -from DIRAC.RequestManagementSystem.Client.Operation import Operation -from DIRAC.RequestManagementSystem.Client.File import File -from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient -from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog - -reqClient = ReqClient() -fc = FileCatalog() - -requestOperation = 'RemoveReplica' -if targetSE == 'All': - requestOperation = 'RemoveFile' - -for lfnList in breakListIntoChunks(lfns, 100): - - oRequest = Request() - requestName = "%s_%s" % ( - md5(repr(time.time()).encode()).hexdigest()[:16], - md5(repr(time.time()).encode()).hexdigest()[:16], - ) - oRequest.RequestName = requestName - - oOperation = Operation() - oOperation.Type = requestOperation - oOperation.TargetSE = targetSE - - res = fc.getFileMetadata(lfnList) - if not res['OK']: - print("Can't get file metadata: %s" % res['Message']) - DIRAC.exit(1) - if res['Value']['Failed']: - print("Could not get the file metadata of the following, so skipping them:") - for fFile in res['Value']['Failed']: - print(fFile) - - lfnMetadata = res['Value']['Successful'] - - for lfn in lfnMetadata: - rarFile = File() - rarFile.LFN = lfn - rarFile.Size = lfnMetadata[lfn]['Size'] - rarFile.Checksum = lfnMetadata[lfn]['Checksum'] - rarFile.GUID = lfnMetadata[lfn]['GUID'] - rarFile.ChecksumType = 'ADLER32' - oOperation.addFile(rarFile) - - oRequest.addOperation(oOperation) - - isValid = RequestValidator().validate(oRequest) - if not isValid['OK']: - print("Request is not valid: ", isValid['Message']) - DIRAC.exit(1) - - result = reqClient.putRequest(oRequest) - if result['OK']: - print('Request %d Submitted' % result['Value']) - else: - print('Failed to submit Request: ', result['Message']) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-data-size.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-data-size.py deleted file mode 100755 index 92c3fca7507..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-data-size.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# $HeadURL$ -######################################################################## -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -import sys -import os -import DIRAC -from DIRAC import gLogger -from DIRAC.Core.Base import Script - -unit = 'GB' -Script.registerSwitch("u:", "Unit=", " Unit to use [default %s] (MB,GB,TB,PB)" % unit) - -Script.setUsageMessage(""" -Get the size of the given file or a list of files - -Usage: - %s -""" % Script.scriptName) - -Script.parseCommandLine(ignoreErrors=False) -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "u" or switch[0].lower() == "unit": - unit = switch[1] -scaleDict = {'MB': 1000 * 1000.0, - 'GB': 1000 * 1000 * 1000.0, - 'TB': 1000 * 1000 * 1000 * 1000.0, - 'PB': 1000 * 1000 * 1000 * 1000 * 1000.0} -if unit not in scaleDict.keys(): - gLogger.error("Unit must be one of MB,GB,TB,PB") - DIRAC.exit(2) -scaleFactor = scaleDict[unit] - -args = Script.getPositionalArgs() -lfns = [] -for inputFileName in args: - if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - inputFile.close() - lfns.extend([lfn.strip() for lfn in string.splitlines()]) - else: - lfns.append(inputFileName) - -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -res = FileCatalog().getFileSize(lfns) -if not res['OK']: - gLogger.error("Failed to get size of data", res['Message']) - DIRAC.exit(-2) -for lfn, reason in res['Value']['Failed'].items(): - gLogger.error("Failed to get size for %s" % lfn, reason) -totalSize = 0 -totalFiles = 0 -for lfn, size in res['Value']['Successful'].items(): - totalFiles += 1 - totalSize += size -gLogger.notice('-' * 30) -gLogger.notice('%s|%s' % ('Files'.ljust(15), ('Size (%s)' % unit).rjust(15))) -gLogger.notice('-' * 30) -gLogger.notice('%s|%s' % (str(totalFiles).ljust(15), str('%.1f' % (totalSize / scaleFactor)).rjust(15))) -gLogger.notice('-' * 30) -DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-directory-sync.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-directory-sync.py deleted file mode 100755 index 6d1e283f1fe..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-directory-sync.py +++ /dev/null @@ -1,579 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# :file: dirac-dms-directory-sync -# :author: Marko Petric -######################################################################## -""" -Provides basic rsync functionality for DIRAC - -Syncs the source destination folder recursivly into the target destination - -If option --sync is used contend that is not in the source directory but is -only in the target directory will be deleted. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from past.builtins import long -import os -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s Source Destination' % Script.scriptName, - ' ', - ' e.g.: Download', - ' %s LFN Path' % Script.scriptName, - ' or Upload', - ' %s Path LFN SE' % Script.scriptName, - 'Arguments:', - ' LFN: Logical File Name (Path to directory)', - ' Path: Local path to the file (Path to directory)', - ' SE: DIRAC Storage Element'] - ) - ) - -Script.registerSwitch("D", "sync", "Make target directory identical to source") -Script.registerSwitch("j:", "parallel=", "Multithreaded download and upload") -Script.parseCommandLine(ignoreErrors=False) - -args = Script.getPositionalArgs() -if len(args) < 1 or len(args) > 3: - Script.showHelp() - -sync = False -parallel = 1 -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "s" or switch[0].lower() == "sync": - sync = True - if switch[0].lower() == "j" or switch[0].lower() == "parallel": - parallel = int(switch[1]) - - -from DIRAC import S_OK, S_ERROR -from DIRAC import gConfig, gLogger -from DIRAC.Core.Utilities.List import breakListIntoChunks -from DIRAC.Core.Utilities.ReturnValues import returnSingleResult -from DIRAC.Core.Utilities.File import mkDir -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -from DIRAC.DataManagementSystem.Client.DataManager import DataManager -from DIRAC.Resources.Storage.StorageElement import StorageElement - -__RCSID__ = "$Id$" - - -def getSetOfLocalDirectoriesAndFiles(path): - """Return a set of all directories and subdirectories and a set of - files contained therein for a given local path - """ - - fullPath = os.path.abspath(path) - - if not os.path.isdir(fullPath): - return S_ERROR("The path: " + fullPath + " does not exist!") - - directories = set() - files = set() - - for dirname, dirnames, filenames in os.walk(path): - # add path to all subdirectories first. - for subdirname in dirnames: - fullSubdirname = os.path.join(dirname, subdirname) - fullSubdirname = os.path.abspath(fullSubdirname) - fullSubdirname = fullSubdirname.replace(fullPath, '').lstrip('/') - directories.add(fullSubdirname) - # add path to all filenames. - for filename in filenames: - fullFilename = os.path.join(dirname, filename) - fullFilename = os.path.abspath(fullFilename) - fullFilename = fullFilename.replace(fullPath, '').lstrip('/') - fileSize = os.path.getsize(fullPath + "/" + fullFilename) - if fileSize > 0: - files.add((fullFilename, long(fileSize))) - - tree = {} - tree["Directories"] = directories - tree["Files"] = files - - return S_OK(tree) - - -def getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories, files): - """ - Recursively traverses all the subdirectories of a directory and returns a set of directories and files - """ - result = fc.listDirectory(path) - if result['OK']: - if result['Value']['Successful']: - for entry in result['Value']['Successful'][path]['Files']: - size = result['Value']['Successful'][path]['Files'][entry]['MetaData']['Size'] - files.add((entry, size)) - for entry in result['Value']['Successful'][path]['SubDirs']: - directories.add(entry) - res = getSetOfRemoteSubDirectoriesAndFiles(entry, fc, directories, files) - if not res['OK']: - return S_ERROR('Error: ' + res['Message']) - return S_OK() - else: - return S_ERROR("Error: %s" % result['Value']) - else: - return S_ERROR("Error:" + result['Message']) - - -def getSetOfRemoteDirectoriesAndFiles(fc, path): - """ - Return a set of all directories and subdirectories and the therein contained files for a given LFN - """ - directories = set() - files = set() - - res = getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories, files) - if not res['OK']: - return S_ERROR('Could not list remote directory: ' + res['Message']) - - return_directories = set() - return_files = set() - - for myfile in files: - return_files.add((myfile[0].replace(path, '').lstrip('/'), myfile[1])) - - for mydirectory in directories: - return_directories.add(mydirectory.replace(path, '').lstrip('/')) - - tree = {} - tree["Directories"] = return_directories - tree["Files"] = return_files - - return S_OK(tree) - - -def isInFileCatalog(fc, path): - """ - Check if the file is in the File Catalog - """ - - result = fc.listDirectory(path) - if result['OK']: - if result['Value']['Successful']: - return S_OK() - else: - return S_ERROR() - else: - return S_ERROR() - - -def getContentToSync(upload, fc, source_dir, dest_dir): - """ - Return list of files and directories to be create and deleted - """ - - if upload: - res = getSetOfRemoteDirectoriesAndFiles(fc, dest_dir) - if not res['OK']: - return S_ERROR(res['Message']) - to_dirs = res['Value']['Directories'] - to_files = res['Value']['Files'] - - res = getSetOfLocalDirectoriesAndFiles(source_dir) - if not res['OK']: - return S_ERROR(res['Message']) - from_dirs = res['Value']['Directories'] - from_files = res['Value']['Files'] - - else: - res = getSetOfLocalDirectoriesAndFiles(dest_dir) - if not res['OK']: - return S_ERROR(res['Message']) - to_dirs = res['Value']['Directories'] - to_files = res['Value']['Files'] - - res = getSetOfRemoteDirectoriesAndFiles(fc, source_dir) - if not res['OK']: - return S_ERROR(res['Message']) - from_dirs = res['Value']['Directories'] - from_files = res['Value']['Files'] - - # Create list of directories to delete - dirs_delete = list(to_dirs - from_dirs) - # Sort the list by depth of directory tree - dirs_delete.sort(key=lambda s: -s.count('/')) - # Create list of directories to create - dirs_create = list(from_dirs - to_dirs) - # Sort the list by depth of directory tree - dirs_create.sort(key=lambda s: s.count('/')) - - # Flatten the list of pairs (filename, size) to list of filename - files_delete = [pair[0] for pair in list(to_files - from_files)] - files_create = [pair[0] for pair in list(from_files - to_files)] - - create = {} - create["Directories"] = dirs_create - create["Files"] = files_create - - delete = {} - delete["Directories"] = dirs_delete - delete["Files"] = files_delete - - tree = {} - tree["Create"] = create - tree["Delete"] = delete - - return S_OK(tree) - - -def removeRemoteFiles(dm, lfns): - """ - Remove file from the catalog - """ - for lfnList in breakListIntoChunks(lfns, 100): - res = dm.removeFile(lfnList) - if not res['OK']: - return S_ERROR("Failed to remove files:" + lfnList + res['Message']) - else: - return S_OK() - - -def uploadLocalFile(dm, lfn, localfile, storage): - """ - Upload a local file to a storage element - """ - res = dm.putAndRegister(lfn, localfile, storage, None) - if not res['OK']: - return S_ERROR('Error: failed to upload %s to %s' % (lfn, storage)) - else: - return S_OK('Successfully uploaded file to %s' % storage) - - -def downloadRemoteFile(dm, lfn, destination): - """ - Download a file from the system - """ - res = dm.getFile(lfn, destination) - if not res['OK']: - return S_ERROR('Error: failed to download %s ' % lfn) - else: - return S_OK('Successfully uploaded file %s' % lfn) - - -def removeStorageDirectoryFromSE(directory, storageElement): - """ - Delete directory on selected storage element - """ - - se = StorageElement(storageElement, False) - res = returnSingleResult(se.exists(directory)) - - if not res['OK']: - return S_ERROR("Failed to obtain existence of directory" + res['Message']) - - exists = res['Value'] - if not exists: - return S_OK("The directory %s does not exist at %s " % (directory, storageElement)) - - res = returnSingleResult(se.removeDirectory(directory, recursive=True)) - if not res['OK']: - return S_ERROR("Failed to remove storage directory" + res['Message']) - - return S_OK() - - -def removeRemoteDirectory(fc, lfn): - """ - Remove file from the catalog - """ - storageElements = gConfig.getValue('Resources/StorageElementGroups/SE_Cleaning_List', []) - - for storageElement in sorted(storageElements): - res = removeStorageDirectoryFromSE(lfn, storageElement) - if not res['OK']: - return S_ERROR("Failed to clean storage directory at all SE:" + res['Message']) - res = returnSingleResult(fc.removeDirectory(lfn, recursive=True)) - if not res['OK']: - return S_ERROR("Failed to clean storage directory at all SE:" + res['Message']) - - return S_OK("Successfully removed directory") - - -def createRemoteDirectory(fc, newdir): - """ - Create directory in file catalog - """ - result = fc.createDirectory(newdir) - if result['OK']: - if result['Value']['Successful'] and newdir in result['Value']['Successful']: - return S_OK("Successfully created directory:" + newdir) - elif result['Value']['Failed'] and newdir in result['Value']['Failed']: - return S_ERROR('Failed to create directory: ' + result['Value']['Failed'][newdir]) - else: - return S_ERROR('Failed to create directory:' + result['Message']) - - -def createLocalDirectory(directory): - """ - Create local directory - """ - mkDir(directory) - if not os.path.exists(directory): - return S_ERROR('Directory creation failed') - return S_OK('Created directory successfully') - - -def removeLocalFile(path): - """ - Remove local file - """ - try: - os.remove(path) - except OSError as e: - return S_ERROR('Directory creation failed:' + e.strerror) - - if os.path.isfile(path): - return S_ERROR('File deleting failed') - return S_OK('Removed file successfully') - - -def removeLocaDirectory(path): - """ - Remove local directory - """ - try: - os.rmdir(path) - except OSError as e: - return S_ERROR('Deleting directory failed: ' + e.strerror) - - if os.path.isdir(path): - return S_ERROR('Directory deleting failed') - return S_OK('Removed directory successfully') - - -def doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads): - """ - Wrapper for uploading files - """ - if delete: - lfns = [dest_dir + "/" + filename for filename in result['Value']['Delete']['Files']] - if len(lfns) > 0: - res = removeRemoteFiles(dm, lfns) - if not res['OK']: - gLogger.fatal('Deleting of files: ' + lfns + " -X- [FAILED]" + res['Message']) - DIRAC.exit(1) - else: - gLogger.notice("Deleting " + ', '.join(lfns) + " -> [DONE]") - - for directoryname in result['Value']['Delete']['Directories']: - res = removeRemoteDirectory(fc, dest_dir + "/" + directoryname) - if not res['OK']: - gLogger.fatal('Deleting of directory: ' + directoryname + " -X- [FAILED] " + res['Message']) - DIRAC.exit(1) - else: - gLogger.notice("Deleting " + directoryname + " -> [DONE]") - - for directoryname in result['Value']['Create']['Directories']: - res = createRemoteDirectory(fc, dest_dir + "/" + directoryname) - if not res['OK']: - gLogger.fatal('Creation of directory: ' + directoryname + " -X- [FAILED] " + res['Message']) - DIRAC.exit(1) - else: - gLogger.notice("Creating " + directoryname + " -> [DONE]") - - listOfFiles = result['Value']['Create']['Files'] - # Chech that we do not have to many threads - if nthreads > len(listOfFiles): - nthreads = len(listOfFiles) - - if nthreads == 0: - return S_OK('Upload finished successfully') - - listOfListOfFiles = chunkList(listOfFiles, nthreads) - res = runInParallel( - arguments=[ - dm, - source_dir, - dest_dir, - storage], - listOfLists=listOfListOfFiles, - function=uploadListOfFiles) - if not res['OK']: - return S_ERROR("Upload of files failed") - - return S_OK('Upload finished successfully') - - -def uploadListOfFiles(dm, source_dir, dest_dir, storage, listOfFiles, tID): - """ - Wrapper for multithreaded uploading of a list of files - """ - log = gLogger.getSubLogger("[Thread %s] " % tID) - threadLine = "[Thread %s]" % tID - for filename in listOfFiles: - res = uploadLocalFile(dm, dest_dir + "/" + filename, source_dir + "/" + filename, storage) - if not res['OK']: - log.fatal(threadLine + ' Uploading ' + filename + ' -X- [FAILED] ' + res['Message']) - DIRAC.exit(1) - else: - log.notice(threadLine + " Uploading " + filename + " -> [DONE]") - - -def doDownload(dm, result, source_dir, dest_dir, delete, nthreads): - """ - Wrapper for downloading files - """ - if delete: - for filename in result['Value']['Delete']['Files']: - res = removeLocalFile(dest_dir + "/" + filename) - if not res['OK']: - gLogger.fatal('Deleting of file: ' + filename + ' -X- [FAILED] ' + res['Message']) - DIRAC.exit(1) - else: - gLogger.notice("Deleting " + filename + " -> [DONE]") - - for directoryname in result['Value']['Delete']['Directories']: - res = removeLocaDirectory(dest_dir + "/" + directoryname) - if not res['OK']: - gLogger.fatal('Deleting of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message']) - DIRAC.exit(1) - else: - gLogger.notice("Deleting " + directoryname + " -> [DONE]") - - for directoryname in result['Value']['Create']['Directories']: - res = createLocalDirectory(dest_dir + "/" + directoryname) - if not res['OK']: - gLogger.fatal('Creation of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message']) - DIRAC.exit(1) - else: - gLogger.notice("Creating " + directoryname + " -> [DONE]") - - listOfFiles = result['Value']['Create']['Files'] - # Chech that we do not have to many threads - if nthreads > len(listOfFiles): - nthreads = len(listOfFiles) - - if nthreads == 0: - return S_OK('Upload finished successfully') - - listOfListOfFiles = chunkList(listOfFiles, nthreads) - res = runInParallel(arguments=[dm, source_dir, dest_dir], listOfLists=listOfListOfFiles, function=downloadListOfFiles) - - if not res['OK']: - return S_ERROR("Download of files failed") - - return S_OK('Upload finished successfully') - - -def chunkList(alist, nchunks): - """ - Split a list into a list of equaliy sized lists - """ - avg = len(alist) / float(nchunks) - out = [] - last = 0.0 - - while last < len(alist): - out.append(alist[int(last):int(last + avg)]) - last += avg - - return out - - -def downloadListOfFiles(dm, source_dir, dest_dir, listOfFiles, tID): - """ - Wrapper for multithreaded downloading of a list of files - """ - log = gLogger.getSubLogger("[Thread %s] " % tID) - threadLine = "[Thread %s]" % tID - for filename in listOfFiles: - res = downloadRemoteFile(dm, source_dir + "/" + filename, dest_dir + ("/" + filename).rsplit("/", 1)[0]) - if not res['OK']: - log.fatal(threadLine + ' Downloading ' + filename + ' -X- [FAILED] ' + res['Message']) - DIRAC.exit(1) - else: - log.notice(threadLine + " Downloading " + filename + " -> [DONE]") - - -def runInParallel(arguments, listOfLists, function): - """ - Helper for execution of uploads and downloads in parallel - """ - from multiprocessing import Process - processes = [] - for tID, alist in enumerate(listOfLists): - argums = arguments + [alist] + [tID] - pro = Process(target=function, args=argums) - pro.start() - processes.append(pro) - for process in processes: - process.join() - - for process in processes: - if process.exitcode == 1: - return S_ERROR() - return S_OK() - - -def syncDestinations(upload, source_dir, dest_dir, storage, delete, nthreads): - """ - Top level wrapper to execute functions - """ - - fc = FileCatalog() - dm = DataManager() - - result = getContentToSync(upload, fc, source_dir, dest_dir) - if not result['OK']: - return S_ERROR(result['Message']) - - if upload: - res = doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads) - if not res['OK']: - return S_ERROR('Upload failed: ' + res['Message']) - else: - res = doDownload(dm, result, source_dir, dest_dir, delete, nthreads) - if not res['OK']: - return S_ERROR('Download failed: ' + res['Message']) - - return S_OK('Mirroring successfully finished') - - -def run(parameters, delete, nthreads): - """ - The main user interface - """ - - source_dir = parameters[0] - dest_dir = parameters[1] - upload = False - storage = None - - if len(parameters) == 3: - storage = parameters[2] - source_dir = os.path.abspath(source_dir) - dest_dir = dest_dir.rstrip('/') - upload = True - if not os.path.isdir(source_dir): - gLogger.fatal("Source directory does not exist") - DIRAC.exit(1) - - if len(parameters) == 2: - dest_dir = os.path.abspath(dest_dir) - source_dir = source_dir.rstrip('/') - if not os.path.isdir(dest_dir): - gLogger.fatal("Destination directory does not exist") - DIRAC.exit(1) - - res = syncDestinations(upload, source_dir, dest_dir, storage, delete, nthreads) - if not res['OK']: - return S_ERROR(res['Message']) - - return S_OK("Successfully mirrored " + source_dir + " into " + dest_dir) - - -if __name__ == "__main__": - returnValue = run(args, sync, parallel) - if not returnValue['OK']: - gLogger.fatal(returnValue['Message']) - DIRAC.exit(1) - else: - gLogger.notice(returnValue['Value']) - DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-filecatalog-cli.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-filecatalog-cli.py deleted file mode 100755 index cd5bd7368b3..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-filecatalog-cli.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -import sys - -from DIRAC.Core.Base import Script -Script.setUsageMessage(""" -Launch the File Catalog shell - -Usage: - %s [option] -""" % Script.scriptName) - -fcType = 'FileCatalog' -Script.registerSwitch("f:", "file-catalog=", " Catalog client type to use (default %s)" % fcType) -Script.parseCommandLine(ignoreErrors=False) - -from DIRAC import gConfig, exit as dexit -from DIRAC.Resources.Catalog.FileCatalogFactory import FileCatalogFactory - -__RCSID__ = "$Id$" - -fcType = gConfig.getValue("/LocalSite/FileCatalog", "") - -res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True) -if not res['OK']: - dexit(1) -fcList = res['Value'] -if not fcType: - if res['OK']: - fcType = res['Value'][0] - -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "f" or switch[0].lower() == "file-catalog": - fcType = switch[1] - -if not fcType: - print("No file catalog given and defaults could not be obtained") - sys.exit(1) - -from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI - -result = FileCatalogFactory().createCatalog(fcType) -if not result['OK']: - print(result['Message']) - if fcList: - print("Possible choices are:") - for fc in fcList: - print(' ' * 5, fc) - sys.exit(1) -print("Starting %s client" % fcType) -catalog = result['Value'] -cli = FileCatalogClientCLI(catalog) -cli.cmdloop() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-catalog-files.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-catalog-files.py deleted file mode 100755 index 97454be243b..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-catalog-files.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# $Header: $ -######################################################################## -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script -from DIRAC import exit as dexit -from DIRAC import gLogger -Script.setUsageMessage(""" -Remove the given file or a list of files from the File Catalog - -Usage: - %s -""" % Script.scriptName) - -Script.parseCommandLine() - -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -allowUsers = Operations().getValue("DataManagement/AllowUserReplicaManagement", False) - -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -res = getProxyInfo() -if not res['OK']: - gLogger.fatal("Can't get proxy info", res['Message']) - dexit(1) -properties = res['Value'].get('groupProperties', []) - -if not allowUsers: - if 'FileCatalogManagement' not in properties: - gLogger.error("You need to use a proxy from a group with FileCatalogManagement") - dexit(5) - -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -fc = FileCatalog() -import os - -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) -else: - inputFileName = args[0] - -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - lfns = [lfn.strip() for lfn in string.splitlines()] - inputFile.close() -else: - lfns = [inputFileName] - -res = fc.removeFile(lfns) -if not res['OK']: - print("Error:", res['Message']) - dexit(1) -for lfn in sorted(res['Value']['Failed'].keys()): - message = res['Value']['Failed'][lfn] - print('Error: failed to remove %s: %s' % (lfn, message)) -print('Successfully removed %d catalog files.' % (len(res['Value']['Successful']))) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-catalog-replicas.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-catalog-replicas.py deleted file mode 100755 index 19b250f9241..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-catalog-replicas.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import os - -from DIRAC import exit as dexit -from DIRAC.Core.Base import Script -from DIRAC import gLogger - -Script.setUsageMessage(""" -Remove the given file replica or a list of file replicas from the File Catalog -This script should be used with great care as it may leave dark data in the storage! -Use dirac-dms-remove-replicas instead - -Usage: - %s -""" % Script.scriptName) - -Script.parseCommandLine() - -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -allowUsers = Operations().getValue("DataManagement/AllowUserReplicaManagement", False) - -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -res = getProxyInfo() -if not res['OK']: - gLogger.fatal("Can't get proxy info", res['Message']) - dexit(1) -properties = res['Value'].get('groupProperties', []) - -if not allowUsers: - if 'FileCatalogManagement' not in properties: - gLogger.error("You need to use a proxy from a group with FileCatalogManagement") - dexit(5) - -from DIRAC.DataManagementSystem.Client.DataManager import DataManager -dm = DataManager() -args = Script.getPositionalArgs() -if len(args) < 2: - Script.showHelp(exitCode=1) -else: - inputFileName = args[0] - storageElementName = args[1] - -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - lfns = [lfn.strip() for lfn in string.splitlines()] - inputFile.close() -else: - lfns = [inputFileName] - -res = dm.removeReplicaFromCatalog(storageElementName, lfns) -if not res['OK']: - print(res['Message']) - dexit(0) -for lfn in sorted(res['Value']['Failed']): - message = res['Value']['Failed'][lfn] - print('Failed to remove %s replica of %s: %s' % (storageElementName, lfn, message)) -print('Successfully remove %d catalog replicas at %s' % (len(res['Value']['Successful']), storageElementName)) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-files.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-files.py deleted file mode 100755 index c0edd1a9aa6..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-files.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# $HeadURL$ -######################################################################## -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script - -Script.setUsageMessage(""" -Remove the given file or a list of files from the File Catalog and from the storage - -Usage: - %s -""" % Script.scriptName) - -Script.parseCommandLine() - -import sys -import os -import DIRAC -from DIRAC import gLogger - -args = Script.getPositionalArgs() -lfns = [] -for inputFileName in args: - if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - inputFile.close() - lfns.extend([lfn.strip() for lfn in string.splitlines()]) - else: - lfns.append(inputFileName) - -from DIRAC.Core.Utilities.List import breakListIntoChunks -from DIRAC.DataManagementSystem.Client.DataManager import DataManager -dm = DataManager() - -errorReasons = {} -successfullyRemoved = 0 -for lfnList in breakListIntoChunks(lfns, 100): - res = dm.removeFile(lfnList) - if not res['OK']: - gLogger.error("Failed to remove data", res['Message']) - DIRAC.exit(-2) - for lfn, r in res['Value']['Failed'].items(): - reason = str(r) - if reason not in errorReasons: - errorReasons[reason] = [] - errorReasons[reason].append(lfn) - successfullyRemoved += len(res['Value']['Successful']) - -for reason, lfns in errorReasons.items(): - gLogger.notice("Failed to remove %d files with error: %s" % (len(lfns), reason)) -if successfullyRemoved > 0: - gLogger.notice("Successfully removed %d files" % successfullyRemoved) -DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-replica-metadata.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-replica-metadata.py deleted file mode 100755 index b8d7db387ea..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-replica-metadata.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import os - -from DIRAC import exit as DIRACExit -from DIRAC.Core.Base import Script - -Script.setUsageMessage(""" -Get the given file replica metadata from the File Catalog - -Usage: - %s SE -""" % Script.scriptName) - -Script.parseCommandLine() - -from DIRAC import gLogger -from DIRAC.DataManagementSystem.Client.DataManager import DataManager - -args = Script.getPositionalArgs() -if not len(args) == 2: - Script.showHelp(exitCode=1) -else: - inputFileName = args[0] - storageElement = args[1] - -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - lfns = [lfn.strip() for lfn in string.splitlines()] - inputFile.close() -else: - lfns = [inputFileName] - -res = DataManager().getReplicaMetadata(lfns, storageElement) -if not res['OK']: - print('Error:', res['Message']) - DIRACExit(1) - -print('%s %s %s %s' % ('File'.ljust(100), 'Migrated'.ljust(8), 'Cached'.ljust(8), 'Size (bytes)'.ljust(10))) -for lfn, metadata in res['Value']['Successful'].items(): - print( - '%s %s %s %s' % - (lfn.ljust(100), str( - metadata['Migrated']).ljust(8), str( - metadata.get( - 'Cached', metadata['Accessible'])).ljust(8), str( - metadata['Size']).ljust(10))) -for lfn, reason in res['Value']['Failed'].items(): - print('%s %s' % (lfn.ljust(100), reason.ljust(8))) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-resolve-guid.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-resolve-guid.py deleted file mode 100755 index 9254f4fbb79..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-resolve-guid.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script - -Script.setUsageMessage(""" -Returns the LFN matching given GUIDs -Usage: - %s -""" % Script.scriptName) - -Script.parseCommandLine() - -import DIRAC -from DIRAC import gLogger - -args = Script.getPositionalArgs() -if len(args) != 1: - Script.showHelp() -guids = args[0] - -try: - guids = guids.split(',') -except BaseException: - pass - -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog - -fc = FileCatalog() -res = fc.getLFNForGUID(guids) -if not res['OK']: - gLogger.error("Failed to get the LFNs", res['Message']) - DIRAC.exit(-2) - -errorGuid = {} -for guid, reason in res['Value']['Failed'].items(): - errorGuid.setdefault(reason, []).append(guid) - -for error, guidList in errorGuid.items(): - gLogger.notice("Error '%s' for guids %s" % (error, guidList)) - -for guid, lfn in res['Value']['Successful'].items(): - gLogger.notice("%s -> %s" % (guid, lfn)) - -DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-set-replica-status.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-set-replica-status.py deleted file mode 100755 index 9bc69abe39c..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-set-replica-status.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# $HeadURL$ -######################################################################## -""" -Set the status of the replicas of given files at the provided SE -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - '\nUsage:', - ' %s [option|cfgfile] ... SE Status' % Script.scriptName, - 'Arguments:', - ' LFN: LFN', - ' File: File name containing a list of affected LFNs', - ' SE: Name of Storage Element', - ' Status: New Status for the replica'])) - -Script.parseCommandLine(ignoreErrors=False) - -import DIRAC -from DIRAC import gConfig, gLogger -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -import os - -args = Script.getPositionalArgs() -if not len(args) == 3: - Script.showHelp() - -inputFileName = args[0] -storageElement = args[1] -status = args[2] - -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - inputFile.close() - lfns = sorted(string.splitlines()) -else: - lfns = [inputFileName] - -fc = FileCatalog() - -replicaDict = {} -res = fc.getReplicas(lfns, allStatus=True) -if not res['OK']: - gLogger.error("Failed to get catalog replicas.", res['Message']) - DIRAC.exit(-1) -lfnDict = {} -for lfn, error in res['Value']['Failed'].items(): - gLogger.error("Failed to get replicas for file.", "%s:%s" % (lfn, error)) -for lfn, replicas in res['Value']['Successful'].items(): - if storageElement not in replicas.keys(): - gLogger.error("LFN not registered at provided storage element.", "%s %s" % (lfn, storageElement)) - else: - lfnDict[lfn] = {'SE': storageElement, 'PFN': replicas[storageElement], 'Status': status} -if not lfnDict: - gLogger.error("No files found at the supplied storage element.") - DIRAC.exit(2) - -res = fc.setReplicaStatus(lfnDict) -if not res['OK']: - gLogger.error("Failed to set catalog replica status.", res['Message']) - DIRAC.exit(-1) -for lfn, error in res['Value']['Failed'].items(): - gLogger.error("Failed to set replica status for file.", "%s:%s" % (lfn, error)) -gLogger.notice("Successfully updated the status of %d files at %s." % - (len(res['Value']['Successful'].keys()), storageElement)) -DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-show-se-status.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-show-se-status.py deleted file mode 100755 index cb91c953f33..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-show-se-status.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -from DIRAC import S_OK, exit as DIRACexit -from DIRAC.Core.Base import Script - -__RCSID__ = "$Id$" - -Script.setUsageMessage(""" -Get status of the available Storage Elements - -Usage: - %s [] -""" % Script.scriptName) - -vo = None - - -def setVO(arg): - global vo - vo = arg - return S_OK() - - -allVOsFlag = False - - -def setAllVO(arg): - global allVOsFlag - allVOsFlag = True - return S_OK() - - -noVOFlag = False - - -def setNoVO(arg): - global noVOFlag, allVOsFlag - noVOFlag = True - allVOsFlag = False - return S_OK() - - -Script.registerSwitch("V:", "vo=", "Virtual Organization", setVO) -Script.registerSwitch("a", "all", "All Virtual Organizations flag", setAllVO) -Script.registerSwitch("n", "noVO", "No Virtual Organizations assigned flag", setNoVO) - -Script.parseCommandLine() - -from DIRAC import gConfig, gLogger -from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus -from DIRAC.Core.Utilities.PrettyPrint import printTable -from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup - -storageCFGBase = "/Resources/StorageElements" - -res = gConfig.getSections(storageCFGBase, True) -if not res['OK']: - gLogger.error('Failed to get storage element info') - gLogger.error(res['Message']) - DIRACexit(1) - -gLogger.info("%s %s %s" % ('Storage Element'.ljust(25), 'Read Status'.rjust(15), 'Write Status'.rjust(15))) - -seList = sorted(res['Value']) - -resourceStatus = ResourceStatus() - -res = resourceStatus.getElementStatus(seList, "StorageElement") -if not res['OK']: - gLogger.error("Failed to get StorageElement status for %s" % str(seList)) - DIRACexit(1) - -fields = ['SE', 'ReadAccess', 'WriteAccess', 'RemoveAccess', 'CheckAccess'] -records = [] - -if vo is None and not allVOsFlag: - result = getVOfromProxyGroup() - if not result['OK']: - gLogger.error('Failed to determine the user VO') - DIRACexit(1) - vo = result['Value'] - -for se, statusDict in res['Value'].items(): - - # Check if the SE is allowed for the user VO - if not allVOsFlag: - voList = gConfig.getValue('/Resources/StorageElements/%s/VO' % se, []) - if noVOFlag and voList: - continue - if voList and vo not in voList: - continue - - record = [se] - for status in fields[1:]: - value = statusDict.get(status, 'Unknown') - record.append(value) - records.append(record) - -printTable(fields, records, numbering=False, sortField='SE') - -DIRACexit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-user-lfns.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-user-lfns.py deleted file mode 100755 index 0aaf28a5a10..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-user-lfns.py +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# $HeadURL$ -######################################################################## -""" -Get the list of all the user files. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script -days = 0 -months = 0 -years = 0 -wildcard = None -baseDir = '' -emptyDirsFlag = False -Script.registerSwitch("D:", "Days=", "Match files older than number of days [%s]" % days) -Script.registerSwitch("M:", "Months=", "Match files older than number of months [%s]" % months) -Script.registerSwitch("Y:", "Years=", "Match files older than number of years [%s]" % years) -Script.registerSwitch("w:", "Wildcard=", "Wildcard for matching filenames [All]") -Script.registerSwitch("b:", "BaseDir=", "Base directory to begin search (default /[vo]/user/[initial]/[username])") -Script.registerSwitch("e", "EmptyDirs", "Create a list of empty directories") - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % Script.scriptName, ])) - -Script.parseCommandLine(ignoreErrors=False) - -for switch in Script.getUnprocessedSwitches(): - if switch[0] == "D" or switch[0].lower() == "days": - days = int(switch[1]) - if switch[0] == "M" or switch[0].lower() == "months": - months = int(switch[1]) - if switch[0] == "Y" or switch[0].lower() == "years": - years = int(switch[1]) - if switch[0].lower() == "w" or switch[0].lower() == "wildcard": - wildcard = '*' + switch[1] - if switch[0].lower() == "b" or switch[0].lower() == "basedir": - baseDir = switch[1] - if switch[0].lower() == "e" or switch[0].lower() == "emptydirs": - emptyDirsFlag = True - -import DIRAC -from DIRAC import gLogger -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -from datetime import datetime, timedelta -import sys -import os -import time -import fnmatch -fc = FileCatalog() - - -def isOlderThan(cTimeStruct, days): - timeDelta = timedelta(days=days) - maxCTime = datetime.utcnow() - timeDelta - if cTimeStruct < maxCTime: - return True - return False - - -withMetadata = False -if days or months or years: - withMetadata = True -totalDays = 0 -if years: - totalDays += 365 * years -if months: - totalDays += 30 * months -if days: - totalDays += days - -res = getProxyInfo(False, False) -if not res['OK']: - gLogger.error("Failed to get client proxy information.", res['Message']) - DIRAC.exit(2) -proxyInfo = res['Value'] -if proxyInfo['secondsLeft'] == 0: - gLogger.error("Proxy expired") - DIRAC.exit(2) -username = proxyInfo['username'] -vo = '' -if 'group' in proxyInfo: - vo = getVOForGroup(proxyInfo['group']) -if not baseDir: - if not vo: - gLogger.error('Could not determine VO') - Script.showHelp() - baseDir = '/%s/user/%s/%s' % (vo, username[0], username) - -baseDir = baseDir.rstrip('/') - -gLogger.notice('Will search for files in %s%s' % (baseDir, (' matching %s' % wildcard) if wildcard else '')) -activeDirs = [baseDir] - -allFiles = [] -emptyDirs = [] - -while len(activeDirs) > 0: - currentDir = activeDirs.pop() - res = fc.listDirectory(currentDir, withMetadata, timeout=360) - if not res['OK']: - gLogger.error("Error retrieving directory contents", "%s %s" % (currentDir, res['Message'])) - elif currentDir in res['Value']['Failed']: - gLogger.error("Error retrieving directory contents", "%s %s" % (currentDir, res['Value']['Failed'][currentDir])) - else: - dirContents = res['Value']['Successful'][currentDir] - subdirs = dirContents['SubDirs'] - files = dirContents['Files'] - if not subdirs and not files: - emptyDirs.append(currentDir) - gLogger.notice('%s: empty directory' % currentDir) - else: - for subdir in sorted(subdirs, reverse=True): - if (not withMetadata) or isOlderThan(subdirs[subdir]['CreationDate'], totalDays): - activeDirs.append(subdir) - for filename in sorted(files): - fileOK = False - if (not withMetadata) or isOlderThan(files[filename]['MetaData']['CreationDate'], totalDays): - if wildcard is None or fnmatch.fnmatch(filename, wildcard): - fileOK = True - if not fileOK: - files.pop(filename) - allFiles += sorted(files) - - if len(files) or len(subdirs): - gLogger.notice("%s: %d files%s, %d sub-directories" % - (currentDir, len(files), ' matching' if withMetadata or wildcard else '', len(subdirs))) - -outputFileName = '%s.lfns' % baseDir.replace('/%s' % vo, '%s' % vo).replace('/', '-') -outputFile = open(outputFileName, 'w') -for lfn in sorted(allFiles): - outputFile.write(lfn + '\n') -outputFile.close() -gLogger.notice('%d matched files have been put in %s' % (len(allFiles), outputFileName)) - -if emptyDirsFlag: - outputFileName = '%s.emptydirs' % baseDir.replace('/%s' % vo, '%s' % vo).replace('/', '-') - outputFile = open(outputFileName, 'w') - for dir in sorted(emptyDirs): - outputFile.write(dir + '\n') - outputFile.close() - gLogger.notice('%d empty directories have been put in %s' % (len(emptyDirs), outputFileName)) - -DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-user-quota.py b/src/DIRAC/DataManagementSystem/scripts/dirac-dms-user-quota.py deleted file mode 100755 index f6efb12f4ca..00000000000 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-user-quota.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script - -Script.setUsageMessage(""" -Get the currently defined user data volume quotas - -Usage: - %s [options] -""" % Script.scriptName) - -Script.parseCommandLine(ignoreErrors=False) - -import DIRAC -from DIRAC import gLogger, gConfig -from DIRAC.Core.Security.ProxyInfo import getProxyInfo - -res = getProxyInfo(False, False) -if not res['OK']: - gLogger.error("Failed to get client proxy information.", res['Message']) - DIRAC.exit(2) -proxyInfo = res['Value'] -username = proxyInfo['username'] - -try: - quota = gConfig.getValue('/Registry/DefaultStorageQuota', 0.) - quota = gConfig.getValue('/Registry/Users/%s/Quota' % username, quota) - gLogger.notice('Current quota found to be %.1f GB' % quota) - DIRAC.exit(0) -except Exception as x: - gLogger.exception("Failed to convert retrieved quota", '', x) - DIRAC.exit(-1) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py new file mode 100755 index 00000000000..a8ce614b4f6 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python +""" Enable using one or more Storage Elements +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + read = False + write = False + check = False + remove = False + site = '' + mute = False + + Script.setUsageMessage(""" + Enable using one or more Storage Elements + + Usage: + %s SE1 [SE2 ...] + """ % Script.scriptName) + + Script.registerSwitch("r", "AllowRead", " Allow only reading from the storage element") + Script.registerSwitch("w", "AllowWrite", " Allow only writing to the storage element") + Script.registerSwitch("k", "AllowCheck", " Allow only check access to the storage element") + Script.registerSwitch("v", "AllowRemove", " Allow only remove access to the storage element") + Script.registerSwitch("a", "All", " Allow all access to the storage element") + Script.registerSwitch("m", "Mute", " Do not send email") + Script.registerSwitch("S:", "Site=", " Allow all SEs associated to site") + + Script.parseCommandLine(ignoreErrors=True) + + ses = Script.getPositionalArgs() + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() in ("r", "allowread"): + read = True + if switch[0].lower() in ("w", "allowwrite"): + write = True + if switch[0].lower() in ("k", "allowcheck"): + check = True + if switch[0].lower() in ("v", "allowremove"): + remove = True + if switch[0].lower() in ("a", "all"): + read = True + write = True + check = True + remove = True + if switch[0].lower() in ("m", "mute"): + mute = True + if switch[0].lower() in ("s", "site"): + site = switch[1] + + # imports + from DIRAC import gConfig, gLogger + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus + + if not (read or write or check or remove): + # No switch was specified, means we need all of them + gLogger.notice("No option given, all accesses will be allowed if they were not") + read = True + write = True + check = True + remove = True + + ses = resolveSEGroup(ses) + diracAdmin = DiracAdmin() + errorList = [] + setup = gConfig.getValue('/DIRAC/Setup', '') + if not setup: + print('ERROR: Could not contact Configuration Service') + DIRAC.exit(2) + + res = getProxyInfo() + if not res['OK']: + gLogger.error('Failed to get proxy information', res['Message']) + DIRAC.exit(2) + + userName = res['Value'].get('username') + if not userName: + gLogger.error('Failed to get username for proxy') + DIRAC.exit(2) + + if site: + res = getSites() + if not res['OK']: + gLogger.error(res['Message']) + DIRAC.exit(-1) + if site not in res['Value']: + gLogger.error('The provided site (%s) is not known.' % site) + DIRAC.exit(-1) + ses.extend(res['Value']['SE'].replace(' ', '').split(',')) + if not ses: + gLogger.error('There were no SEs provided') + DIRAC.exit() + + STATUS_TYPES = ["ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess"] + ALLOWED_STATUSES = ["Unknown", "InActive", "Banned", "Probing", "Degraded"] + + statusAllowedDict = {} + for statusType in STATUS_TYPES: + statusAllowedDict[statusType] = [] + + statusFlagDict = {} + statusFlagDict['ReadAccess'] = read + statusFlagDict['WriteAccess'] = write + statusFlagDict['CheckAccess'] = check + statusFlagDict['RemoveAccess'] = remove + + resourceStatus = ResourceStatus() + + res = resourceStatus.getElementStatus(ses, "StorageElement") + if not res['OK']: + gLogger.error('Storage Element %s does not exist' % ses) + DIRAC.exit(-1) + + reason = 'Forced with dirac-admin-allow-se by %s' % userName + + for se, seOptions in res['Value'].items(): + # InActive is used on the CS model, Banned is the equivalent in RSS + for statusType in STATUS_TYPES: + if statusFlagDict[statusType]: + if seOptions.get(statusType) == "Active": + gLogger.notice('%s status of %s is already Active' % (statusType, se)) + continue + if statusType in seOptions: + if not seOptions[statusType] in ALLOWED_STATUSES: + gLogger.notice('%s option for %s is %s, instead of %s' % + (statusType, se, seOptions['ReadAccess'], ALLOWED_STATUSES)) + gLogger.notice('Try specifying the command switches') + else: + resR = resourceStatus.setElementStatus(se, "StorageElement", statusType, 'Active', reason, userName) + if not resR['OK']: + gLogger.fatal("Failed to update %s %s to Active, exit -" % (se, statusType), resR['Message']) + DIRAC.exit(-1) + else: + gLogger.notice("Successfully updated %s %s to Active" % (se, statusType)) + statusAllowedDict[statusType].append(se) + + totalAllowed = 0 + totalAllowedSEs = [] + for statusType in STATUS_TYPES: + totalAllowed += len(statusAllowedDict[statusType]) + totalAllowedSEs += statusAllowedDict[statusType] + totalAllowedSEs = list(set(totalAllowedSEs)) + + if not totalAllowed: + gLogger.info("No storage elements were allowed") + DIRAC.exit(-1) + + if mute: + gLogger.notice('Email is muted by script switch') + DIRAC.exit(0) + + subject = '%s storage elements allowed for use' % len(totalAllowedSEs) + addressPath = 'EMail/Production' + address = Operations().getValue(addressPath, '') + + body = '' + if read: + body = "%s\n\nThe following storage elements were allowed for reading:" % body + for se in statusAllowedDict['ReadAccess']: + body = "%s\n%s" % (body, se) + if write: + body = "%s\n\nThe following storage elements were allowed for writing:" % body + for se in statusAllowedDict['WriteAccess']: + body = "%s\n%s" % (body, se) + if check: + body = "%s\n\nThe following storage elements were allowed for checking:" % body + for se in statusAllowedDict['CheckAccess']: + body = "%s\n%s" % (body, se) + if remove: + body = "%s\n\nThe following storage elements were allowed for removing:" % body + for se in statusAllowedDict['RemoveAccess']: + body = "%s\n%s" % (body, se) + + if not address: + gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) + DIRAC.exit(0) + + res = diracAdmin.sendMail(address, subject, body) + gLogger.notice('Notifying %s' % address) + if res['OK']: + gLogger.notice(res['Value']) + else: + gLogger.notice(res['Message']) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py new file mode 100755 index 00000000000..5bc1392e1b3 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python + +""" Ban one or more Storage Elements for usage +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + read = True + write = True + check = True + remove = True + site = '' + mute = False + + Script.setUsageMessage(""" + Ban one or more Storage Elements for usage + + Usage: + %s SE1 [SE2 ...] + """ % Script.scriptName) + + Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element") + Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element") + Script.registerSwitch("k", "BanCheck", " Ban check access to the storage element") + Script.registerSwitch("v", "BanRemove", " Ban remove access to the storage element") + Script.registerSwitch("a", "All", " Ban all access to the storage element") + Script.registerSwitch("m", "Mute", " Do not send email") + Script.registerSwitch( + "S:", + "Site=", + " Ban all SEs associate to site (note that if writing is allowed, check is always allowed)") + Script.parseCommandLine(ignoreErrors=True) + + ses = Script.getPositionalArgs() + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() in ("r", "banread"): + write = False + check = False + remove = False + if switch[0].lower() in ("w", "banwrite"): + read = False + check = False + remove = False + if switch[0].lower() in ("k", "bancheck"): + read = False + write = False + remove = False + if switch[0].lower() in ("v", "banremove"): + read = False + write = False + check = False + if switch[0].lower() in ("a", "all"): + pass + if switch[0].lower() in ("m", "mute"): + mute = True + if switch[0].lower() in ("s", "site"): + site = switch[1] + + # from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI + from DIRAC import gConfig, gLogger + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus + from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup + + ses = resolveSEGroup(ses) + diracAdmin = DiracAdmin() + errorList = [] + setup = gConfig.getValue('/DIRAC/Setup', '') + if not setup: + print('ERROR: Could not contact Configuration Service') + DIRAC.exit(2) + + res = getProxyInfo() + if not res['OK']: + gLogger.error('Failed to get proxy information', res['Message']) + DIRAC.exit(2) + + userName = res['Value'].get('username') + if not userName: + gLogger.error('Failed to get username for proxy') + DIRAC.exit(2) + + if site: + res = getSites() + if not res['OK']: + gLogger.error(res['Message']) + DIRAC.exit(-1) + if site not in res['Value']: + gLogger.error('The provided site (%s) is not known.' % site) + DIRAC.exit(-1) + ses.extend(res['Value']['SE'].replace(' ', '').split(',')) + + if not ses: + gLogger.error('There were no SEs provided') + DIRAC.exit(-1) + + readBanned = [] + writeBanned = [] + checkBanned = [] + removeBanned = [] + + resourceStatus = ResourceStatus() + + res = resourceStatus.getElementStatus(ses, "StorageElement") + if not res['OK']: + gLogger.error("Storage Element %s does not exist" % ses) + DIRAC.exit(-1) + + reason = 'Forced with dirac-admin-ban-se by %s' % userName + + for se, seOptions in res['Value'].items(): + + resW = resC = resR = {'OK': False} + + # Eventually, we will get rid of the notion of InActive, as we always write Banned. + if read and 'ReadAccess' in seOptions: + + if not seOptions['ReadAccess'] in ['Active', 'Degraded', 'Probing']: + gLogger.notice('Read option for %s is %s, instead of %s' % + (se, seOptions['ReadAccess'], ['Active', 'Degraded', 'Probing'])) + gLogger.notice('Try specifying the command switches') + else: + + resR = resourceStatus.setElementStatus(se, 'StorageElement', 'ReadAccess', 'Banned', reason, userName) + # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) + if not resR['OK']: + gLogger.error('Failed to update %s read access to Banned' % se) + else: + gLogger.notice('Successfully updated %s read access to Banned' % se) + readBanned.append(se) + + # Eventually, we will get rid of the notion of InActive, as we always write Banned. + if write and 'WriteAccess' in seOptions: + + if not seOptions['WriteAccess'] in ['Active', 'Degraded', 'Probing']: + gLogger.notice('Write option for %s is %s, instead of %s' % + (se, seOptions['WriteAccess'], ['Active', 'Degraded', 'Probing'])) + gLogger.notice('Try specifying the command switches') + else: + + resW = resourceStatus.setElementStatus(se, 'StorageElement', 'WriteAccess', 'Banned', reason, userName) + # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) + if not resW['OK']: + gLogger.error("Failed to update %s write access to Banned" % se) + else: + gLogger.notice("Successfully updated %s write access to Banned" % se) + writeBanned.append(se) + + # Eventually, we will get rid of the notion of InActive, as we always write Banned. + if check and 'CheckAccess' in seOptions: + + if not seOptions['CheckAccess'] in ['Active', 'Degraded', 'Probing']: + gLogger.notice('Check option for %s is %s, instead of %s' % + (se, seOptions['CheckAccess'], ['Active', 'Degraded', 'Probing'])) + gLogger.notice('Try specifying the command switches') + else: + + resC = resourceStatus.setElementStatus(se, 'StorageElement', 'CheckAccess', 'Banned', reason, userName) + # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) + if not resC['OK']: + gLogger.error("Failed to update %s check access to Banned" % se) + else: + gLogger.notice("Successfully updated %s check access to Banned" % se) + checkBanned.append(se) + + # Eventually, we will get rid of the notion of InActive, as we always write Banned. + if remove and 'RemoveAccess' in seOptions: + + if not seOptions['RemoveAccess'] in ['Active', 'Degraded', 'Probing']: + gLogger.notice('Remove option for %s is %s, instead of %s' % + (se, seOptions['RemoveAccess'], ['Active', 'Degraded', 'Probing'])) + gLogger.notice('Try specifying the command switches') + else: + + resC = resourceStatus.setElementStatus(se, 'StorageElement', 'RemoveAccess', 'Banned', reason, userName) + # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) + if not resC['OK']: + gLogger.error("Failed to update %s remove access to Banned" % se) + else: + gLogger.notice("Successfully updated %s remove access to Banned" % se) + removeBanned.append(se) + + if not(resR['OK'] or resW['OK'] or resC['OK']): + DIRAC.exit(-1) + + if not (writeBanned or readBanned or checkBanned or removeBanned): + gLogger.notice("No storage elements were banned") + DIRAC.exit(-1) + + if mute: + gLogger.notice('Email is muted by script switch') + DIRAC.exit(0) + + subject = '%s storage elements banned for use' % len(writeBanned + readBanned + checkBanned + removeBanned) + addressPath = 'EMail/Production' + address = Operations().getValue(addressPath, '') + + body = '' + if read: + body = "%s\n\nThe following storage elements were banned for reading:" % body + for se in readBanned: + body = "%s\n%s" % (body, se) + if write: + body = "%s\n\nThe following storage elements were banned for writing:" % body + for se in writeBanned: + body = "%s\n%s" % (body, se) + if check: + body = "%s\n\nThe following storage elements were banned for check access:" % body + for se in checkBanned: + body = "%s\n%s" % (body, se) + if remove: + body = "%s\n\nThe following storage elements were banned for remove access:" % body + for se in removeBanned: + body = "%s\n%s" % (body, se) + + if not address: + gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) + DIRAC.exit(0) + + res = diracAdmin.sendMail(address, subject, body) + gLogger.notice('Notifying %s' % address) + if res['OK']: + gLogger.notice(res['Value']) + else: + gLogger.notice(res['Message']) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py new file mode 100755 index 00000000000..e1ef550cda9 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +######################################################################## +# $HeadURL$ +######################################################################## +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Show storage quotas for specified users or for all registered users if nobody is specified + + Usage: + %s [user1 ...] + """ % Script.scriptName) + + Script.parseCommandLine() + users = Script.getPositionalArgs() + + from DIRAC import gLogger, gConfig + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + + if not users: + res = gConfig.getSections('/Registry/Users') + if not res['OK']: + gLogger.error("Failed to retrieve user list from CS", res['Message']) + DIRAC.exit(2) + users = res['Value'] + + gLogger.notice("-" * 30) + gLogger.notice("%s|%s" % ('Username'.ljust(15), 'Quota (GB)'.rjust(15))) + gLogger.notice("-" * 30) + for user in sorted(users): + quota = gConfig.getValue('/Registry/Users/%s/Quota' % user, 0) + if not quota: + quota = gConfig.getValue('/Registry/DefaultStorageQuota') + gLogger.notice("%s|%s" % (user.ljust(15), str(quota).rjust(15))) + gLogger.notice("-" * 30) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py new file mode 100755 index 00000000000..097596f2ece --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-dms-add-file +# Author : Stuart Paterson +######################################################################## +""" + Upload a file to the grid storage and register it in the File Catalog +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC import S_OK +import os +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + +overwrite = False + + +def setOverwrite(arg): + global overwrite + overwrite = True + return S_OK() + + +def getDict(item_list): + """ + From the input list, populate the dictionary + """ + lfn_dict = {} + lfn_dict['lfn'] = item_list[0].replace('LFN:', '').replace('lfn:', '') + lfn_dict['localfile'] = item_list[1] + lfn_dict['SE'] = item_list[2] + guid = None + if len(item_list) > 3: + guid = item_list[3] + lfn_dict['guid'] = guid + return lfn_dict + + +@DIRACScript() +def main(): + global overwrite + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + '\nUsage:', + ' %s [option|cfgfile] ... LFN Path SE [GUID]' % Script.scriptName, + '\nArguments:', + ' LFN: Logical File Name', + ' Path: Local path to the file', + ' SE: DIRAC Storage Element', + ' GUID: GUID to use in the registration (optional)', + '', + '**OR**', + '', + 'Usage:', + ' %s [option|cfgfile] ... LocalFile' % Script.scriptName, + '\nArguments:', + ' LocalFile: Path to local file containing all the above, i.e.:', + ' lfn1 localfile1 SE [GUID1]', + ' lfn2 localfile2 SE [GUID2]' + ])) + + Script.registerSwitch("f", "force", "Force overwrite of existing file", setOverwrite) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + if len(args) < 1 or len(args) > 4: + Script.showHelp(exitCode=1) + + from DIRAC.DataManagementSystem.Client.DataManager import DataManager + from DIRAC import gLogger + import DIRAC + exitCode = 0 + + lfns = [] + if len(args) == 1: + inputFileName = args[0] + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + for line in inputFile: + line = line.rstrip() + items = line.split() + items[0] = items[0].replace('LFN:', '').replace('lfn:', '') + lfns.append(getDict(items)) + inputFile.close() + else: + gLogger.error("Error: LFN list '%s' missing." % inputFileName) + exitCode = 4 + else: + lfns.append(getDict(args)) + + dm = DataManager() + for lfn in lfns: + if not os.path.exists(lfn['localfile']): + gLogger.error("File %s must exist locally" % lfn['localfile']) + exitCode = 1 + continue + if not os.path.isfile(lfn['localfile']): + gLogger.error("%s is not a file" % lfn['localfile']) + exitCode = 2 + continue + + gLogger.notice("\nUploading %s" % lfn['lfn']) + res = dm.putAndRegister(lfn['lfn'], lfn['localfile'], lfn['SE'], lfn['guid'], overwrite=overwrite) + if not res['OK']: + exitCode = 3 + gLogger.error('Error: failed to upload %s to %s' % (lfn['lfn'], lfn['SE'])) + continue + else: + gLogger.notice('Successfully uploaded file to %s' % lfn['SE']) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_catalog_metadata.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_catalog_metadata.py new file mode 100755 index 00000000000..83305171614 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_catalog_metadata.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +from DIRAC import exit as DIRACExit +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Get metadata for the given file specified by its Logical File Name or for a list of files + contained in the specifed file + + Usage: + %s [Catalog] + """ % Script.scriptName) + + Script.parseCommandLine() + + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + + import os + args = Script.getPositionalArgs() + + if not len(args) >= 1: + Script.showHelp(exitCode=1) + else: + inputFileName = args[0] + catalogs = [] + if len(args) == 2: + catalogs = [args[1]] + + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + lfns = string.splitlines() + inputFile.close() + else: + lfns = [inputFileName] + + res = FileCatalog(catalogs=catalogs).getFileMetadata(lfns) + if not res['OK']: + print("ERROR:", res['Message']) + DIRACExit(-1) + + print('FileName'.ljust(100), + 'Size'.ljust(10), + 'GUID'.ljust(40), + 'Status'.ljust(8), + 'Checksum'.ljust(10)) + for lfn in sorted(res['Value']['Successful'].keys()): + metadata = res['Value']['Successful'][lfn] + checksum = '' + if 'Checksum' in metadata: + checksum = str(metadata['Checksum']) + size = '' + if 'Size' in metadata: + size = str(metadata['Size']) + guid = '' + if 'GUID' in metadata: + guid = str(metadata['GUID']) + status = '' + if 'Status' in metadata: + status = str(metadata['Status']) + print('%s %s %s %s %s' % (lfn.ljust(100), size.ljust(10), guid.ljust(40), status.ljust(8), checksum.ljust(10))) + + for lfn in sorted(res['Value']['Failed'].keys()): + message = res['Value']['Failed'][lfn] + print(lfn, message) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py new file mode 100755 index 00000000000..19365d6a50f --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py @@ -0,0 +1,68 @@ +#! /usr/bin/env python +######################################################################## +# $HeadURL$ +######################################################################## +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +from DIRAC import exit as DIRACExit +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Change status of replica of a given file or a list of files at a given Storage Element + + Usage: + %s + """ % Script.scriptName) + + Script.parseCommandLine() + + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + catalog = FileCatalog() + import os + args = Script.getPositionalArgs() + if not len(args) == 3: + Script.showHelp(exitCode=1) + else: + inputFileName = args[0] + se = args[1] + newStatus = args[2] + + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + lfns = string.splitlines() + inputFile.close() + else: + lfns = [inputFileName] + + res = catalog.getReplicas(lfns, True) + if not res['OK']: + print(res['Message']) + DIRACExit(-1) + replicas = res['Value']['Successful'] + + lfnDict = {} + for lfn in lfns: + lfnDict[lfn] = {} + lfnDict[lfn]['SE'] = se + lfnDict[lfn]['Status'] = newStatus + lfnDict[lfn]['PFN'] = replicas[lfn][se] + + res = catalog.setReplicaStatus(lfnDict) + if not res['OK']: + print("ERROR:", res['Message']) + if res['Value']['Failed']: + print("Failed to update %d replica status" % len(res['Value']['Failed'])) + if res['Value']['Successful']: + print("Successfully updated %d replica status" % len(res['Value']['Successful'])) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py new file mode 100755 index 00000000000..02e8aeaa21c --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py @@ -0,0 +1,60 @@ +#! /usr/bin/env python + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +import os + +from DIRAC import exit as DIRACExit, gLogger +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Clean the given directory or a list of directories by removing it and all the + contained files and subdirectories from the physical storage and from the + file catalogs. + + Usage: + %s + """ % Script.scriptName) + + Script.parseCommandLine() + + args = Script.getPositionalArgs() + if len(args) != 1: + Script.showHelp(exitCode=1) + + inputFileName = args[0] + + if os.path.exists(inputFileName): + lfns = [lfn.strip().split()[0] for lfn in sorted(open(inputFileName, 'r').read().splitlines())] + else: + lfns = [inputFileName] + + from DIRAC.DataManagementSystem.Client.DataManager import DataManager + dm = DataManager() + retVal = 0 + for lfn in [lfn for lfn in lfns if lfn]: + gLogger.notice("Cleaning directory %r ... " % lfn) + result = dm.cleanLogicalDirectory(lfn) + if not result['OK']: + gLogger.error('Failed to clean directory', result['Message']) + retVal = -1 + else: + if not result['Value']['Failed']: + gLogger.notice('OK') + else: + for folder, message in result['Value']['Failed'].items(): + gLogger.error('Failed to clean folder', "%r: %s" % (folder, message)) + retVal = -1 + + DIRACExit(retVal) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-archive-request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py similarity index 99% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-archive-request.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py index 85f97755344..9f2542db535 100644 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-archive-request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py @@ -42,6 +42,7 @@ from DIRAC.Core.Utilities import DEncode from DIRAC.Core.Utilities.ReturnValues import returnSingleResult from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.FrameworkSystem.private.standardLogging.LogLevels import LogLevels from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.Client.Request import Request @@ -520,7 +521,8 @@ def _replicateSourceFiles(self, request, lfns): request.addOperation(registerSource) -if __name__ == '__main__': +@DIRACScript() +def main(): try: CAR = CreateArchiveRequest() CAR.run() @@ -531,3 +533,7 @@ def _replicateSourceFiles(self, request, lfns): sLog.error('ERROR: Failed to create Archive Request:', str(e)) exit(1) exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-moving-request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py similarity index 98% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-moving-request.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py index b25e3d57c63..d160dfe3d8b 100644 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-create-moving-request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py @@ -17,6 +17,7 @@ import DIRAC from DIRAC import gLogger from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Utilities.List import breakListIntoChunks from DIRAC.Core.Utilities.ReturnValues import returnSingleResult from DIRAC.FrameworkSystem.private.standardLogging.LogLevels import LogLevels @@ -292,7 +293,8 @@ def putOrRunRequests(self): return 1 -if __name__ == '__main__': +@DIRACScript() +def main(): try: CMR = CreateMovingRequest() CMR.run() @@ -303,3 +305,7 @@ def putOrRunRequests(self): sLog.error('ERROR: Failed to create Moving Request:', str(e)) exit(1) exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py new file mode 100755 index 00000000000..76d09d4f5e0 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +""" Create a DIRAC RemoveReplica|RemoveFile request to be executed by the RMS +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "ea64b42 (2012-07-29 16:45:05 +0200) ricardo " + +import os +from hashlib import md5 +import time +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.Core.Utilities.List import breakListIntoChunks + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[0], + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... SE LFN ...' % Script.scriptName, + 'Arguments:', + ' SE: StorageElement|All', + ' LFN: LFN or file containing a List of LFNs'])) + + Script.parseCommandLine(ignoreErrors=False) + + args = Script.getPositionalArgs() + if len(args) < 2: + Script.showHelp() + + targetSE = args.pop(0) + + lfns = [] + for inputFileName in args: + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + inputFile.close() + lfns.extend([lfn.strip() for lfn in string.splitlines()]) + else: + lfns.append(inputFileName) + + from DIRAC.Resources.Storage.StorageElement import StorageElement + import DIRAC + # Check is provided SE is OK + if targetSE != 'All': + se = StorageElement(targetSE) + if not se.valid: + print(se.errorReason) + print() + Script.showHelp() + + from DIRAC.RequestManagementSystem.Client.Request import Request + from DIRAC.RequestManagementSystem.Client.Operation import Operation + from DIRAC.RequestManagementSystem.Client.File import File + from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient + from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + + reqClient = ReqClient() + fc = FileCatalog() + + requestOperation = 'RemoveReplica' + if targetSE == 'All': + requestOperation = 'RemoveFile' + + for lfnList in breakListIntoChunks(lfns, 100): + + oRequest = Request() + requestName = "%s_%s" % ( + md5(repr(time.time()).encode()).hexdigest()[:16], + md5(repr(time.time()).encode()).hexdigest()[:16], + ) + oRequest.RequestName = requestName + + oOperation = Operation() + oOperation.Type = requestOperation + oOperation.TargetSE = targetSE + + res = fc.getFileMetadata(lfnList) + if not res['OK']: + print("Can't get file metadata: %s" % res['Message']) + DIRAC.exit(1) + if res['Value']['Failed']: + print("Could not get the file metadata of the following, so skipping them:") + for fFile in res['Value']['Failed']: + print(fFile) + + lfnMetadata = res['Value']['Successful'] + + for lfn in lfnMetadata: + rarFile = File() + rarFile.LFN = lfn + rarFile.Size = lfnMetadata[lfn]['Size'] + rarFile.Checksum = lfnMetadata[lfn]['Checksum'] + rarFile.GUID = lfnMetadata[lfn]['GUID'] + rarFile.ChecksumType = 'ADLER32' + oOperation.addFile(rarFile) + + oRequest.addOperation(oOperation) + + isValid = RequestValidator().validate(oRequest) + if not isValid['OK']: + print("Request is not valid: ", isValid['Message']) + DIRAC.exit(1) + + result = reqClient.putRequest(oRequest) + if result['OK']: + print('Request %d Submitted' % result['Value']) + else: + print('Failed to submit Request: ', result['Message']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py new file mode 100755 index 00000000000..dc380de68b3 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +######################################################################## +# $HeadURL$ +######################################################################## +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +import sys +import os +import DIRAC +from DIRAC import gLogger +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + unit = 'GB' + Script.registerSwitch("u:", "Unit=", " Unit to use [default %s] (MB,GB,TB,PB)" % unit) + + Script.setUsageMessage(""" + Get the size of the given file or a list of files + + Usage: + %s + """ % Script.scriptName) + + Script.parseCommandLine(ignoreErrors=False) + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "u" or switch[0].lower() == "unit": + unit = switch[1] + scaleDict = {'MB': 1000 * 1000.0, + 'GB': 1000 * 1000 * 1000.0, + 'TB': 1000 * 1000 * 1000 * 1000.0, + 'PB': 1000 * 1000 * 1000 * 1000 * 1000.0} + if unit not in scaleDict.keys(): + gLogger.error("Unit must be one of MB,GB,TB,PB") + DIRAC.exit(2) + scaleFactor = scaleDict[unit] + + args = Script.getPositionalArgs() + lfns = [] + for inputFileName in args: + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + inputFile.close() + lfns.extend([lfn.strip() for lfn in string.splitlines()]) + else: + lfns.append(inputFileName) + + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + res = FileCatalog().getFileSize(lfns) + if not res['OK']: + gLogger.error("Failed to get size of data", res['Message']) + DIRAC.exit(-2) + for lfn, reason in res['Value']['Failed'].items(): + gLogger.error("Failed to get size for %s" % lfn, reason) + totalSize = 0 + totalFiles = 0 + for lfn, size in res['Value']['Successful'].items(): + totalFiles += 1 + totalSize += size + gLogger.notice('-' * 30) + gLogger.notice('%s|%s' % ('Files'.ljust(15), ('Size (%s)' % unit).rjust(15))) + gLogger.notice('-' * 30) + gLogger.notice('%s|%s' % (str(totalFiles).ljust(15), str('%.1f' % (totalSize / scaleFactor)).rjust(15))) + gLogger.notice('-' * 30) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py new file mode 100755 index 00000000000..8ac7c47a760 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py @@ -0,0 +1,566 @@ +#!/usr/bin/env python +######################################################################## +# :file: dirac-dms-directory-sync +# :author: Marko Petric +######################################################################## +""" +Provides basic rsync functionality for DIRAC + +Syncs the source destination folder recursivly into the target destination + +If option --sync is used contend that is not in the source directory but is +only in the target directory will be deleted. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from past.builtins import long +import os +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s Source Destination' % Script.scriptName, + ' ', + ' e.g.: Download', + ' %s LFN Path' % Script.scriptName, + ' or Upload', + ' %s Path LFN SE' % Script.scriptName, + 'Arguments:', + ' LFN: Logical File Name (Path to directory)', + ' Path: Local path to the file (Path to directory)', + ' SE: DIRAC Storage Element' + ])) + + Script.registerSwitch("D", "sync", "Make target directory identical to source") + Script.registerSwitch("j:", "parallel=", "Multithreaded download and upload") + Script.parseCommandLine(ignoreErrors=False) + + args = Script.getPositionalArgs() + if len(args) < 1 or len(args) > 3: + Script.showHelp() + + sync = False + parallel = 1 + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "s" or switch[0].lower() == "sync": + sync = True + if switch[0].lower() == "j" or switch[0].lower() == "parallel": + parallel = int(switch[1]) + + from DIRAC import S_OK, S_ERROR + from DIRAC import gConfig, gLogger + from DIRAC.Core.Utilities.List import breakListIntoChunks + from DIRAC.Core.Utilities.ReturnValues import returnSingleResult + from DIRAC.Core.Utilities.File import mkDir + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + from DIRAC.DataManagementSystem.Client.DataManager import DataManager + from DIRAC.Resources.Storage.StorageElement import StorageElement + + __RCSID__ = "$Id$" + + def getSetOfLocalDirectoriesAndFiles(path): + """Return a set of all directories and subdirectories and a set of + files contained therein for a given local path + """ + + fullPath = os.path.abspath(path) + + if not os.path.isdir(fullPath): + return S_ERROR("The path: " + fullPath + " does not exist!") + + directories = set() + files = set() + + for dirname, dirnames, filenames in os.walk(path): + # add path to all subdirectories first. + for subdirname in dirnames: + fullSubdirname = os.path.join(dirname, subdirname) + fullSubdirname = os.path.abspath(fullSubdirname) + fullSubdirname = fullSubdirname.replace(fullPath, '').lstrip('/') + directories.add(fullSubdirname) + # add path to all filenames. + for filename in filenames: + fullFilename = os.path.join(dirname, filename) + fullFilename = os.path.abspath(fullFilename) + fullFilename = fullFilename.replace(fullPath, '').lstrip('/') + fileSize = os.path.getsize(fullPath + "/" + fullFilename) + if fileSize > 0: + files.add((fullFilename, long(fileSize))) + + tree = {} + tree["Directories"] = directories + tree["Files"] = files + + return S_OK(tree) + + def getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories, files): + """ + Recursively traverses all the subdirectories of a directory and returns a set of directories and files + """ + result = fc.listDirectory(path) + if result['OK']: + if result['Value']['Successful']: + for entry in result['Value']['Successful'][path]['Files']: + size = result['Value']['Successful'][path]['Files'][entry]['MetaData']['Size'] + files.add((entry, size)) + for entry in result['Value']['Successful'][path]['SubDirs']: + directories.add(entry) + res = getSetOfRemoteSubDirectoriesAndFiles(entry, fc, directories, files) + if not res['OK']: + return S_ERROR('Error: ' + res['Message']) + return S_OK() + else: + return S_ERROR("Error: %s" % result['Value']) + else: + return S_ERROR("Error:" + result['Message']) + + def getSetOfRemoteDirectoriesAndFiles(fc, path): + """ + Return a set of all directories and subdirectories and the therein contained files for a given LFN + """ + directories = set() + files = set() + + res = getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories, files) + if not res['OK']: + return S_ERROR('Could not list remote directory: ' + res['Message']) + + return_directories = set() + return_files = set() + + for myfile in files: + return_files.add((myfile[0].replace(path, '').lstrip('/'), myfile[1])) + + for mydirectory in directories: + return_directories.add(mydirectory.replace(path, '').lstrip('/')) + + tree = {} + tree["Directories"] = return_directories + tree["Files"] = return_files + + return S_OK(tree) + + def isInFileCatalog(fc, path): + """ + Check if the file is in the File Catalog + """ + + result = fc.listDirectory(path) + if result['OK']: + if result['Value']['Successful']: + return S_OK() + else: + return S_ERROR() + else: + return S_ERROR() + + def getContentToSync(upload, fc, source_dir, dest_dir): + """ + Return list of files and directories to be create and deleted + """ + + if upload: + res = getSetOfRemoteDirectoriesAndFiles(fc, dest_dir) + if not res['OK']: + return S_ERROR(res['Message']) + to_dirs = res['Value']['Directories'] + to_files = res['Value']['Files'] + + res = getSetOfLocalDirectoriesAndFiles(source_dir) + if not res['OK']: + return S_ERROR(res['Message']) + from_dirs = res['Value']['Directories'] + from_files = res['Value']['Files'] + + else: + res = getSetOfLocalDirectoriesAndFiles(dest_dir) + if not res['OK']: + return S_ERROR(res['Message']) + to_dirs = res['Value']['Directories'] + to_files = res['Value']['Files'] + + res = getSetOfRemoteDirectoriesAndFiles(fc, source_dir) + if not res['OK']: + return S_ERROR(res['Message']) + from_dirs = res['Value']['Directories'] + from_files = res['Value']['Files'] + + # Create list of directories to delete + dirs_delete = list(to_dirs - from_dirs) + # Sort the list by depth of directory tree + dirs_delete.sort(key=lambda s: -s.count('/')) + # Create list of directories to create + dirs_create = list(from_dirs - to_dirs) + # Sort the list by depth of directory tree + dirs_create.sort(key=lambda s: s.count('/')) + + # Flatten the list of pairs (filename, size) to list of filename + files_delete = [pair[0] for pair in list(to_files - from_files)] + files_create = [pair[0] for pair in list(from_files - to_files)] + + create = {} + create["Directories"] = dirs_create + create["Files"] = files_create + + delete = {} + delete["Directories"] = dirs_delete + delete["Files"] = files_delete + + tree = {} + tree["Create"] = create + tree["Delete"] = delete + + return S_OK(tree) + + def removeRemoteFiles(dm, lfns): + """ + Remove file from the catalog + """ + for lfnList in breakListIntoChunks(lfns, 100): + res = dm.removeFile(lfnList) + if not res['OK']: + return S_ERROR("Failed to remove files:" + lfnList + res['Message']) + else: + return S_OK() + + def uploadLocalFile(dm, lfn, localfile, storage): + """ + Upload a local file to a storage element + """ + res = dm.putAndRegister(lfn, localfile, storage, None) + if not res['OK']: + return S_ERROR('Error: failed to upload %s to %s' % (lfn, storage)) + else: + return S_OK('Successfully uploaded file to %s' % storage) + + def downloadRemoteFile(dm, lfn, destination): + """ + Download a file from the system + """ + res = dm.getFile(lfn, destination) + if not res['OK']: + return S_ERROR('Error: failed to download %s ' % lfn) + else: + return S_OK('Successfully uploaded file %s' % lfn) + + def removeStorageDirectoryFromSE(directory, storageElement): + """ + Delete directory on selected storage element + """ + + se = StorageElement(storageElement, False) + res = returnSingleResult(se.exists(directory)) + + if not res['OK']: + return S_ERROR("Failed to obtain existence of directory" + res['Message']) + + exists = res['Value'] + if not exists: + return S_OK("The directory %s does not exist at %s " % (directory, storageElement)) + + res = returnSingleResult(se.removeDirectory(directory, recursive=True)) + if not res['OK']: + return S_ERROR("Failed to remove storage directory" + res['Message']) + + return S_OK() + + def removeRemoteDirectory(fc, lfn): + """ + Remove file from the catalog + """ + storageElements = gConfig.getValue('Resources/StorageElementGroups/SE_Cleaning_List', []) + + for storageElement in sorted(storageElements): + res = removeStorageDirectoryFromSE(lfn, storageElement) + if not res['OK']: + return S_ERROR("Failed to clean storage directory at all SE:" + res['Message']) + res = returnSingleResult(fc.removeDirectory(lfn, recursive=True)) + if not res['OK']: + return S_ERROR("Failed to clean storage directory at all SE:" + res['Message']) + + return S_OK("Successfully removed directory") + + def createRemoteDirectory(fc, newdir): + """ + Create directory in file catalog + """ + result = fc.createDirectory(newdir) + if result['OK']: + if result['Value']['Successful'] and newdir in result['Value']['Successful']: + return S_OK("Successfully created directory:" + newdir) + elif result['Value']['Failed'] and newdir in result['Value']['Failed']: + return S_ERROR('Failed to create directory: ' + result['Value']['Failed'][newdir]) + else: + return S_ERROR('Failed to create directory:' + result['Message']) + + def createLocalDirectory(directory): + """ + Create local directory + """ + mkDir(directory) + if not os.path.exists(directory): + return S_ERROR('Directory creation failed') + return S_OK('Created directory successfully') + + def removeLocalFile(path): + """ + Remove local file + """ + try: + os.remove(path) + except OSError as e: + return S_ERROR('Directory creation failed:' + e.strerror) + + if os.path.isfile(path): + return S_ERROR('File deleting failed') + return S_OK('Removed file successfully') + + def removeLocaDirectory(path): + """ + Remove local directory + """ + try: + os.rmdir(path) + except OSError as e: + return S_ERROR('Deleting directory failed: ' + e.strerror) + + if os.path.isdir(path): + return S_ERROR('Directory deleting failed') + return S_OK('Removed directory successfully') + + def doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads): + """ + Wrapper for uploading files + """ + if delete: + lfns = [dest_dir + "/" + filename for filename in result['Value']['Delete']['Files']] + if len(lfns) > 0: + res = removeRemoteFiles(dm, lfns) + if not res['OK']: + gLogger.fatal('Deleting of files: ' + lfns + " -X- [FAILED]" + res['Message']) + DIRAC.exit(1) + else: + gLogger.notice("Deleting " + ', '.join(lfns) + " -> [DONE]") + + for directoryname in result['Value']['Delete']['Directories']: + res = removeRemoteDirectory(fc, dest_dir + "/" + directoryname) + if not res['OK']: + gLogger.fatal('Deleting of directory: ' + directoryname + " -X- [FAILED] " + res['Message']) + DIRAC.exit(1) + else: + gLogger.notice("Deleting " + directoryname + " -> [DONE]") + + for directoryname in result['Value']['Create']['Directories']: + res = createRemoteDirectory(fc, dest_dir + "/" + directoryname) + if not res['OK']: + gLogger.fatal('Creation of directory: ' + directoryname + " -X- [FAILED] " + res['Message']) + DIRAC.exit(1) + else: + gLogger.notice("Creating " + directoryname + " -> [DONE]") + + listOfFiles = result['Value']['Create']['Files'] + # Chech that we do not have to many threads + if nthreads > len(listOfFiles): + nthreads = len(listOfFiles) + + if nthreads == 0: + return S_OK('Upload finished successfully') + + listOfListOfFiles = chunkList(listOfFiles, nthreads) + res = runInParallel( + arguments=[ + dm, + source_dir, + dest_dir, + storage], + listOfLists=listOfListOfFiles, + function=uploadListOfFiles) + if not res['OK']: + return S_ERROR("Upload of files failed") + + return S_OK('Upload finished successfully') + + def uploadListOfFiles(dm, source_dir, dest_dir, storage, listOfFiles, tID): + """ + Wrapper for multithreaded uploading of a list of files + """ + log = gLogger.getSubLogger("[Thread %s] " % tID) + threadLine = "[Thread %s]" % tID + for filename in listOfFiles: + res = uploadLocalFile(dm, dest_dir + "/" + filename, source_dir + "/" + filename, storage) + if not res['OK']: + log.fatal(threadLine + ' Uploading ' + filename + ' -X- [FAILED] ' + res['Message']) + DIRAC.exit(1) + else: + log.notice(threadLine + " Uploading " + filename + " -> [DONE]") + + def doDownload(dm, result, source_dir, dest_dir, delete, nthreads): + """ + Wrapper for downloading files + """ + if delete: + for filename in result['Value']['Delete']['Files']: + res = removeLocalFile(dest_dir + "/" + filename) + if not res['OK']: + gLogger.fatal('Deleting of file: ' + filename + ' -X- [FAILED] ' + res['Message']) + DIRAC.exit(1) + else: + gLogger.notice("Deleting " + filename + " -> [DONE]") + + for directoryname in result['Value']['Delete']['Directories']: + res = removeLocaDirectory(dest_dir + "/" + directoryname) + if not res['OK']: + gLogger.fatal('Deleting of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message']) + DIRAC.exit(1) + else: + gLogger.notice("Deleting " + directoryname + " -> [DONE]") + + for directoryname in result['Value']['Create']['Directories']: + res = createLocalDirectory(dest_dir + "/" + directoryname) + if not res['OK']: + gLogger.fatal('Creation of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message']) + DIRAC.exit(1) + else: + gLogger.notice("Creating " + directoryname + " -> [DONE]") + + listOfFiles = result['Value']['Create']['Files'] + # Chech that we do not have to many threads + if nthreads > len(listOfFiles): + nthreads = len(listOfFiles) + + if nthreads == 0: + return S_OK('Upload finished successfully') + + listOfListOfFiles = chunkList(listOfFiles, nthreads) + res = runInParallel( + arguments=[dm, source_dir, dest_dir], + listOfLists=listOfListOfFiles, + function=downloadListOfFiles, + ) + + if not res['OK']: + return S_ERROR("Download of files failed") + + return S_OK('Upload finished successfully') + + def chunkList(alist, nchunks): + """ + Split a list into a list of equaliy sized lists + """ + avg = len(alist) / float(nchunks) + out = [] + last = 0.0 + + while last < len(alist): + out.append(alist[int(last):int(last + avg)]) + last += avg + + return out + + def downloadListOfFiles(dm, source_dir, dest_dir, listOfFiles, tID): + """ + Wrapper for multithreaded downloading of a list of files + """ + log = gLogger.getSubLogger("[Thread %s] " % tID) + threadLine = "[Thread %s]" % tID + for filename in listOfFiles: + res = downloadRemoteFile(dm, source_dir + "/" + filename, dest_dir + ("/" + filename).rsplit("/", 1)[0]) + if not res['OK']: + log.fatal(threadLine + ' Downloading ' + filename + ' -X- [FAILED] ' + res['Message']) + DIRAC.exit(1) + else: + log.notice(threadLine + " Downloading " + filename + " -> [DONE]") + + def runInParallel(arguments, listOfLists, function): + """ + Helper for execution of uploads and downloads in parallel + """ + from multiprocessing import Process + processes = [] + for tID, alist in enumerate(listOfLists): + argums = arguments + [alist] + [tID] + pro = Process(target=function, args=argums) + pro.start() + processes.append(pro) + for process in processes: + process.join() + + for process in processes: + if process.exitcode == 1: + return S_ERROR() + return S_OK() + + def syncDestinations(upload, source_dir, dest_dir, storage, delete, nthreads): + """ + Top level wrapper to execute functions + """ + + fc = FileCatalog() + dm = DataManager() + + result = getContentToSync(upload, fc, source_dir, dest_dir) + if not result['OK']: + return S_ERROR(result['Message']) + + if upload: + res = doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads) + if not res['OK']: + return S_ERROR('Upload failed: ' + res['Message']) + else: + res = doDownload(dm, result, source_dir, dest_dir, delete, nthreads) + if not res['OK']: + return S_ERROR('Download failed: ' + res['Message']) + + return S_OK('Mirroring successfully finished') + + def run(parameters, delete, nthreads): + """ + The main user interface + """ + + source_dir = parameters[0] + dest_dir = parameters[1] + upload = False + storage = None + + if len(parameters) == 3: + storage = parameters[2] + source_dir = os.path.abspath(source_dir) + dest_dir = dest_dir.rstrip('/') + upload = True + if not os.path.isdir(source_dir): + gLogger.fatal("Source directory does not exist") + DIRAC.exit(1) + + if len(parameters) == 2: + dest_dir = os.path.abspath(dest_dir) + source_dir = source_dir.rstrip('/') + if not os.path.isdir(dest_dir): + gLogger.fatal("Destination directory does not exist") + DIRAC.exit(1) + + res = syncDestinations(upload, source_dir, dest_dir, storage, delete, nthreads) + if not res['OK']: + return S_ERROR(res['Message']) + + return S_OK("Successfully mirrored " + source_dir + " into " + dest_dir) + + returnValue = run(args, sync, parallel) + if not returnValue['OK']: + gLogger.fatal(returnValue['Message']) + DIRAC.exit(1) + else: + gLogger.notice(returnValue['Value']) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py new file mode 100755 index 00000000000..6152a3d1357 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +import sys + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Launch the File Catalog shell + + Usage: + %s [option] + """ % Script.scriptName) + + fcType = 'FileCatalog' + Script.registerSwitch("f:", "file-catalog=", " Catalog client type to use (default %s)" % fcType) + Script.parseCommandLine(ignoreErrors=False) + + from DIRAC import gConfig, exit as dexit + from DIRAC.Resources.Catalog.FileCatalogFactory import FileCatalogFactory + + __RCSID__ = "$Id$" + + fcType = gConfig.getValue("/LocalSite/FileCatalog", "") + + res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True) + if not res['OK']: + dexit(1) + fcList = res['Value'] + if not fcType: + if res['OK']: + fcType = res['Value'][0] + + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "f" or switch[0].lower() == "file-catalog": + fcType = switch[1] + + if not fcType: + print("No file catalog given and defaults could not be obtained") + sys.exit(1) + + from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI + + result = FileCatalogFactory().createCatalog(fcType) + if not result['OK']: + print(result['Message']) + if fcList: + print("Possible choices are:") + for fc in fcList: + print(' ' * 5, fc) + sys.exit(1) + print("Starting %s client" % fcType) + catalog = result['Value'] + cli = FileCatalogClientCLI(catalog) + cli.cmdloop() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-find-lfns.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py similarity index 96% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-find-lfns.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py index 2ae25a1169c..61cb8f5421e 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-find-lfns.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py @@ -8,9 +8,11 @@ from __future__ import absolute_import from __future__ import division import DIRAC +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if __name__ == "__main__": +@DIRACScript() +def main(): from DIRAC.Core.Base import Script Script.registerSwitch('', 'Path=', ' Path to search for') @@ -78,3 +80,7 @@ lfnList = sorted(result['Value']) gLogger.notice('\n'.join(lfn for lfn in lfnList)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-move-replica-request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py similarity index 86% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-move-replica-request.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py index 9fa7de43fe2..2268aba5375 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-move-replica-request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py @@ -11,14 +11,7 @@ from hashlib import md5 import time from DIRAC.Core.Base import Script -Script.setUsageMessage('\n'.join([__doc__.split('\n')[0], - __doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... sourceSE LFN targetSE1 [targetSE2...]' % Script.scriptName, - 'Arguments:', - ' sourceSE: source SE', - ' targetSE: target SE', - ' LFN: LFN or file containing a List of LFNs'])) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript def getLFNList(arg): @@ -31,8 +24,18 @@ def getLFNList(arg): return list(set(lfnList)) -# # execution -if __name__ == "__main__": +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[0], + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... sourceSE LFN targetSE1 [targetSE2...]' % Script.scriptName, + 'Arguments:', + ' sourceSE: source SE', + ' targetSE: target SE', + ' LFN: LFN or file containing a List of LFNs' + ])) from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() @@ -126,3 +129,7 @@ def getLFNList(arg): if multiRequests: gLogger.always("%d requests have been submitted" % (count)) DIRAC.exit(error) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-protocol-matrix.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py similarity index 90% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-protocol-matrix.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py index 46337e08a73..4b87c7d73e7 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-protocol-matrix.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py @@ -46,16 +46,19 @@ from collections import defaultdict from DIRAC.Core.Base import Script -Script.registerSwitch('', 'FromSE=', 'SE1[,SE2,...]') -Script.registerSwitch('', 'TargetSE=', 'SE1[,SE2,...]') -Script.registerSwitch('', 'OutputFile=', 'CSV output file (default /tmp/protocol-matrix.csv)') -Script.registerSwitch('', 'Bidirection', 'If FromSE or TargetSE are specified, make a square matrix ') -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s [option|cfgfile] % Script.scriptName'])) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if __name__ == '__main__': +@DIRACScript() +def main(): + Script.registerSwitch('', 'FromSE=', 'SE1[,SE2,...]') + Script.registerSwitch('', 'TargetSE=', 'SE1[,SE2,...]') + Script.registerSwitch('', 'OutputFile=', 'CSV output file (default /tmp/protocol-matrix.csv)') + Script.registerSwitch('', 'Bidirection', 'If FromSE or TargetSE are specified, make a square matrix ') + Script.setUsageMessage('\n'.join([__doc__, + 'Usage:', + ' %s [option|cfgfile] % Script.scriptName'])) + from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() from DIRAC import gConfig, gLogger @@ -177,3 +180,7 @@ srcRow.append(tpMatrix[src].get(dst, 'NA')) csvWriter.writerow(srcRow) gLogger.notice('Wrote Matrix to', outputFile) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-put-and-register-request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py similarity index 82% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-put-and-register-request.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py index 163343a7810..a3f5d8a8e50 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-put-and-register-request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py @@ -12,17 +12,21 @@ import os from DIRAC.Core.Base import Script -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s [option|cfgfile] requestName LFN localFile targetSE' % Script.scriptName, - 'Arguments:', - ' requestName: a request name', - ' LFN: logical file name' - ' localFile: local file you want to put', - ' targetSE: target SE'])) - -# # execution -if __name__ == "__main__": +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__, + 'Usage:', + ' %s [option|cfgfile] requestName LFN localFile targetSE' % Script.scriptName, + 'Arguments:', + ' requestName: a request name', + ' LFN: logical file name' + ' localFile: local file you want to put', + ' targetSE: target SE' + ])) from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() @@ -91,3 +95,7 @@ gLogger.always("Request '%s' has been put to ReqDB for execution." % requestName) gLogger.always("You can monitor its status using command: 'dirac-rms-request %s'" % requestName) DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py new file mode 100755 index 00000000000..b0f6c99358d --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +######################################################################## +# $Header: $ +######################################################################## +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC import exit as dexit +from DIRAC import gLogger + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Remove the given file or a list of files from the File Catalog + + Usage: + %s + """ % Script.scriptName) + + Script.parseCommandLine() + + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + allowUsers = Operations().getValue("DataManagement/AllowUserReplicaManagement", False) + + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + res = getProxyInfo() + if not res['OK']: + gLogger.fatal("Can't get proxy info", res['Message']) + dexit(1) + properties = res['Value'].get('groupProperties', []) + + if not allowUsers: + if 'FileCatalogManagement' not in properties: + gLogger.error("You need to use a proxy from a group with FileCatalogManagement") + dexit(5) + + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + fc = FileCatalog() + import os + + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + else: + inputFileName = args[0] + + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + lfns = [lfn.strip() for lfn in string.splitlines()] + inputFile.close() + else: + lfns = [inputFileName] + + res = fc.removeFile(lfns) + if not res['OK']: + print("Error:", res['Message']) + dexit(1) + for lfn in sorted(res['Value']['Failed'].keys()): + message = res['Value']['Failed'][lfn] + print('Error: failed to remove %s: %s' % (lfn, message)) + print('Successfully removed %d catalog files.' % (len(res['Value']['Successful']))) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py new file mode 100755 index 00000000000..89584c11ad7 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import os + +from DIRAC import exit as dexit +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC import gLogger + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Remove the given file replica or a list of file replicas from the File Catalog + This script should be used with great care as it may leave dark data in the storage! + Use dirac-dms-remove-replicas instead + + Usage: + %s + """ % Script.scriptName) + + Script.parseCommandLine() + + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + allowUsers = Operations().getValue("DataManagement/AllowUserReplicaManagement", False) + + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + res = getProxyInfo() + if not res['OK']: + gLogger.fatal("Can't get proxy info", res['Message']) + dexit(1) + properties = res['Value'].get('groupProperties', []) + + if not allowUsers: + if 'FileCatalogManagement' not in properties: + gLogger.error("You need to use a proxy from a group with FileCatalogManagement") + dexit(5) + + from DIRAC.DataManagementSystem.Client.DataManager import DataManager + dm = DataManager() + args = Script.getPositionalArgs() + if len(args) < 2: + Script.showHelp(exitCode=1) + else: + inputFileName = args[0] + storageElementName = args[1] + + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + lfns = [lfn.strip() for lfn in string.splitlines()] + inputFile.close() + else: + lfns = [inputFileName] + + res = dm.removeReplicaFromCatalog(storageElementName, lfns) + if not res['OK']: + print(res['Message']) + dexit(0) + for lfn in sorted(res['Value']['Failed']): + message = res['Value']['Failed'][lfn] + print('Failed to remove %s replica of %s: %s' % (storageElementName, lfn, message)) + print('Successfully remove %d catalog replicas at %s' % (len(res['Value']['Successful']), storageElementName)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py new file mode 100755 index 00000000000..450215a6373 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +######################################################################## +# $HeadURL$ +######################################################################## +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Remove the given file or a list of files from the File Catalog and from the storage + + Usage: + %s + """ % Script.scriptName) + + Script.parseCommandLine() + + import sys + import os + import DIRAC + from DIRAC import gLogger + + args = Script.getPositionalArgs() + lfns = [] + for inputFileName in args: + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + inputFile.close() + lfns.extend([lfn.strip() for lfn in string.splitlines()]) + else: + lfns.append(inputFileName) + + from DIRAC.Core.Utilities.List import breakListIntoChunks + from DIRAC.DataManagementSystem.Client.DataManager import DataManager + dm = DataManager() + + errorReasons = {} + successfullyRemoved = 0 + for lfnList in breakListIntoChunks(lfns, 100): + res = dm.removeFile(lfnList) + if not res['OK']: + gLogger.error("Failed to remove data", res['Message']) + DIRAC.exit(-2) + for lfn, r in res['Value']['Failed'].items(): + reason = str(r) + if reason not in errorReasons: + errorReasons[reason] = [] + errorReasons[reason].append(lfn) + successfullyRemoved += len(res['Value']['Successful']) + + for reason, lfns in errorReasons.items(): + gLogger.notice("Failed to remove %d files with error: %s" % (len(lfns), reason)) + if successfullyRemoved > 0: + gLogger.notice("Successfully removed %d files" % successfullyRemoved) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-replicas.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_replicas.py similarity index 94% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-replicas.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_replicas.py index 9c9ab589333..028c65409fb 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-remove-replicas.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_replicas.py @@ -7,9 +7,11 @@ from DIRAC import exit as DIRACExit from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if __name__ == "__main__": +@DIRACScript() +def main(): Script.setUsageMessage(""" Remove the given file replica or a list of file replicas from the File Catalog @@ -53,3 +55,7 @@ for lfn in sorted(res['Value']['Failed']): message = res['Value']['Failed'][lfn] print('Error: failed to remove %s replica of %s: %s' % (storageElementName, lfn, message)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py new file mode 100755 index 00000000000..47744c0b87b --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import os + +from DIRAC import exit as DIRACExit +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Get the given file replica metadata from the File Catalog + + Usage: + %s SE + """ % Script.scriptName) + + Script.parseCommandLine() + + from DIRAC import gLogger + from DIRAC.DataManagementSystem.Client.DataManager import DataManager + + args = Script.getPositionalArgs() + if not len(args) == 2: + Script.showHelp(exitCode=1) + else: + inputFileName = args[0] + storageElement = args[1] + + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + lfns = [lfn.strip() for lfn in string.splitlines()] + inputFile.close() + else: + lfns = [inputFileName] + + res = DataManager().getReplicaMetadata(lfns, storageElement) + if not res['OK']: + print('Error:', res['Message']) + DIRACExit(1) + + print('%s %s %s %s' % ('File'.ljust(100), 'Migrated'.ljust(8), 'Cached'.ljust(8), 'Size (bytes)'.ljust(10))) + for lfn, metadata in res['Value']['Successful'].items(): + print( + '%s %s %s %s' % + (lfn.ljust(100), str( + metadata['Migrated']).ljust(8), str( + metadata.get( + 'Cached', metadata['Accessible'])).ljust(8), str( + metadata['Size']).ljust(10))) + for lfn, reason in res['Value']['Failed'].items(): + print('%s %s' % (lfn.ljust(100), reason.ljust(8))) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-replicate-and-register-request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py similarity index 84% rename from src/DIRAC/DataManagementSystem/scripts/dirac-dms-replicate-and-register-request.py rename to src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py index 6edd6ca797c..73c15036314 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac-dms-replicate-and-register-request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py @@ -6,22 +6,9 @@ __RCSID__ = "$Id$" import os from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC import gLogger import DIRAC -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s [option|cfgfile] requestName LFNs targetSE1 [targetSE2 ...]' % Script.scriptName, - 'Arguments:', - ' requestName: a request name', - ' LFNs: single LFN or file with LFNs', - ' targetSE: target SE'])) - -catalog = None -Script.registerSwitch("C:", "Catalog=", "Catalog to use") -Script.parseCommandLine() -for switch in Script.getUnprocessedSwitches(): - if switch[0] == "C" or switch[0].lower() == "catalog": - catalog = switch[1] def getLFNList(arg): @@ -34,8 +21,24 @@ def getLFNList(arg): return list(set(lfnList)) -# # execution -if __name__ == "__main__": +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__, + 'Usage:', + ' %s [option|cfgfile] requestName LFNs targetSE1 [targetSE2 ...]' % Script.scriptName, + 'Arguments:', + ' requestName: a request name', + ' LFNs: single LFN or file with LFNs', + ' targetSE: target SE', + ])) + + catalog = None + Script.registerSwitch("C:", "Catalog=", "Catalog to use") + Script.parseCommandLine() + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "C" or switch[0].lower() == "catalog": + catalog = switch[1] args = Script.getPositionalArgs() @@ -126,3 +129,7 @@ def getLFNList(arg): gLogger.always("RequestID(s): %s" % " ".join(requestIDs)) gLogger.always("You can monitor requests' status using command: 'dirac-rms-request '") DIRAC.exit(error) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_resolve_guid.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_resolve_guid.py new file mode 100755 index 00000000000..8b5efb3693f --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_resolve_guid.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Returns the LFN matching given GUIDs + Usage: + %s + """ % Script.scriptName) + + Script.parseCommandLine() + + import DIRAC + from DIRAC import gLogger + + args = Script.getPositionalArgs() + if len(args) != 1: + Script.showHelp() + guids = args[0] + + try: + guids = guids.split(',') + except BaseException: + pass + + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + + fc = FileCatalog() + res = fc.getLFNForGUID(guids) + if not res['OK']: + gLogger.error("Failed to get the LFNs", res['Message']) + DIRAC.exit(-2) + + errorGuid = {} + for guid, reason in res['Value']['Failed'].items(): + errorGuid.setdefault(reason, []).append(guid) + + for error, guidList in errorGuid.items(): + gLogger.notice("Error '%s' for guids %s" % (error, guidList)) + + for guid, lfn in res['Value']['Successful'].items(): + gLogger.notice("%s -> %s" % (guid, lfn)) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py new file mode 100755 index 00000000000..fd31066cab4 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +######################################################################## +# $HeadURL$ +######################################################################## +""" +Set the status of the replicas of given files at the provided SE +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + '\nUsage:', + ' %s [option|cfgfile] ... SE Status' % Script.scriptName, + 'Arguments:', + ' LFN: LFN', + ' File: File name containing a list of affected LFNs', + ' SE: Name of Storage Element', + ' Status: New Status for the replica'])) + + Script.parseCommandLine(ignoreErrors=False) + + import DIRAC + from DIRAC import gConfig, gLogger + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + import os + + args = Script.getPositionalArgs() + if not len(args) == 3: + Script.showHelp() + + inputFileName = args[0] + storageElement = args[1] + status = args[2] + + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + inputFile.close() + lfns = sorted(string.splitlines()) + else: + lfns = [inputFileName] + + fc = FileCatalog() + + replicaDict = {} + res = fc.getReplicas(lfns, allStatus=True) + if not res['OK']: + gLogger.error("Failed to get catalog replicas.", res['Message']) + DIRAC.exit(-1) + lfnDict = {} + for lfn, error in res['Value']['Failed'].items(): + gLogger.error("Failed to get replicas for file.", "%s:%s" % (lfn, error)) + for lfn, replicas in res['Value']['Successful'].items(): + if storageElement not in replicas.keys(): + gLogger.error("LFN not registered at provided storage element.", "%s %s" % (lfn, storageElement)) + else: + lfnDict[lfn] = {'SE': storageElement, 'PFN': replicas[storageElement], 'Status': status} + if not lfnDict: + gLogger.error("No files found at the supplied storage element.") + DIRAC.exit(2) + + res = fc.setReplicaStatus(lfnDict) + if not res['OK']: + gLogger.error("Failed to set catalog replica status.", res['Message']) + DIRAC.exit(-1) + for lfn, error in res['Value']['Failed'].items(): + gLogger.error("Failed to set replica status for file.", "%s:%s" % (lfn, error)) + gLogger.notice("Successfully updated the status of %d files at %s." % + (len(res['Value']['Successful'].keys()), storageElement)) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py new file mode 100755 index 00000000000..5ad4d83debb --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +from DIRAC import S_OK, exit as DIRACexit +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + +__RCSID__ = "$Id$" + +vo = None + + +def setVO(arg): + global vo + vo = arg + return S_OK() + + +allVOsFlag = False + + +def setAllVO(arg): + global allVOsFlag + allVOsFlag = True + return S_OK() + + +noVOFlag = False + + +def setNoVO(arg): + global noVOFlag, allVOsFlag + noVOFlag = True + allVOsFlag = False + return S_OK() + + +@DIRACScript() +def main(): + global vo + global noVOFlag + global allVOsFlag + + Script.setUsageMessage(""" +Get status of the available Storage Elements + +Usage: + %s [] +""" % Script.scriptName) + + Script.registerSwitch("V:", "vo=", "Virtual Organization", setVO) + Script.registerSwitch("a", "all", "All Virtual Organizations flag", setAllVO) + Script.registerSwitch("n", "noVO", "No Virtual Organizations assigned flag", setNoVO) + + Script.parseCommandLine() + + from DIRAC import gConfig, gLogger + from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus + from DIRAC.Core.Utilities.PrettyPrint import printTable + from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup + + storageCFGBase = "/Resources/StorageElements" + + res = gConfig.getSections(storageCFGBase, True) + if not res['OK']: + gLogger.error('Failed to get storage element info') + gLogger.error(res['Message']) + DIRACexit(1) + + gLogger.info("%s %s %s" % ('Storage Element'.ljust(25), 'Read Status'.rjust(15), 'Write Status'.rjust(15))) + + seList = sorted(res['Value']) + + resourceStatus = ResourceStatus() + + res = resourceStatus.getElementStatus(seList, "StorageElement") + if not res['OK']: + gLogger.error("Failed to get StorageElement status for %s" % str(seList)) + DIRACexit(1) + + fields = ['SE', 'ReadAccess', 'WriteAccess', 'RemoveAccess', 'CheckAccess'] + records = [] + + if vo is None and not allVOsFlag: + result = getVOfromProxyGroup() + if not result['OK']: + gLogger.error('Failed to determine the user VO') + DIRACexit(1) + vo = result['Value'] + + for se, statusDict in res['Value'].items(): + + # Check if the SE is allowed for the user VO + if not allVOsFlag: + voList = gConfig.getValue('/Resources/StorageElements/%s/VO' % se, []) + if noVOFlag and voList: + continue + if voList and vo not in voList: + continue + + record = [se] + for status in fields[1:]: + value = statusDict.get(status, 'Unknown') + record.append(value) + records.append(record) + + printTable(fields, records, numbering=False, sortField='SE') + + DIRACexit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py new file mode 100755 index 00000000000..23ce9e99821 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +######################################################################## +# $HeadURL$ +######################################################################## +""" +Get the list of all the user files. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + days = 0 + months = 0 + years = 0 + wildcard = None + baseDir = '' + emptyDirsFlag = False + Script.registerSwitch("D:", "Days=", "Match files older than number of days [%s]" % days) + Script.registerSwitch("M:", "Months=", "Match files older than number of months [%s]" % months) + Script.registerSwitch("Y:", "Years=", "Match files older than number of years [%s]" % years) + Script.registerSwitch("w:", "Wildcard=", "Wildcard for matching filenames [All]") + Script.registerSwitch("b:", "BaseDir=", "Base directory to begin search (default /[vo]/user/[initial]/[username])") + Script.registerSwitch("e", "EmptyDirs", "Create a list of empty directories") + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % Script.scriptName, ])) + + Script.parseCommandLine(ignoreErrors=False) + + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "D" or switch[0].lower() == "days": + days = int(switch[1]) + if switch[0] == "M" or switch[0].lower() == "months": + months = int(switch[1]) + if switch[0] == "Y" or switch[0].lower() == "years": + years = int(switch[1]) + if switch[0].lower() == "w" or switch[0].lower() == "wildcard": + wildcard = '*' + switch[1] + if switch[0].lower() == "b" or switch[0].lower() == "basedir": + baseDir = switch[1] + if switch[0].lower() == "e" or switch[0].lower() == "emptydirs": + emptyDirsFlag = True + + import DIRAC + from DIRAC import gLogger + from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + from datetime import datetime, timedelta + import sys + import os + import time + import fnmatch + fc = FileCatalog() + + def isOlderThan(cTimeStruct, days): + timeDelta = timedelta(days=days) + maxCTime = datetime.utcnow() - timeDelta + if cTimeStruct < maxCTime: + return True + return False + + withMetadata = False + if days or months or years: + withMetadata = True + totalDays = 0 + if years: + totalDays += 365 * years + if months: + totalDays += 30 * months + if days: + totalDays += days + + res = getProxyInfo(False, False) + if not res['OK']: + gLogger.error("Failed to get client proxy information.", res['Message']) + DIRAC.exit(2) + proxyInfo = res['Value'] + if proxyInfo['secondsLeft'] == 0: + gLogger.error("Proxy expired") + DIRAC.exit(2) + username = proxyInfo['username'] + vo = '' + if 'group' in proxyInfo: + vo = getVOForGroup(proxyInfo['group']) + if not baseDir: + if not vo: + gLogger.error('Could not determine VO') + Script.showHelp() + baseDir = '/%s/user/%s/%s' % (vo, username[0], username) + + baseDir = baseDir.rstrip('/') + + gLogger.notice('Will search for files in %s%s' % (baseDir, (' matching %s' % wildcard) if wildcard else '')) + activeDirs = [baseDir] + + allFiles = [] + emptyDirs = [] + + while len(activeDirs) > 0: + currentDir = activeDirs.pop() + res = fc.listDirectory(currentDir, withMetadata, timeout=360) + if not res['OK']: + gLogger.error("Error retrieving directory contents", "%s %s" % (currentDir, res['Message'])) + elif currentDir in res['Value']['Failed']: + gLogger.error("Error retrieving directory contents", "%s %s" % (currentDir, res['Value']['Failed'][currentDir])) + else: + dirContents = res['Value']['Successful'][currentDir] + subdirs = dirContents['SubDirs'] + files = dirContents['Files'] + if not subdirs and not files: + emptyDirs.append(currentDir) + gLogger.notice('%s: empty directory' % currentDir) + else: + for subdir in sorted(subdirs, reverse=True): + if (not withMetadata) or isOlderThan(subdirs[subdir]['CreationDate'], totalDays): + activeDirs.append(subdir) + for filename in sorted(files): + fileOK = False + if (not withMetadata) or isOlderThan(files[filename]['MetaData']['CreationDate'], totalDays): + if wildcard is None or fnmatch.fnmatch(filename, wildcard): + fileOK = True + if not fileOK: + files.pop(filename) + allFiles += sorted(files) + + if len(files) or len(subdirs): + gLogger.notice("%s: %d files%s, %d sub-directories" % + (currentDir, len(files), ' matching' if withMetadata or wildcard else '', len(subdirs))) + + outputFileName = '%s.lfns' % baseDir.replace('/%s' % vo, '%s' % vo).replace('/', '-') + outputFile = open(outputFileName, 'w') + for lfn in sorted(allFiles): + outputFile.write(lfn + '\n') + outputFile.close() + gLogger.notice('%d matched files have been put in %s' % (len(allFiles), outputFileName)) + + if emptyDirsFlag: + outputFileName = '%s.emptydirs' % baseDir.replace('/%s' % vo, '%s' % vo).replace('/', '-') + outputFile = open(outputFileName, 'w') + for dir in sorted(emptyDirs): + outputFile.write(dir + '\n') + outputFile.close() + gLogger.notice('%d empty directories have been put in %s' % (len(emptyDirs), outputFileName)) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py new file mode 100755 index 00000000000..968e8dd7012 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Get the currently defined user data volume quotas + + Usage: + %s [options] + """ % Script.scriptName) + + Script.parseCommandLine(ignoreErrors=False) + + import DIRAC + from DIRAC import gLogger, gConfig + from DIRAC.Core.Security.ProxyInfo import getProxyInfo + + res = getProxyInfo(False, False) + if not res['OK']: + gLogger.error("Failed to get client proxy information.", res['Message']) + DIRAC.exit(2) + proxyInfo = res['Value'] + username = proxyInfo['username'] + + try: + quota = gConfig.getValue('/Registry/DefaultStorageQuota', 0.) + quota = gConfig.getValue('/Registry/Users/%s/Quota' % username, quota) + gLogger.notice('Current quota found to be %.1f GB' % quota) + DIRAC.exit(0) + except Exception as x: + gLogger.exception("Failed to convert retrieved quota", '', x) + DIRAC.exit(-1) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py index 2452b6ad430..f21e897bebc 100644 --- a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py +++ b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py @@ -1914,11 +1914,11 @@ def installComponent(self, componentType, system, component, extensions, compone [[ "%(componentType)s" = "agent" ]] && renice 20 -p $$ #%(bashVariables)s # -exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py \ +exec python $DIRAC/DIRAC/Core/scripts/dirac_%(componentType)s.py \ %(system)s/%(component)s --cfg %(componentCfg)s < /dev/null """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), 'bashVariables': bashVars, - 'componentType': componentType, + 'componentType': componentType.replace("-", "_"), 'system': system, 'component': component, 'componentCfg': componentCfg}) @@ -2541,7 +2541,7 @@ def installTornado(self): exec 2>&1 # # - exec python $DIRAC/DIRAC/Core/Tornado/scripts/tornado-start-all.py -ddd + exec python $DIRAC/DIRAC/Core/Tornado/scripts/tornado_start_all.py -ddd """ % {'bashrc': os.path.join(self.instancePath, 'bashrc')}) os.chmod(runFile, self.gDefaultPerms) diff --git a/src/DIRAC/FrameworkSystem/scripts/__init__.py b/src/DIRAC/FrameworkSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-get-CAs.py b/src/DIRAC/FrameworkSystem/scripts/dirac-admin-get-CAs.py deleted file mode 100755 index 167c9680d9d..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-get-CAs.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-get-CAs -# Author : Ricardo Graciani -######################################################################## -"""Refresh the local copy of the CA certificates and revocation lists. - -Connects to the BundleDelivery service to obtain the tar balls. Needed when proxies appear to be -invalid. - -Usage: - - dirac-admin-get-CAs (|)* -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import DIRAC -from DIRAC.Core.Base import Script -from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient - -__RCSID__ = "$Id$" -Script.addDefaultOptionValue('/DIRAC/Security/SkipCAChecks', 'yes') -Script.setUsageMessage(__doc__) - -Script.parseCommandLine(ignoreErrors=True) - -bdc = BundleDeliveryClient() - -result = bdc.syncCAs() -if not result['OK']: - DIRAC.gLogger.error("Error while updating CAs", result['Message']) - DIRAC.exit(1) -elif result['Value']: - DIRAC.gLogger.notice("CAs got updated") -else: - DIRAC.gLogger.notice("CAs are already synchronized") - -result = bdc.syncCRLs() -if not result['OK']: - DIRAC.gLogger.error("Error while updating CRLs", result['Message']) - DIRAC.exit(1) -elif result['Value']: - DIRAC.gLogger.notice("CRLs got updated") -else: - DIRAC.gLogger.notice("CRLs are already synchronized") - -DIRAC.exit(0) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-get-proxy.py b/src/DIRAC/FrameworkSystem/scripts/dirac-admin-get-proxy.py deleted file mode 100755 index 444654d12c2..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-get-proxy.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-get-proxy -# Author : Stuart Paterson -######################################################################## -""" - Retrieve a delegated proxy for the given user and group -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -import os -import DIRAC - -from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Core.Base import Script -from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager -from DIRAC.ConfigurationSystem.Client.Helpers import Registry - -__RCSID__ = "$Id$" - - -class Params(object): - - limited = False - proxyPath = False - proxyLifeTime = 86400 - enableVOMS = False - vomsAttr = None - - def setLimited(self, args): - """ Set limited - - :param bool args: is limited - - :return: S_OK()/S_ERROR() - """ - self.limited = True - return S_OK() - - def setProxyLocation(self, args): - """ Set proxy location - - :param str args: proxy path - - :return: S_OK()/S_ERROR() - """ - self.proxyPath = args - return S_OK() - - def setProxyLifeTime(self, arg): - """ Set proxy lifetime - - :param int arg: lifetime in a seconds - - :return: S_OK()/S_ERROR() - """ - try: - fields = [f.strip() for f in arg.split(":")] - self.proxyLifeTime = int(fields[0]) * 3600 + int(fields[1]) * 60 - except BaseException: - gLogger.notice("Can't parse %s time! Is it a HH:MM?" % arg) - return S_ERROR("Can't parse time argument") - return S_OK() - - def automaticVOMS(self, arg): - """ Enable VOMS - - :param bool arg: enable VOMS - - :return: S_OK()/S_ERROR() - """ - self.enableVOMS = True - return S_OK() - - def setVOMSAttr(self, arg): - """ Register CLI switches - - :param str arg: VOMS attribute - """ - self.enableVOMS = True - self.vomsAttr = arg - return S_OK() - - def registerCLISwitches(self): - """ Register CLI switches - """ - Script.registerSwitch("v:", "valid=", "Valid HH:MM for the proxy. By default is 24 hours", self.setProxyLifeTime) - Script.registerSwitch("l", "limited", "Get a limited proxy", self.setLimited) - Script.registerSwitch("u:", "out=", "File to write as proxy", self.setProxyLocation) - Script.registerSwitch("a", "voms", "Get proxy with VOMS extension mapped to the DIRAC group", self.automaticVOMS) - Script.registerSwitch("m:", "vomsAttr=", "VOMS attribute to require", self.setVOMSAttr) - - -params = Params() -params.registerCLISwitches() - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... group' % Script.scriptName, - 'Arguments:', - ' DN: DN of the user', - ' user: DIRAC user name (will fail if there is more than 1 DN registered)', - ' group: DIRAC group name'])) - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) != 2: - Script.showHelp() - -userGroup = str(args[1]) -userDN = str(args[0]) -userName = False -if userDN.find("/") != 0: - userName = userDN - retVal = Registry.getDNForUsername(userName) - if not retVal['OK']: - gLogger.notice("Cannot discover DN for username %s\n\t%s" % (userName, retVal['Message'])) - DIRAC.exit(2) - DNList = retVal['Value'] - if len(DNList) > 1: - gLogger.notice("Username %s has more than one DN registered" % userName) - ind = 0 - for dn in DNList: - gLogger.notice("%d %s" % (ind, dn)) - ind += 1 - inp = raw_input("Which DN do you want to download? [default 0] ") - if not inp: - inp = 0 - else: - inp = int(inp) - userDN = DNList[inp] - else: - userDN = DNList[0] - -if not params.proxyPath: - if not userName: - result = Registry.getUsernameForDN(userDN) - if not result['OK']: - gLogger.notice("DN '%s' is not registered in DIRAC" % userDN) - DIRAC.exit(2) - userName = result['Value'] - params.proxyPath = "%s/proxy.%s.%s" % (os.getcwd(), userName, userGroup) - -if params.enableVOMS: - result = gProxyManager.downloadVOMSProxy(userDN, userGroup, limited=params.limited, - requiredTimeLeft=params.proxyLifeTime, - requiredVOMSAttribute=params.vomsAttr) -else: - result = gProxyManager.downloadProxy(userDN, userGroup, limited=params.limited, - requiredTimeLeft=params.proxyLifeTime) -if not result['OK']: - gLogger.notice('Proxy file cannot be retrieved: %s' % result['Message']) - DIRAC.exit(2) -chain = result['Value'] -result = chain.dumpAllToFile(params.proxyPath) -if not result['OK']: - gLogger.notice('Proxy file cannot be written to %s: %s' % (params.proxyPath, result['Message'])) - DIRAC.exit(2) -gLogger.notice("Proxy downloaded to %s" % params.proxyPath) -DIRAC.exit(0) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-sysadmin-cli.py b/src/DIRAC/FrameworkSystem/scripts/dirac-admin-sysadmin-cli.py deleted file mode 100755 index 86394b7a93a..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-sysadmin-cli.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from DIRAC.Core.Base import Script - -__RCSID__ = "$Id$" - -host = None -Script.registerSwitch("H:", "host=", " Target host") -Script.parseCommandLine(ignoreErrors=False) -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "h" or switch[0].lower() == "host": - host = switch[1] - -from DIRAC.FrameworkSystem.Client.SystemAdministratorClientCLI import SystemAdministratorClientCLI - -cli = SystemAdministratorClientCLI(host) -cli.cmdloop() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-update-instance.py b/src/DIRAC/FrameworkSystem/scripts/dirac-admin-update-instance.py deleted file mode 100755 index 6ffdb2e3c3c..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-update-instance.py +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env python -""" -Script to apply update to all or some dirac servers and restart them -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -from io import open - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s version' % Script.scriptName, - ' ', - 'Arguments:', - ' version: version of DIRAC you want to update to'] - ) - ) -Script.registerSwitch("", "hosts=", "Comma separated list of hosts or file containing row wise list of hosts" - " targeted for update (leave empty for all)") -Script.registerSwitch("", "retry=", "Number of retry attempts on hosts that have failed to update") - -Script.parseCommandLine(ignoreErrors=False) - -args = Script.getPositionalArgs() -if len(args) < 1 or len(args) > 2: - Script.showHelp() - -version = args[0] -retry = 0 -hosts = [] - -for switch in Script.getUnprocessedSwitches(): - if switch[0] == "hosts": - hosts = switch[1] - if switch[0] == "retry": - retry = int(switch[1]) - -try: - with open(hosts, 'r') as f: - hosts = f.read().splitlines() - hosts = [str(host) for host in hosts] -except Exception: - pass - -if not isinstance(hosts, list): - hosts = hosts.split(',') - -from concurrent.futures import ThreadPoolExecutor, as_completed - -from DIRAC import S_OK, S_ERROR -from DIRAC import gLogger -from DIRAC.FrameworkSystem.Client.SystemAdministratorClient import SystemAdministratorClient -from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient - - -def getListOfHosts(): - """ - Return the list of all hosts that constitute a DIRAC instance - """ - client = ComponentMonitoringClient() - result = client.getHosts({}, False, False) - if result['OK']: - hosts = [host['HostName'] for host in result['Value']] - return S_OK(hosts) - return S_ERROR('Cannot get list of hosts: %s' % result['Message']) - - -def parseHostname(hostName): - """ - Separate the hostname from the port - - :param str hostName: hostname you want to parse - """ - hostList = hostName.split(':') - host = hostList[0] - if len(hostList) == 2: - port = hostList[1] - else: - port = None - return host, port - - -def updateHost(hostName, version): - """ - Apply update to specific host - - :param str hostName: name of the host you want to update - :param str version: version vArBpC you want to update to - """ - host, port = parseHostname(hostName) - - client = SystemAdministratorClient(host, port) - result = client.ping() - if not result['OK']: - gLogger.error("Cannot connect to %s" % host) - return result - - gLogger.notice("Initiating software update of %s, this can take a while, please be patient ..." % host) - result = client.updateSoftware(version, '', '', timeout=600) - if not result['OK']: - return result - return S_OK() - - -def updateHosts(version, hosts=[]): - """ - Apply update to all hosts - - :param str version: version vArBpC you want to update to - :param list[str] hosts: list of hosts to be updated, defaults to [] - """ - if not hosts: - result = getListOfHosts() - if not result['OK']: - return result - hosts = result['Value'] - - updateSuccess = [] - updateFail = [] - - executor = ThreadPoolExecutor(max_workers=len(hosts)) - futureUpdate = {executor.submit(updateHost, host, version): host for host in hosts} - for future in as_completed(futureUpdate): - host = futureUpdate[future] - result = future.result() - if result['OK']: - updateSuccess.append(host) - else: - updateFail.append(host) - - if not updateFail: - gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - gLogger.notice("!!! Successfully updated all hosts !!!") - gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - return S_OK([updateSuccess, updateFail]) - if not updateSuccess: - gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") - gLogger.notice("XXXXX Failed to update all hosts XXXXX") - gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") - return S_OK([updateSuccess, updateFail]) - gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!") - gLogger.notice("X!X!X Partially updated hosts X!X!X!") - gLogger.notice("Succeeded to update:") - for host in updateSuccess: - gLogger.notice(" + %s" % host) - gLogger.notice("Failed to update:") - for host in updateFail: - gLogger.notice(" - %s" % host) - gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!") - return S_OK([updateSuccess, updateFail]) - - -def restartHost(hostName): - """ - Restart all systems and components of a host - - :param str hostName: name of the host you want to restart - """ - host, port = parseHostname(hostName) - - gLogger.notice("Pinging %s ..." % host) - - client = SystemAdministratorClient(host, port) - result = client.ping() - if not result['OK']: - gLogger.error("Could not connect to %s: %s" % (host, result['Message'])) - return result - gLogger.notice("Host %s is active" % host) - - gLogger.notice("Initiating restart of all systems and components") - # This restart call will always return S_ERROR because of SystemAdministrator restart - # Connection will be lost to the host - result = client.restartComponent('*', '*') - if result['Message'] == "Peer closed connection": - gLogger.notice("Restarted all systems on %s : connection to SystemAdministrator lost" % host) - return S_OK(result['Message']) - gLogger.error("Received unxpected message: %s" % result['Message']) - return result - - -def updateInstance(version, hosts, retry): - """ - Update each server of an instance and restart them - - :param str version: version vArBpC you want to update to - :param list[str] hosts: list of hosts to be updated - :param int retry: number of retry attempts on hosts that have failed to update - """ - result = updateHosts(version, hosts) - if not result['OK']: - return result - - updateSuccess = result['Value'][0] - updateFail = result['Value'][1] - restartSuccess = [] - restartFail = [] - for host in updateSuccess: - result = restartHost(host) - if result['OK']: - restartSuccess.append(host) - else: - restartFail.append(host) - - if not restartFail and not updateFail: - return S_OK("Successfully updated and restarted all hosts") - - gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") - gLogger.notice("XXXXX There were problems in the update process XXXXX") - gLogger.notice("Succeeded to update:") - for host in updateSuccess: - gLogger.notice(" + %s" % host) - gLogger.notice("Succeeded to restart:") - for host in restartSuccess: - gLogger.notice(" + %s" % host) - gLogger.notice("Failed to update:") - for host in updateFail: - gLogger.notice(" - %s" % host) - gLogger.notice("Failed to restart:") - for host in restartFail: - gLogger.notice(" - %s" % host) - gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") - - if retry > 0: - retryHosts = list(set(updateFail + restartFail)) - gLogger.notice("Retrying update on (%s attempts remaining):" % retry) - for host in retryHosts: - gLogger.notice(" - %s" % host) - gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") - return updateInstance(version, retryHosts, retry - 1) - - return S_ERROR("Update failed!") - - -if __name__ == "__main__": - result = updateInstance(version, hosts, retry) - if not result['OK']: - gLogger.fatal(result['Message']) - DIRAC.exit(1) - gLogger.notice(result['Value']) - DIRAC.exit(0) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-update-pilot.py b/src/DIRAC/FrameworkSystem/scripts/dirac-admin-update-pilot.py deleted file mode 100755 index cf07745ffb6..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-update-pilot.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -""" -Script to update pilot version in CS -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s version' % Script.scriptName, - ' ', - 'Arguments:', - ' version: pilot version you want to update to'] - ) - ) -Script.registerSwitch( - "v:", "vo=", "Location of pilot version in CS /Operations//Pilot/Version" - " (default value specified in CS under /DIRAC/DefaultSetup)") - -Script.parseCommandLine(ignoreErrors=False) - -args = Script.getPositionalArgs() -if len(args) < 1 or len(args) > 2: - Script.showHelp() - -version = args[0] -vo = None -for switch in Script.getUnprocessedSwitches(): - if switch[0] == "v" or switch[0] == "vo": - vo = switch[1] - -from DIRAC import S_OK, S_ERROR -from DIRAC import gConfig, gLogger -from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI - - -def updatePilot(version, vo): - """ - Update in the CS the pilot version used, - If only one version present in CS it's overwritten. - If two versions present, the new one is added and the last removed - - :param version: version vArBpC of pilot you want to use - :param vo: Location of pilot version in CS /Operations//Pilot/Version - """ - setup = vo - if not vo: - setup = gConfig.getValue('/DIRAC/DefaultSetup') - if not setup: - return S_ERROR("No value set for /DIRAC/DefaultSetup in CS") - - pilotVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup, []) - if not pilotVersion: - return S_ERROR("No pilot version set under Operations/%s/Pilot/Version in CS" % setup) - - pilotVersion.pop() - pilotVersion.insert(0, version) - api = CSAPI() - api.setOption('Operations/%s/Pilot/Version' % setup, ", ".join(pilotVersion)) - result = api.commit() - if not result['OK']: - gLogger.fatal('Could not commit new version of pilot!') - return result - - newVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup) - return S_OK("New version of pilot set to %s" % newVersion) - - -if __name__ == "__main__": - result = updatePilot(version, vo) - if not result['OK']: - gLogger.fatal(result['Message']) - DIRAC.exit(1) - gLogger.notice(result['Value']) - DIRAC.exit(0) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-users-with-proxy.py b/src/DIRAC/FrameworkSystem/scripts/dirac-admin-users-with-proxy.py deleted file mode 100755 index fe1511412d6..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-users-with-proxy.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -import DIRAC -from DIRAC.Core.Base import Script -from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager -from DIRAC.Core.Utilities import Time - -__RCSID__ = "$Id$" - - -class Params(object): - - limited = False - proxyPath = False - proxyLifeTime = 3600 - - def setProxyLifeTime(self, arg): - try: - fields = [f.strip() for f in arg.split(":")] - self.proxyLifeTime = int(fields[0]) * 3600 + int(fields[1]) * 60 - except BaseException: - print("Can't parse %s time! Is it a HH:MM?" % arg) - return DIRAC.S_ERROR("Can't parse time argument") - return DIRAC.S_OK() - - def registerCLISwitches(self): - Script.registerSwitch("v:", "valid=", "Required HH:MM for the users", self.setProxyLifeTime) - - -params = Params() -params.registerCLISwitches() - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -result = gProxyManager.getDBContents() -if not result['OK']: - print("Can't retrieve list of users: %s" % result['Message']) - DIRAC.exit(1) - -keys = result['Value']['ParameterNames'] -records = result['Value']['Records'] -dataDict = {} -now = Time.dateTime() -for record in records: - expirationDate = record[3] - dt = expirationDate - now - secsLeft = dt.days * 86400 + dt.seconds - if secsLeft > params.proxyLifeTime: - userName, userDN, userGroup, _, persistent = record - if userName not in dataDict: - dataDict[userName] = [] - dataDict[userName].append((userDN, userGroup, expirationDate, persistent)) - - -for userName in dataDict: - print("* %s" % userName) - for iP in range(len(dataDict[userName])): - data = dataDict[userName][iP] - print(" DN : %s" % data[0]) - print(" group : %s" % data[1]) - print(" not after : %s" % Time.toString(data[2])) - print(" persistent : %s" % data[3]) - if iP < len(dataDict[userName]) - 1: - print(" -") - - -DIRAC.exit(0) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-install-component.py b/src/DIRAC/FrameworkSystem/scripts/dirac-install-component.py deleted file mode 100755 index de62a892638..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-install-component.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python -""" -Do the initial installation and configuration of a DIRAC component -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from DIRAC import gConfig, gLogger, S_OK -from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions -from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities -from DIRAC.Core.Base import Script -from DIRAC import exit as DIRACexit -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller - -__RCSID__ = "$Id$" - -gComponentInstaller.exitOnError = True - -overwrite = False - - -def setOverwrite(opVal): - global overwrite - overwrite = True - return S_OK() - - -module = '' -specialOptions = {} - - -def setModule(optVal): - global specialOptions, module - specialOptions['Module'] = optVal - module = optVal - return S_OK() - - -def setSpecialOption(optVal): - global specialOptions - option, value = optVal.split('=') - specialOptions[option] = value - return S_OK() - - -Script.registerSwitch("w", "overwrite", "Overwrite the configuration in the global CS", setOverwrite) -Script.registerSwitch("m:", "module=", "Python module name for the component code", setModule) -Script.registerSwitch("p:", "parameter=", "Special component option ", setSpecialOption) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, - 'Arguments:', - ' System: Name of the DIRAC system (ie: WorkloadManagement)', - ' Service: Name of the DIRAC component (ie: Matcher)'])) - -Script.parseCommandLine() -args = Script.getPositionalArgs() - -if len(args) == 1: - args = args[0].split('/') - -if len(args) != 2: - Script.showHelp(exitCode=1) - -cType = None -system = args[0] -component = args[1] -compOrMod = module if module else component - - -result = gComponentInstaller.getSoftwareComponents(getCSExtensions()) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) -else: - availableComponents = result['Value'] - -for compType in availableComponents: - if system in availableComponents[compType] and compOrMod in availableComponents[compType][system]: - cType = compType[:-1].lower() - break - -if not cType: - gLogger.error('Component %s/%s is not available for installation' % (system, component)) - DIRACexit(1) - -if module: - result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, module, - getCSExtensions(), - overwrite=overwrite) - result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, component, - getCSExtensions(), - specialOptions=specialOptions, - overwrite=overwrite, - addDefaultOptions=False) -else: - result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, component, - getCSExtensions(), - specialOptions=specialOptions, - overwrite=overwrite) - -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) -else: - result = gComponentInstaller.installComponent(cType, system, component, getCSExtensions(), module) - if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - else: - gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) - result = gComponentInstaller.setupComponent(cType, system, component, getCSExtensions(), module) - if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - if component == 'ComponentMonitoring': - result = MonitoringUtilities.monitorInstallation('DB', system, 'InstalledComponentsDB') - if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - result = MonitoringUtilities.monitorInstallation(cType, system, component, module) - if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - gLogger.notice('Successfully completed the installation of %s/%s' % (system, component)) - DIRACexit() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-install-tornado-service.py b/src/DIRAC/FrameworkSystem/scripts/dirac-install-tornado-service.py deleted file mode 100755 index f857fb196b4..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-install-tornado-service.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python -""" -Do the initial installation and configuration of a DIRAC service based on tornado -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -from DIRAC import gConfig, gLogger, S_OK -from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions -from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities -from DIRAC.Core.Base import Script -from DIRAC import exit as DIRACexit -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller - -__RCSID__ = "$Id$" - -gComponentInstaller.exitOnError = True - -overwrite = False - - -def setOverwrite(opVal): - global overwrite - overwrite = True - return S_OK() - - -module = '' -specialOptions = {} - - -def setModule(optVal): - global specialOptions, module - specialOptions['Module'] = optVal - module = optVal - return S_OK() - - -def setSpecialOption(optVal): - global specialOptions - option, value = optVal.split('=') - specialOptions[option] = value - return S_OK() - - -Script.registerSwitch("w", "overwrite", "Overwrite the configuration in the global CS", setOverwrite) -Script.registerSwitch("m:", "module=", "Python module name for the component code", setModule) -Script.registerSwitch("p:", "parameter=", "Special component option ", setSpecialOption) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, - 'Arguments:', - ' System: Name of the DIRAC system (ie: WorkloadManagement)', - ' Service: Name of the DIRAC component (ie: Matcher)'])) - -Script.parseCommandLine() -args = Script.getPositionalArgs() - -if len(args) == 1: - args = args[0].split('/') - -if len(args) != 2: - Script.showHelp() - DIRACexit(1) - -system = args[0] -component = args[1] -compOrMod = module if module else component - - -result = gComponentInstaller.addDefaultOptionsToCS(gConfig, 'service', system, component, - getCSExtensions(), - specialOptions=specialOptions, - overwrite=overwrite) - -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - - -result = gComponentInstaller.addTornadoOptionsToCS(gConfig) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - -result = gComponentInstaller.installTornado() -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - - -gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) -result = gComponentInstaller.setupTornadoService(system, component, getCSExtensions(), module) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - -result = MonitoringUtilities.monitorInstallation('service', system, component, module) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) -gLogger.notice('Successfully completed the installation of %s/%s' % (system, component)) -DIRACexit() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-monitoring-get-components-status.py b/src/DIRAC/FrameworkSystem/scripts/dirac-monitoring-get-components-status.py deleted file mode 100755 index 234c627763e..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-monitoring-get-components-status.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -import sys -from DIRAC.Core.Base import Script - -__RCSID__ = "$Id$" - - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -fieldsToShow = ('ComponentName', 'Type', 'Host', 'Port', 'Status', 'Message') - -from DIRAC.FrameworkSystem.Client.MonitoringClient import gMonitor - -result = gMonitor.getComponentsStatusWebFormatted(sortingList=[['ComponentName', 'ASC']]) -if not result['OK']: - print("ERROR: %s" % result['Message']) - sys.exit(1) -paramNames = result['Value']['ParameterNames'] -records = result['Value']['Records'] -fieldLengths = [] -for param in paramNames: - fieldLengths.append(len(param)) - -for record in records: - for i in range(len(record)): - if paramNames[i] in fieldsToShow: - fieldLengths[i] = max(fieldLengths[i], len(str(record[i]))) -# Print time! -line = [] -sepLine = [] -for i in range(len(paramNames)): - param = paramNames[i] - if param in fieldsToShow: - line.append("%s%s" % (param, " " * (fieldLengths[i] - len(param)))) - sepLine.append("-" * fieldLengths[i]) -print("|".join(line)) -sepLine = "+".join(sepLine) -print(sepLine) -for record in records: - line = [] - for i in range(len(record)): - if paramNames[i] in fieldsToShow: - val = str(record[i]) - line.append("%s%s" % (val, " " * (fieldLengths[i] - len(val)))) - print("|".join(line)) - # print sepLine diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-myproxy-upload.py b/src/DIRAC/FrameworkSystem/scripts/dirac-myproxy-upload.py deleted file mode 100755 index 2f5313e2f09..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-myproxy-upload.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-proxy-init.py -# Author : Adrian Casajus -######################################################################## - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -import sys -import DIRAC -from DIRAC.Core.Base import Script - - -class Params: - - proxyLoc = False - dnAsUsername = False - - def setProxyLocation(self, arg): - self.proxyLoc = arg - return DIRAC.S_OK() - - def setDNAsUsername(self, arg): - self.dnAsUsername = True - return DIRAC.S_OK() - - def showVersion(self, arg): - print("Version:") - print(" ", __RCSID__) - sys.exit(0) - return DIRAC.S_OK() - - -params = Params() - -Script.registerSwitch("f:", "file=", "File to use as proxy", params.setProxyLocation) -Script.registerSwitch("D", "DN", "Use DN as myproxy username", params.setDNAsUsername) -Script.registerSwitch("i", "version", "Print version", params.showVersion) - -Script.addDefaultOptionValue("LogLevel", "always") -Script.parseCommandLine() - -from DIRAC.Core.Security.MyProxy import MyProxy -from DIRAC.Core.Security import Locations - -if not params.proxyLoc: - params.proxyLoc = Locations.getProxyLocation() - -if not params.proxyLoc: - print("Can't find any valid proxy") - sys.exit(1) -print("Uploading proxy file %s" % params.proxyLoc) - -mp = MyProxy() -retVal = mp.uploadProxy(params.proxyLoc, params.dnAsUsername) -if not retVal['OK']: - print("Can't upload proxy:") - print(" ", retVal['Message']) - sys.exit(1) -print("Proxy uploaded") -sys.exit(0) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-populate-component-db.py b/src/DIRAC/FrameworkSystem/scripts/dirac-populate-component-db.py deleted file mode 100755 index 672dbb441c3..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-populate-component-db.py +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env python - -""" -Populates the database with the current installations of components -This script assumes that the InstalledComponentsDB, the -ComponentMonitoring service and the Notification service are installed and running -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -from datetime import datetime -from DIRAC import exit as DIRACexit -from DIRAC import S_OK, gLogger, gConfig -from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI -from DIRAC.Core.Base import Script -from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient -from DIRAC.FrameworkSystem.Client.SystemAdministratorIntegrator \ - import SystemAdministratorIntegrator -from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient \ - import ComponentMonitoringClient -from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.Core.Security.ProxyInfo import getProxyInfo - -__RCSID__ = "$Id$" - -global excludedHosts -excludedHosts = [] - - -def setExcludedHosts(value): - global excludedHosts - - excludedHosts = value.split(',') - return S_OK() - - -Script.registerSwitch( - "e:", - "exclude=", - "Comma separated list of hosts to be excluded from the scanning process", - setExcludedHosts) - - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... [debug]' % Script.scriptName])) - -Script.parseCommandLine(ignoreErrors=False) -args = Script.getPositionalArgs() - -componentType = '' - -# Get my setup -mySetup = gConfig.getValue('DIRAC/Setup') - -# Retrieve information from all the hosts -client = SystemAdministratorIntegrator(exclude=excludedHosts) -resultAll = client.getOverallStatus() - -# Retrieve user installing the component -result = getProxyInfo() -if result['OK']: - user = result['Value']['username'] -else: - DIRACexit(-1) -if not user: - user = 'unknown' - -notificationClient = NotificationClient() -for host in resultAll['Value']: - if not resultAll['Value'][host]['OK']: - # If the host cannot be contacted, exclude it and send message - excludedHosts.append(host) - - result = notificationClient.sendMail( - Operations().getValue( - 'EMail/Production', - []), - 'Unreachable host', - '\ndirac-populate-component-db: Could not fill the database with the components from unreachable host %s\n' % - host) - if not result['OK']: - gLogger.error('Can not send unreachable host notification mail: %s' % result['Message']) - -if not resultAll['OK']: - gLogger.error(resultAll['Message']) - DIRACexit(-1) -resultHosts = client.getHostInfo() -if not resultHosts['OK']: - gLogger.error(resultHosts['Message']) - DIRACexit(-1) -resultInfo = client.getInfo() -if not resultInfo['OK']: - gLogger.error(resultInfo['Message']) - DIRACexit(-1) -resultMySQL = client.getMySQLStatus() -if not resultMySQL['OK']: - gLogger.error(resultMySQL['Message']) - DIRACexit(-1) -resultAllDB = client.getDatabases() -if not resultAllDB['OK']: - gLogger.error(resultAllDB['Message']) - DIRACexit(-1) -resultAvailableDB = client.getAvailableDatabases() -if not resultAvailableDB['OK']: - gLogger.error(resultAvailableDB['Message']) - DIRACexit(-1) - -records = [] -finalSet = list(set(resultAll['Value']) - set(excludedHosts)) -for host in finalSet: - hasMySQL = True - result = resultAll['Value'][host] - hostResult = resultHosts['Value'][host] - infoResult = resultInfo['Value'][host] - mySQLResult = resultMySQL['Value'][host] - allDBResult = resultAllDB['Value'][host] - availableDBResult = resultAvailableDB['Value'][host] - - if not result['OK']: - gLogger.error('Host %s: %s' % (host, result['Message'])) - continue - if not hostResult['OK']: - gLogger.error('Host %s: %s' % (host, hostResult['Message'])) - continue - if not infoResult['OK']: - gLogger.error('Host %s: %s' % (host, infoResult['Message'])) - continue - if mySQLResult['OK']: - if not allDBResult['OK']: - gLogger.error('Host %s: %s' % (host, allDBResult['Message'])) - continue - if not availableDBResult['OK']: - gLogger.error('Host %s: %s' % (host, availableDBResult['Message'])) - continue - else: - hasMySQL = False - - setup = infoResult['Value']['Setup'] - if setup != mySetup: - continue - - cpu = hostResult['Value']['CPUModel'].strip() - rDict = result['Value'] - # Components other than databases - for compType in rDict: - if componentType and componentType != compType: - continue - for system in rDict[compType]: - components = sorted(rDict[compType][system]) - for component in components: - record = {'Installation': {}, 'Component': {}, 'Host': {}} - if rDict[compType][system][component]['Installed'] and \ - component != 'ComponentMonitoring': - runitStatus = \ - str(rDict[compType][system][component]['RunitStatus']) - if runitStatus != 'Unknown': - module = \ - str(rDict[compType][system][component]['Module']) - record['Component']['System'] = system - record['Component']['Module'] = module - # Transform 'Services' into 'service', 'Agents' into 'agent' ... - record['Component']['Type'] = compType.lower()[:-1] - record['Host']['HostName'] = host - record['Host']['CPU'] = cpu - record['Installation']['Instance'] = component - record['Installation']['InstallationTime'] = datetime.utcnow() - record['Installation']['InstalledBy'] = user - records.append(record) - - # Databases - csClient = CSAPI() - cfg = csClient.getCurrentCFG()['Value'] - - if hasMySQL: - allDB = allDBResult['Value'] - availableDB = availableDBResult['Value'] - - for db in allDB: - # Check for DIRAC only databases - if db in availableDB and db != 'InstalledComponentsDB': - # Check for 'installed' databases - isSection = cfg.isSection('Systems/' + availableDB[db]['System'] + '/' + - cfg.getOption('DIRAC/Setups/' + setup + '/' + - availableDB[db]['System']) + '/Databases/' + db + - '/') - if isSection: - record = {'Installation': {}, 'Component': {}, 'Host': {}} - record['Component']['System'] = availableDB[db]['System'] - record['Component']['Module'] = db - record['Component']['Type'] = 'DB' - record['Host']['HostName'] = host - record['Host']['CPU'] = cpu - record['Installation']['Instance'] = db - record['Installation']['InstallationTime'] = datetime.utcnow() - record['Installation']['InstalledBy'] = user - records.append(record) - -monitoringClient = ComponentMonitoringClient() - -# Add the installations to the database -for record in records: - result = MonitoringUtilities.monitorInstallation( - record['Component']['Type'], record['Component']['System'], - record['Installation']['Instance'], record['Component']['Module'], - record['Host']['CPU'], record['Host']['HostName']) - if not result['OK']: - gLogger.error(result['Message']) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-get-uploaded-info.py b/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-get-uploaded-info.py deleted file mode 100755 index 68401aacd5f..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-get-uploaded-info.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-proxy-init.py -# Author : Adrian Casajus -######################################################################## -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -import sys - -from DIRAC import gLogger, S_OK -from DIRAC.Core.Base import Script -from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient -from DIRAC.Core.Security import Properties -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.ConfigurationSystem.Client.Helpers import Registry - -__RCSID__ = "$Id:" - -userName = False - - -def setUser(arg): - """ Set user - - :param str arg: user name - - :return: S_OK() - """ - global userName - userName = arg - return S_OK() - - -Script.registerSwitch("u:", "user=", "User to query (by default oneself)", setUser) - -Script.parseCommandLine() - -result = getProxyInfo() -if not result['OK']: - gLogger.notice("Do you have a valid proxy?") - gLogger.notice(result['Message']) - sys.exit(1) -proxyProps = result['Value'] - -userName = userName or proxyProps.get('username') -if not userName: - gLogger.notice("Your proxy don`t have username extension") - sys.exit(1) - -if userName in Registry.getAllUsers(): - if Properties.PROXY_MANAGEMENT not in proxyProps['groupProperties']: - if userName != proxyProps['username'] and userName != proxyProps['issuer']: - gLogger.notice("You can only query info about yourself!") - sys.exit(1) - result = Registry.getDNForUsername(userName) - if not result['OK']: - gLogger.notice("Oops %s" % result['Message']) - dnList = result['Value'] - if not dnList: - gLogger.notice("User %s has no DN defined!" % userName) - sys.exit(1) - userDNs = dnList -else: - userDNs = [userName] - - -gLogger.notice("Checking for DNs %s" % " | ".join(userDNs)) -pmc = ProxyManagerClient() -result = pmc.getDBContents({'UserDN': userDNs}) -if not result['OK']: - gLogger.notice("Could not retrieve the proxy list: %s" % result['Value']) - sys.exit(1) - -data = result['Value'] -colLengths = [] -for pN in data['ParameterNames']: - colLengths.append(len(pN)) -for row in data['Records']: - for i in range(len(row)): - colLengths[i] = max(colLengths[i], len(str(row[i]))) - -lines = [""] -for i in range(len(data['ParameterNames'])): - pN = data['ParameterNames'][i] - lines[0] += "| %s " % pN.ljust(colLengths[i]) -lines[0] += "|" -tL = len(lines[0]) -lines.insert(0, "-" * tL) -lines.append("-" * tL) -for row in data['Records']: - nL = "" - for i in range(len(row)): - nL += "| %s " % str(row[i]).ljust(colLengths[i]) - nL += "|" - lines.append(nL) - lines.append("-" * tL) - -gLogger.notice("\n".join(lines)) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-info.py b/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-info.py deleted file mode 100755 index 15360b22e71..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-info.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-proxy-info.py -# Author : Adrian Casajus -######################################################################## - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -from DIRAC.Core.Utilities.ReturnValues import S_OK - - -class Params(object): - - proxyLoc = False - vomsEnabled = True - csEnabled = True - steps = False - checkValid = False - checkClock = True - uploadedInfo = False - - def showVersion(self, arg): - print("Version:") - print(" ", __RCSID__) - sys.exit(0) - return S_OK() - - def setProxyLocation(self, arg): - self.proxyLoc = arg - return S_OK() - - def disableVOMS(self, arg): - self.vomsEnabled = False - return S_OK() - - def disableCS(self, arg): - self.csEnabled = False - return S_OK() - - def showSteps(self, arg): - self.steps = True - return S_OK() - - def validityCheck(self, arg): - self.checkValid = True - return S_OK() - - def disableClockCheck(self, arg): - self.checkClock = False - return S_OK() - - def setManagerInfo(self, arg): - self.uploadedInfo = True - return S_OK() - - -params = Params() - -import sys - -from DIRAC.Core.Base import Script -Script.registerSwitch("f:", "file=", "File to use as user key", params.setProxyLocation) -Script.registerSwitch("i", "version", "Print version", params.showVersion) -Script.registerSwitch("n", "novoms", "Disable VOMS", params.disableVOMS) -Script.registerSwitch("v", "checkvalid", "Return error if the proxy is invalid", params.validityCheck) -Script.registerSwitch("x", "nocs", "Disable CS", params.disableCS) -Script.registerSwitch("e", "steps", "Show steps info", params.showSteps) -Script.registerSwitch("j", "noclockcheck", "Disable checking if time is ok", params.disableClockCheck) -Script.registerSwitch("m", "uploadedinfo", "Show uploaded proxies info", params.setManagerInfo) - -Script.disableCS() -Script.parseCommandLine() - -from DIRAC.Core.Utilities.NTP import getClockDeviation -from DIRAC import gLogger -from DIRAC.Core.Security.ProxyInfo import getProxyInfo, getProxyStepsInfo, formatProxyInfoAsString, formatProxyStepsInfoAsString -from DIRAC.Core.Security import VOMS -from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager -from DIRAC.ConfigurationSystem.Client.Helpers import Registry - - -__RCSID__ = "$Id$" - - -if params.csEnabled: - retVal = Script.enableCS() - if not retVal['OK']: - print("Cannot contact CS to get user list") - -if params.checkClock: - result = getClockDeviation() - if result['OK']: - deviation = result['Value'] - if deviation > 600: - gLogger.error("Your host clock seems to be off by more than TEN MINUTES! Thats really bad.") - elif deviation > 180: - gLogger.error("Your host clock seems to be off by more than THREE minutes! Thats bad.") - elif deviation > 60: - gLogger.error("Your host clock seems to be off by more than a minute! Thats not good.") - - -result = getProxyInfo(params.proxyLoc, not params.vomsEnabled) -if not result['OK']: - gLogger.error(result['Message']) - sys.exit(1) -infoDict = result['Value'] -gLogger.notice(formatProxyInfoAsString(infoDict)) -if not infoDict['isProxy']: - gLogger.error('==============================\n!!! The proxy is not valid !!!') - -if params.steps: - gLogger.notice("== Steps extended info ==") - chain = infoDict['chain'] - stepInfo = getProxyStepsInfo(chain)['Value'] - gLogger.notice(formatProxyStepsInfoAsString(stepInfo)) - - -def invalidProxy(msg): - gLogger.error("Invalid proxy:", msg) - sys.exit(1) - - -if params.uploadedInfo: - result = gProxyManager.getUserProxiesInfo() - if not result['OK']: - gLogger.error("Could not retrieve the uploaded proxies info", result['Message']) - else: - uploadedInfo = result['Value'] - if not uploadedInfo: - gLogger.notice("== No proxies uploaded ==") - if uploadedInfo: - gLogger.notice("== Proxies uploaded ==") - maxDNLen = 0 - maxGroupLen = 0 - for userDN in uploadedInfo: - maxDNLen = max(maxDNLen, len(userDN)) - for group in uploadedInfo[userDN]: - maxGroupLen = max(maxGroupLen, len(group)) - gLogger.notice(" %s | %s | Until (GMT)" % ("DN".ljust(maxDNLen), "Group".ljust(maxGroupLen))) - for userDN in uploadedInfo: - for group in uploadedInfo[userDN]: - gLogger.notice(" %s | %s | %s" % (userDN.ljust(maxDNLen), - group.ljust(maxGroupLen), - uploadedInfo[userDN][group].strftime("%Y/%m/%d %H:%M"))) - -if params.checkValid: - if infoDict['secondsLeft'] == 0: - invalidProxy("Proxy is expired") - if params.csEnabled and not infoDict['validGroup']: - invalidProxy("Group %s is not valid" % infoDict['group']) - if 'hasVOMS' in infoDict and infoDict['hasVOMS']: - requiredVOMS = Registry.getVOMSAttributeForGroup(infoDict['group']) - if 'VOMS' not in infoDict or not infoDict['VOMS']: - invalidProxy("Unable to retrieve VOMS extension") - if len(infoDict['VOMS']) > 1: - invalidProxy("More than one voms attribute found") - if requiredVOMS not in infoDict['VOMS']: - invalidProxy("Unexpected VOMS extension %s. Extension expected for DIRAC group is %s" % ( - infoDict['VOMS'][0], - requiredVOMS)) - result = VOMS.VOMS().getVOMSProxyInfo(infoDict['chain'], 'actimeleft') - if not result['OK']: - invalidProxy("Cannot determine life time of VOMS attributes: %s" % result['Message']) - if int(result['Value'].strip()) == 0: - invalidProxy("VOMS attributes are expired") - -sys.exit(0) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-restart-component.py b/src/DIRAC/FrameworkSystem/scripts/dirac-restart-component.py deleted file mode 100755 index 20ab5a0bad4..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-restart-component.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -""" - Restart DIRAC component using runsvctrl utility -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" -# -from DIRAC.Core.Base import Script -Script.disableCS() -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... [System [Service|Agent]]' % Script.scriptName, - 'Arguments:', - ' System: Name of the system for the component (default *: all)', - ' Service|Agent: Name of the particular component (default *: all)'])) -Script.parseCommandLine() -args = Script.getPositionalArgs() -if len(args) > 2: - Script.showHelp(exitCode=1) - -system = '*' -component = '*' -if args: - system = args[0] -if system != '*': - if len(args) > 1: - component = args[1] -# -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller -# -gComponentInstaller.exitOnError = True -# -result = gComponentInstaller.runsvctrlComponent(system, component, 't') -if not result['OK']: - print('ERROR:', result['Message']) - exit(-1) - -gComponentInstaller.printStartupStatus(result['Value']) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-start-component.py b/src/DIRAC/FrameworkSystem/scripts/dirac-start-component.py deleted file mode 100755 index 10276533821..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-start-component.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -""" -Start DIRAC component using runsvctrl utility -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script -Script.disableCS() -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... [system [service|agent]]' % Script.scriptName, - 'Arguments:', - ' system: Name of the system for the component (default *: all)', - ' service|agent: Name of the particular component (default *: all)'])) -Script.parseCommandLine() -args = Script.getPositionalArgs() -if len(args) > 2: - Script.showHelp(exitCode=1) - -system = '*' -component = '*' -if len(args) > 0: - system = args[0] -if system != '*': - if len(args) > 1: - component = args[1] -# -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller -# -gComponentInstaller.exitOnError = True -# -result = gComponentInstaller.runsvctrlComponent(system, component, 'u') -if not result['OK']: - print('ERROR:', result['Message']) - exit(-1) - -gComponentInstaller.printStartupStatus(result['Value']) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-status-component.py b/src/DIRAC/FrameworkSystem/scripts/dirac-status-component.py deleted file mode 100755 index 969a4ee4edf..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-status-component.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -""" -Status of DIRAC components using runsvstat utility -""" -# -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -from DIRAC.Core.Base import Script -Script.disableCS() - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... [system [service|agent]]' % Script.scriptName, - 'Arguments:', - ' system: Name of the system for the component (default *: all)', - ' service|agent: Name of the particular component (default *: all)'])) -Script.parseCommandLine() -args = Script.getPositionalArgs() - -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller - - -__RCSID__ = "$Id$" - -if len(args) > 2: - Script.showHelp(exitCode=1) - -system = '*' -component = '*' -if len(args) > 0: - system = args[0] -if system != '*': - if len(args) > 1: - component = args[1] -# -gComponentInstaller.exitOnError = True -# -result = gComponentInstaller.getStartupComponentStatus([system, component]) -if not result['OK']: - print('ERROR:', result['Message']) - exit(-1) - -gComponentInstaller.printStartupStatus(result['Value']) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-stop-component.py b/src/DIRAC/FrameworkSystem/scripts/dirac-stop-component.py deleted file mode 100755 index 55f2f0a27ec..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-stop-component.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -""" -Do the initial installation and configuration of the DIRAC MySQL server -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from DIRAC.Core.Base import Script -Script.disableCS() -Script.setUsageMessage('\n'.join(['Stop DIRAC component using runsvctrl utility', - 'Usage:', - ' %s [option|cfgfile] ... [system [service|agent]]' % Script.scriptName, - 'Arguments:', - ' system: Name of the system for the component (default *: all)', - ' service|agent: Name of the particular component (default *: all)'])) -Script.parseCommandLine() -args = Script.getPositionalArgs() - -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller - -__RCSID__ = "$Id$" - -if len(args) > 2: - Script.showHelp(exitCode=1) - -system = '*' -component = '*' -if len(args) > 0: - system = args[0] -if system != '*': - if len(args) > 1: - component = args[1] -# -# -gComponentInstaller.exitOnError = True -# -result = gComponentInstaller.runsvctrlComponent(system, component, 'd') -if not result['OK']: - print('ERROR:', result['Message']) - exit(-1) - -gComponentInstaller.printStartupStatus(result['Value']) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-sys-sendmail.py b/src/DIRAC/FrameworkSystem/scripts/dirac-sys-sendmail.py deleted file mode 100755 index dd08dfab72c..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-sys-sendmail.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-sys-sendmail -# Author : Matvey Sapunov -######################################################################## - -""" - Utility to send an e-mail using DIRAC notification service. - - Arguments: - Formated text message. The message consists of e-mail headers and e-mail body - separated by two newline characters. Headers are key : value pairs separated - by newline character. Meaningful headers are "To:", "From:", "Subject:". - Other keys will be ommited. - Message body is an arbitrary string. - - Options: - There are no options. - - Examples: - dirac-sys-sendmail "From: source@email.com\\nTo: destination@email.com\\nSubject: Test\\n\\nMessage body" - echo "From: source@email.com\\nSubject: Test\\n\\nMessage body" | dirac-sys-sendmail destination@email.com -""" - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -import socket -import sys -import os - -from DIRAC import gLogger, exit as DIRACexit -from DIRAC.Core.Base import Script -from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient - -Script.setUsageMessage(''.join(__doc__)) - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -arg = "".join(args) - -if not len(arg) > 0: - gLogger.error("Missing argument") - DIRACexit(2) - -try: - head, body = arg.split("\\n\\n") -except Exception as x: - head = "To: %s" % arg - body = sys.stdin.read() - -try: - tmp, body = body.split("\\n\\n") - head = tmp + "\\n" + head -except Exception as x: - pass - -body = "".join(body.strip()) - -try: - headers = dict((i.strip(), j.strip()) for i, j in - (item.split(':') for item in head.split('\\n'))) -except BaseException: - gLogger.error("Failed to convert string: %s to email headers" % head) - DIRACexit(4) - -if "To" not in headers: - gLogger.error("Failed to get 'To:' field from headers %s" % head) - DIRACexit(5) -to = headers["To"] - -origin = "%s@%s" % (os.getenv("LOGNAME", "dirac"), socket.getfqdn()) -if "From" in headers: - origin = headers["From"] - -subject = "Sent from %s" % socket.getfqdn() -if "Subject" in headers: - subject = headers["Subject"] - -ntc = NotificationClient() -print("sendMail(%s,%s,%s,%s,%s)" % (to, subject, body, origin, False)) -result = ntc.sendMail(to, subject, body, origin, localAttempt=False) -if not result["OK"]: - gLogger.error(result["Message"]) - DIRACexit(6) - -DIRACexit(0) diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-uninstall-component.py b/src/DIRAC/FrameworkSystem/scripts/dirac-uninstall-component.py deleted file mode 100755 index 144667a7069..00000000000 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-uninstall-component.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python - -""" -Uninstallation of a DIRAC component -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import socket -from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient -from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities -from DIRAC import gLogger, S_OK -from DIRAC.Core.Base import Script -from DIRAC.Core.Utilities.PromptUser import promptUser -from DIRAC import exit as DIRACexit -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller - -__RCSID__ = "$Id$" - -gComponentInstaller.exitOnError = True - -force = False - - -def setForce(opVal): - global force - force = True - return S_OK() - - -Script.registerSwitch("f", "force", "Forces the removal of the logs", setForce) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, - 'Arguments:', - ' System: Name of the DIRAC system (ie: WorkloadManagement)', - ' Component: Name of the DIRAC component (ie: Matcher)'])) - -Script.parseCommandLine() -args = Script.getPositionalArgs() - -if len(args) == 1: - args = args[0].split('/') - -if len(args) < 2: - Script.showHelp(exitCode=1) - -system = args[0] -component = args[1] - -monitoringClient = ComponentMonitoringClient() -result = monitoringClient.getInstallations({'Instance': component, 'UnInstallationTime': None}, - {'System': system}, - {'HostName': socket.getfqdn()}, True) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) -if len(result['Value']) < 1: - gLogger.warn('Given component does not exist') - DIRACexit(1) -if len(result['Value']) > 1: - gLogger.error('Too many components match') - DIRACexit(1) - -removeLogs = False -if force: - removeLogs = True -else: - if result['Value'][0]['Component']['Type'] in gComponentInstaller.componentTypes: - result = promptUser('Remove logs?', ['y', 'n'], 'n') - if result['OK']: - removeLogs = result['Value'] == 'y' - else: - gLogger.error(result['Message']) - DIRACexit(1) - -result = gComponentInstaller.uninstallComponent(system, component, removeLogs) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) - -result = MonitoringUtilities.monitorUninstallation(system, component) -if not result['OK']: - gLogger.error(result['Message']) - DIRACexit(1) -gLogger.notice('Successfully uninstalled component %s/%s' % (system, component)) -DIRACexit() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_CAs.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_CAs.py new file mode 100755 index 00000000000..ff9d72caada --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_CAs.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-get-CAs +# Author : Ricardo Graciani +######################################################################## +"""Refresh the local copy of the CA certificates and revocation lists. + +Connects to the BundleDelivery service to obtain the tar balls. Needed when proxies appear to be +invalid. + +Usage: + + dirac-admin-get-CAs (|)* +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient + +__RCSID__ = "$Id$" + + +@DIRACScript() +def main(): + Script.addDefaultOptionValue('/DIRAC/Security/SkipCAChecks', 'yes') + Script.setUsageMessage(__doc__) + + Script.parseCommandLine(ignoreErrors=True) + + bdc = BundleDeliveryClient() + + result = bdc.syncCAs() + if not result['OK']: + DIRAC.gLogger.error("Error while updating CAs", result['Message']) + DIRAC.exit(1) + elif result['Value']: + DIRAC.gLogger.notice("CAs got updated") + else: + DIRAC.gLogger.notice("CAs are already synchronized") + + result = bdc.syncCRLs() + if not result['OK']: + DIRAC.gLogger.error("Error while updating CRLs", result['Message']) + DIRAC.exit(1) + elif result['Value']: + DIRAC.gLogger.notice("CRLs got updated") + else: + DIRAC.gLogger.notice("CRLs are already synchronized") + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py new file mode 100755 index 00000000000..69830d92b4f --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-get-proxy +# Author : Stuart Paterson +######################################################################## +""" + Retrieve a delegated proxy for the given user and group +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +import os +import DIRAC + +from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager +from DIRAC.ConfigurationSystem.Client.Helpers import Registry + +__RCSID__ = "$Id$" + + +class Params(object): + + limited = False + proxyPath = False + proxyLifeTime = 86400 + enableVOMS = False + vomsAttr = None + + def setLimited(self, args): + """ Set limited + + :param bool args: is limited + + :return: S_OK()/S_ERROR() + """ + self.limited = True + return S_OK() + + def setProxyLocation(self, args): + """ Set proxy location + + :param str args: proxy path + + :return: S_OK()/S_ERROR() + """ + self.proxyPath = args + return S_OK() + + def setProxyLifeTime(self, arg): + """ Set proxy lifetime + + :param int arg: lifetime in a seconds + + :return: S_OK()/S_ERROR() + """ + try: + fields = [f.strip() for f in arg.split(":")] + self.proxyLifeTime = int(fields[0]) * 3600 + int(fields[1]) * 60 + except BaseException: + gLogger.notice("Can't parse %s time! Is it a HH:MM?" % arg) + return S_ERROR("Can't parse time argument") + return S_OK() + + def automaticVOMS(self, arg): + """ Enable VOMS + + :param bool arg: enable VOMS + + :return: S_OK()/S_ERROR() + """ + self.enableVOMS = True + return S_OK() + + def setVOMSAttr(self, arg): + """ Register CLI switches + + :param str arg: VOMS attribute + """ + self.enableVOMS = True + self.vomsAttr = arg + return S_OK() + + def registerCLISwitches(self): + """ Register CLI switches + """ + Script.registerSwitch("v:", "valid=", "Valid HH:MM for the proxy. By default is 24 hours", self.setProxyLifeTime) + Script.registerSwitch("l", "limited", "Get a limited proxy", self.setLimited) + Script.registerSwitch("u:", "out=", "File to write as proxy", self.setProxyLocation) + Script.registerSwitch("a", "voms", "Get proxy with VOMS extension mapped to the DIRAC group", self.automaticVOMS) + Script.registerSwitch("m:", "vomsAttr=", "VOMS attribute to require", self.setVOMSAttr) + + +@DIRACScript() +def main(): + params = Params() + params.registerCLISwitches() + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... group' % Script.scriptName, + 'Arguments:', + ' DN: DN of the user', + ' user: DIRAC user name (will fail if there is more than 1 DN registered)', + ' group: DIRAC group name'])) + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) != 2: + Script.showHelp() + + userGroup = str(args[1]) + userDN = str(args[0]) + userName = False + if userDN.find("/") != 0: + userName = userDN + retVal = Registry.getDNForUsername(userName) + if not retVal['OK']: + gLogger.notice("Cannot discover DN for username %s\n\t%s" % (userName, retVal['Message'])) + DIRAC.exit(2) + DNList = retVal['Value'] + if len(DNList) > 1: + gLogger.notice("Username %s has more than one DN registered" % userName) + ind = 0 + for dn in DNList: + gLogger.notice("%d %s" % (ind, dn)) + ind += 1 + inp = raw_input("Which DN do you want to download? [default 0] ") + if not inp: + inp = 0 + else: + inp = int(inp) + userDN = DNList[inp] + else: + userDN = DNList[0] + + if not params.proxyPath: + if not userName: + result = Registry.getUsernameForDN(userDN) + if not result['OK']: + gLogger.notice("DN '%s' is not registered in DIRAC" % userDN) + DIRAC.exit(2) + userName = result['Value'] + params.proxyPath = "%s/proxy.%s.%s" % (os.getcwd(), userName, userGroup) + + if params.enableVOMS: + result = gProxyManager.downloadVOMSProxy(userDN, userGroup, limited=params.limited, + requiredTimeLeft=params.proxyLifeTime, + requiredVOMSAttribute=params.vomsAttr) + else: + result = gProxyManager.downloadProxy(userDN, userGroup, limited=params.limited, + requiredTimeLeft=params.proxyLifeTime) + if not result['OK']: + gLogger.notice('Proxy file cannot be retrieved: %s' % result['Message']) + DIRAC.exit(2) + chain = result['Value'] + result = chain.dumpAllToFile(params.proxyPath) + if not result['OK']: + gLogger.notice('Proxy file cannot be written to %s: %s' % (params.proxyPath, result['Message'])) + DIRAC.exit(2) + gLogger.notice("Proxy downloaded to %s" % params.proxyPath) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-proxy-upload.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_proxy_upload.py similarity index 88% rename from src/DIRAC/FrameworkSystem/scripts/dirac-admin-proxy-upload.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_admin_proxy_upload.py index 29fe5ab25e1..832cf7cc9f8 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-admin-proxy-upload.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_proxy_upload.py @@ -9,11 +9,14 @@ from __future__ import division import sys from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.FrameworkSystem.Client.ProxyUpload import CLIParams, uploadProxy __RCSID__ = "$Id$" -if __name__ == "__main__": + +@DIRACScript() +def main(): cliParams = CLIParams() cliParams.registerCLISwitches() @@ -24,3 +27,7 @@ print(retVal['Message']) sys.exit(1) sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_sysadmin_cli.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_sysadmin_cli.py new file mode 100755 index 00000000000..2f6383fe0f4 --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_sysadmin_cli.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + +__RCSID__ = "$Id$" + + +@DIRACScript() +def main(): + host = None + Script.registerSwitch("H:", "host=", " Target host") + Script.parseCommandLine(ignoreErrors=False) + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "h" or switch[0].lower() == "host": + host = switch[1] + + from DIRAC.FrameworkSystem.Client.SystemAdministratorClientCLI import SystemAdministratorClientCLI + + cli = SystemAdministratorClientCLI(host) + cli.cmdloop() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_instance.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_instance.py new file mode 100755 index 00000000000..26f139b839e --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_instance.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python +""" +Script to apply update to all or some dirac servers and restart them +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from io import open + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s version' % Script.scriptName, + ' ', + 'Arguments:', + ' version: version of DIRAC you want to update to', + ])) + Script.registerSwitch("", "hosts=", "Comma separated list of hosts or file containing row wise list of hosts" + " targeted for update (leave empty for all)") + Script.registerSwitch("", "retry=", "Number of retry attempts on hosts that have failed to update") + + Script.parseCommandLine(ignoreErrors=False) + + args = Script.getPositionalArgs() + if len(args) < 1 or len(args) > 2: + Script.showHelp() + + version = args[0] + retry = 0 + hosts = [] + + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "hosts": + hosts = switch[1] + if switch[0] == "retry": + retry = int(switch[1]) + + try: + with open(hosts, 'r') as f: + hosts = f.read().splitlines() + hosts = [str(host) for host in hosts] + except Exception: + pass + + if not isinstance(hosts, list): + hosts = hosts.split(',') + + from concurrent.futures import ThreadPoolExecutor, as_completed + + from DIRAC import S_OK, S_ERROR + from DIRAC import gLogger + from DIRAC.FrameworkSystem.Client.SystemAdministratorClient import SystemAdministratorClient + from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient + + def getListOfHosts(): + """ + Return the list of all hosts that constitute a DIRAC instance + """ + client = ComponentMonitoringClient() + result = client.getHosts({}, False, False) + if result['OK']: + hosts = [host['HostName'] for host in result['Value']] + return S_OK(hosts) + return S_ERROR('Cannot get list of hosts: %s' % result['Message']) + + def parseHostname(hostName): + """ + Separate the hostname from the port + + :param str hostName: hostname you want to parse + """ + hostList = hostName.split(':') + host = hostList[0] + if len(hostList) == 2: + port = hostList[1] + else: + port = None + return host, port + + def updateHost(hostName, version): + """ + Apply update to specific host + + :param str hostName: name of the host you want to update + :param str version: version vArBpC you want to update to + """ + host, port = parseHostname(hostName) + + client = SystemAdministratorClient(host, port) + result = client.ping() + if not result['OK']: + gLogger.error("Cannot connect to %s" % host) + return result + + gLogger.notice("Initiating software update of %s, this can take a while, please be patient ..." % host) + result = client.updateSoftware(version, '', '', timeout=600) + if not result['OK']: + return result + return S_OK() + + def updateHosts(version, hosts=[]): + """ + Apply update to all hosts + + :param str version: version vArBpC you want to update to + :param list[str] hosts: list of hosts to be updated, defaults to [] + """ + if not hosts: + result = getListOfHosts() + if not result['OK']: + return result + hosts = result['Value'] + + updateSuccess = [] + updateFail = [] + + executor = ThreadPoolExecutor(max_workers=len(hosts)) + futureUpdate = {executor.submit(updateHost, host, version): host for host in hosts} + for future in as_completed(futureUpdate): + host = futureUpdate[future] + result = future.result() + if result['OK']: + updateSuccess.append(host) + else: + updateFail.append(host) + + if not updateFail: + gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + gLogger.notice("!!! Successfully updated all hosts !!!") + gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + return S_OK([updateSuccess, updateFail]) + if not updateSuccess: + gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") + gLogger.notice("XXXXX Failed to update all hosts XXXXX") + gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") + return S_OK([updateSuccess, updateFail]) + gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!") + gLogger.notice("X!X!X Partially updated hosts X!X!X!") + gLogger.notice("Succeeded to update:") + for host in updateSuccess: + gLogger.notice(" + %s" % host) + gLogger.notice("Failed to update:") + for host in updateFail: + gLogger.notice(" - %s" % host) + gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!") + return S_OK([updateSuccess, updateFail]) + + def restartHost(hostName): + """ + Restart all systems and components of a host + + :param str hostName: name of the host you want to restart + """ + host, port = parseHostname(hostName) + + gLogger.notice("Pinging %s ..." % host) + + client = SystemAdministratorClient(host, port) + result = client.ping() + if not result['OK']: + gLogger.error("Could not connect to %s: %s" % (host, result['Message'])) + return result + gLogger.notice("Host %s is active" % host) + + gLogger.notice("Initiating restart of all systems and components") + # This restart call will always return S_ERROR because of SystemAdministrator restart + # Connection will be lost to the host + result = client.restartComponent('*', '*') + if result['Message'] == "Peer closed connection": + gLogger.notice("Restarted all systems on %s : connection to SystemAdministrator lost" % host) + return S_OK(result['Message']) + gLogger.error("Received unxpected message: %s" % result['Message']) + return result + + def updateInstance(version, hosts, retry): + """ + Update each server of an instance and restart them + + :param str version: version vArBpC you want to update to + :param list[str] hosts: list of hosts to be updated + :param int retry: number of retry attempts on hosts that have failed to update + """ + result = updateHosts(version, hosts) + if not result['OK']: + return result + + updateSuccess = result['Value'][0] + updateFail = result['Value'][1] + restartSuccess = [] + restartFail = [] + for host in updateSuccess: + result = restartHost(host) + if result['OK']: + restartSuccess.append(host) + else: + restartFail.append(host) + + if not restartFail and not updateFail: + return S_OK("Successfully updated and restarted all hosts") + + gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") + gLogger.notice("XXXXX There were problems in the update process XXXXX") + gLogger.notice("Succeeded to update:") + for host in updateSuccess: + gLogger.notice(" + %s" % host) + gLogger.notice("Succeeded to restart:") + for host in restartSuccess: + gLogger.notice(" + %s" % host) + gLogger.notice("Failed to update:") + for host in updateFail: + gLogger.notice(" - %s" % host) + gLogger.notice("Failed to restart:") + for host in restartFail: + gLogger.notice(" - %s" % host) + gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") + + if retry > 0: + retryHosts = list(set(updateFail + restartFail)) + gLogger.notice("Retrying update on (%s attempts remaining):" % retry) + for host in retryHosts: + gLogger.notice(" - %s" % host) + gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") + return updateInstance(version, retryHosts, retry - 1) + + return S_ERROR("Update failed!") + + result = updateInstance(version, hosts, retry) + if not result['OK']: + gLogger.fatal(result['Message']) + DIRAC.exit(1) + gLogger.notice(result['Value']) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py new file mode 100755 index 00000000000..54badad3bf3 --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_update_pilot.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +""" +Script to update pilot version in CS +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s version' % Script.scriptName, + ' ', + 'Arguments:', + ' version: pilot version you want to update to' + ])) + Script.registerSwitch( + "v:", + "vo=", + "Location of pilot version in CS /Operations//Pilot/Version" + " (default value specified in CS under /DIRAC/DefaultSetup)" + ) + + Script.parseCommandLine(ignoreErrors=False) + + args = Script.getPositionalArgs() + if len(args) < 1 or len(args) > 2: + Script.showHelp() + + version = args[0] + vo = None + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "v" or switch[0] == "vo": + vo = switch[1] + + from DIRAC import S_OK, S_ERROR + from DIRAC import gConfig, gLogger + from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI + + def updatePilot(version, vo): + """ + Update in the CS the pilot version used, + If only one version present in CS it's overwritten. + If two versions present, the new one is added and the last removed + + :param version: version vArBpC of pilot you want to use + :param vo: Location of pilot version in CS /Operations//Pilot/Version + """ + setup = vo + if not vo: + setup = gConfig.getValue('/DIRAC/DefaultSetup') + if not setup: + return S_ERROR("No value set for /DIRAC/DefaultSetup in CS") + + pilotVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup, []) + if not pilotVersion: + return S_ERROR("No pilot version set under Operations/%s/Pilot/Version in CS" % setup) + + pilotVersion.pop() + pilotVersion.insert(0, version) + api = CSAPI() + api.setOption('Operations/%s/Pilot/Version' % setup, ", ".join(pilotVersion)) + result = api.commit() + if not result['OK']: + gLogger.fatal('Could not commit new version of pilot!') + return result + + newVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup) + return S_OK("New version of pilot set to %s" % newVersion) + + result = updatePilot(version, vo) + if not result['OK']: + gLogger.fatal(result['Message']) + DIRAC.exit(1) + gLogger.notice(result['Value']) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py new file mode 100755 index 00000000000..0aea5d1f32f --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager +from DIRAC.Core.Utilities import Time + +__RCSID__ = "$Id$" + + +class Params(object): + + limited = False + proxyPath = False + proxyLifeTime = 3600 + + def setProxyLifeTime(self, arg): + try: + fields = [f.strip() for f in arg.split(":")] + self.proxyLifeTime = int(fields[0]) * 3600 + int(fields[1]) * 60 + except BaseException: + print("Can't parse %s time! Is it a HH:MM?" % arg) + return DIRAC.S_ERROR("Can't parse time argument") + return DIRAC.S_OK() + + def registerCLISwitches(self): + Script.registerSwitch("v:", "valid=", "Required HH:MM for the users", self.setProxyLifeTime) + + +@DIRACScript() +def main(): + params = Params() + params.registerCLISwitches() + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + result = gProxyManager.getDBContents() + if not result['OK']: + print("Can't retrieve list of users: %s" % result['Message']) + DIRAC.exit(1) + + keys = result['Value']['ParameterNames'] + records = result['Value']['Records'] + dataDict = {} + now = Time.dateTime() + for record in records: + expirationDate = record[3] + dt = expirationDate - now + secsLeft = dt.days * 86400 + dt.seconds + if secsLeft > params.proxyLifeTime: + userName, userDN, userGroup, _, persistent = record + if userName not in dataDict: + dataDict[userName] = [] + dataDict[userName].append((userDN, userGroup, expirationDate, persistent)) + + for userName in dataDict: + print("* %s" % userName) + for iP in range(len(dataDict[userName])): + data = dataDict[userName][iP] + print(" DN : %s" % data[0]) + print(" group : %s" % data[1]) + print(" not after : %s" % Time.toString(data[2])) + print(" persistent : %s" % data[3]) + if iP < len(dataDict[userName]) - 1: + print(" -") + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py new file mode 100755 index 00000000000..79f0b4b27da --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_install_component.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +""" +Do the initial installation and configuration of a DIRAC component +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from DIRAC import gConfig, gLogger, S_OK +from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions +from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC import exit as DIRACexit +from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + +__RCSID__ = "$Id$" + +gComponentInstaller.exitOnError = True + +overwrite = False + + +def setOverwrite(opVal): + global overwrite + overwrite = True + return S_OK() + + +module = '' +specialOptions = {} + + +def setModule(optVal): + global specialOptions, module + specialOptions['Module'] = optVal + module = optVal + return S_OK() + + +def setSpecialOption(optVal): + global specialOptions + option, value = optVal.split('=') + specialOptions[option] = value + return S_OK() + + +@DIRACScript() +def main(): + global overwrite + global specialOptions + global module + global specialOptions + + Script.registerSwitch("w", "overwrite", "Overwrite the configuration in the global CS", setOverwrite) + Script.registerSwitch("m:", "module=", "Python module name for the component code", setModule) + Script.registerSwitch("p:", "parameter=", "Special component option ", setSpecialOption) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, + 'Arguments:', + ' System: Name of the DIRAC system (ie: WorkloadManagement)', + ' Service: Name of the DIRAC component (ie: Matcher)'])) + + Script.parseCommandLine() + args = Script.getPositionalArgs() + + if len(args) == 1: + args = args[0].split('/') + + if len(args) != 2: + Script.showHelp(exitCode=1) + + cType = None + system = args[0] + component = args[1] + compOrMod = module if module else component + + result = gComponentInstaller.getSoftwareComponents(getCSExtensions()) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + else: + availableComponents = result['Value'] + + for compType in availableComponents: + if system in availableComponents[compType] and compOrMod in availableComponents[compType][system]: + cType = compType[:-1].lower() + break + + if not cType: + gLogger.error('Component %s/%s is not available for installation' % (system, component)) + DIRACexit(1) + + if module: + result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, module, + getCSExtensions(), + overwrite=overwrite) + result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, component, + getCSExtensions(), + specialOptions=specialOptions, + overwrite=overwrite, + addDefaultOptions=False) + else: + result = gComponentInstaller.addDefaultOptionsToCS(gConfig, cType, system, component, + getCSExtensions(), + specialOptions=specialOptions, + overwrite=overwrite) + + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + else: + result = gComponentInstaller.installComponent(cType, system, component, getCSExtensions(), module) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + else: + gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) + result = gComponentInstaller.setupComponent(cType, system, component, getCSExtensions(), module) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + if component == 'ComponentMonitoring': + result = MonitoringUtilities.monitorInstallation('DB', system, 'InstalledComponentsDB') + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + result = MonitoringUtilities.monitorInstallation(cType, system, component, module) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + gLogger.notice('Successfully completed the installation of %s/%s' % (system, component)) + DIRACexit() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py b/src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py new file mode 100755 index 00000000000..1d88dbe3b24 --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +""" +Do the initial installation and configuration of a DIRAC service based on tornado +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from DIRAC import gConfig, gLogger, S_OK +from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions +from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC import exit as DIRACexit +from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + +__RCSID__ = "$Id$" + +gComponentInstaller.exitOnError = True + +overwrite = False + + +def setOverwrite(opVal): + global overwrite + overwrite = True + return S_OK() + + +module = '' +specialOptions = {} + + +def setModule(optVal): + global specialOptions, module + specialOptions['Module'] = optVal + module = optVal + return S_OK() + + +def setSpecialOption(optVal): + global specialOptions + option, value = optVal.split('=') + specialOptions[option] = value + return S_OK() + + +@DIRACScript() +def main(): + global overwrite + global specialOptions + global module + global specialOptions + Script.registerSwitch("w", "overwrite", "Overwrite the configuration in the global CS", setOverwrite) + Script.registerSwitch("m:", "module=", "Python module name for the component code", setModule) + Script.registerSwitch("p:", "parameter=", "Special component option ", setSpecialOption) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, + 'Arguments:', + ' System: Name of the DIRAC system (ie: WorkloadManagement)', + ' Service: Name of the DIRAC component (ie: Matcher)'])) + + Script.parseCommandLine() + args = Script.getPositionalArgs() + + if len(args) == 1: + args = args[0].split('/') + + if len(args) != 2: + Script.showHelp() + DIRACexit(1) + + system = args[0] + component = args[1] + compOrMod = module if module else component + + result = gComponentInstaller.addDefaultOptionsToCS(gConfig, 'service', system, component, + getCSExtensions(), + specialOptions=specialOptions, + overwrite=overwrite) + + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + result = gComponentInstaller.addTornadoOptionsToCS(gConfig) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + result = gComponentInstaller.installTornado() + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) + result = gComponentInstaller.setupTornadoService(system, component, getCSExtensions(), module) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + result = MonitoringUtilities.monitorInstallation('service', system, component, module) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + gLogger.notice('Successfully completed the installation of %s/%s' % (system, component)) + DIRACexit() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_monitoring_get_components_status.py b/src/DIRAC/FrameworkSystem/scripts/dirac_monitoring_get_components_status.py new file mode 100755 index 00000000000..01ef7db6ea1 --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_monitoring_get_components_status.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +import sys +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + +__RCSID__ = "$Id$" + + +@DIRACScript() +def main(): + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + fieldsToShow = ('ComponentName', 'Type', 'Host', 'Port', 'Status', 'Message') + + from DIRAC.FrameworkSystem.Client.MonitoringClient import gMonitor + + result = gMonitor.getComponentsStatusWebFormatted(sortingList=[['ComponentName', 'ASC']]) + if not result['OK']: + print("ERROR: %s" % result['Message']) + sys.exit(1) + paramNames = result['Value']['ParameterNames'] + records = result['Value']['Records'] + fieldLengths = [] + for param in paramNames: + fieldLengths.append(len(param)) + + for record in records: + for i in range(len(record)): + if paramNames[i] in fieldsToShow: + fieldLengths[i] = max(fieldLengths[i], len(str(record[i]))) + # Print time! + line = [] + sepLine = [] + for i in range(len(paramNames)): + param = paramNames[i] + if param in fieldsToShow: + line.append("%s%s" % (param, " " * (fieldLengths[i] - len(param)))) + sepLine.append("-" * fieldLengths[i]) + print("|".join(line)) + sepLine = "+".join(sepLine) + print(sepLine) + for record in records: + line = [] + for i in range(len(record)): + if paramNames[i] in fieldsToShow: + val = str(record[i]) + line.append("%s%s" % (val, " " * (fieldLengths[i] - len(val)))) + print("|".join(line)) + # print sepLine + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_myproxy_upload.py b/src/DIRAC/FrameworkSystem/scripts/dirac_myproxy_upload.py new file mode 100755 index 00000000000..0e7882a69ea --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_myproxy_upload.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-proxy-init.py +# Author : Adrian Casajus +######################################################################## + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +import sys +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +class Params: + proxyLoc = False + dnAsUsername = False + + def setProxyLocation(self, arg): + self.proxyLoc = arg + return DIRAC.S_OK() + + def setDNAsUsername(self, arg): + self.dnAsUsername = True + return DIRAC.S_OK() + + def showVersion(self, arg): + print("Version:") + print(" ", __RCSID__) + sys.exit(0) + return DIRAC.S_OK() + + +@DIRACScript() +def main(): + params = Params() + + Script.registerSwitch("f:", "file=", "File to use as proxy", params.setProxyLocation) + Script.registerSwitch("D", "DN", "Use DN as myproxy username", params.setDNAsUsername) + Script.registerSwitch("i", "version", "Print version", params.showVersion) + + Script.addDefaultOptionValue("LogLevel", "always") + Script.parseCommandLine() + + from DIRAC.Core.Security.MyProxy import MyProxy + from DIRAC.Core.Security import Locations + + if not params.proxyLoc: + params.proxyLoc = Locations.getProxyLocation() + + if not params.proxyLoc: + print("Can't find any valid proxy") + sys.exit(1) + print("Uploading proxy file %s" % params.proxyLoc) + + mp = MyProxy() + retVal = mp.uploadProxy(params.proxyLoc, params.dnAsUsername) + if not retVal['OK']: + print("Can't upload proxy:") + print(" ", retVal['Message']) + sys.exit(1) + print("Proxy uploaded") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_populate_component_db.py b/src/DIRAC/FrameworkSystem/scripts/dirac_populate_component_db.py new file mode 100755 index 00000000000..799ab52449f --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_populate_component_db.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python + +""" +Populates the database with the current installations of components +This script assumes that the InstalledComponentsDB, the +ComponentMonitoring service and the Notification service are installed and running +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +from datetime import datetime +from DIRAC import exit as DIRACexit +from DIRAC import S_OK, gLogger, gConfig +from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient +from DIRAC.FrameworkSystem.Client.SystemAdministratorIntegrator \ + import SystemAdministratorIntegrator +from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient \ + import ComponentMonitoringClient +from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities +from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations +from DIRAC.Core.Security.ProxyInfo import getProxyInfo + +__RCSID__ = "$Id$" + +global excludedHosts +excludedHosts = [] + + +def setExcludedHosts(value): + global excludedHosts + + excludedHosts = value.split(',') + return S_OK() + + +@DIRACScript() +def main(): + global excludedHosts + Script.registerSwitch( + "e:", + "exclude=", + "Comma separated list of hosts to be excluded from the scanning process", + setExcludedHosts) + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... [debug]' % Script.scriptName])) + + Script.parseCommandLine(ignoreErrors=False) + args = Script.getPositionalArgs() + + componentType = '' + + # Get my setup + mySetup = gConfig.getValue('DIRAC/Setup') + + # Retrieve information from all the hosts + client = SystemAdministratorIntegrator(exclude=excludedHosts) + resultAll = client.getOverallStatus() + + # Retrieve user installing the component + result = getProxyInfo() + if result['OK']: + user = result['Value']['username'] + else: + DIRACexit(-1) + if not user: + user = 'unknown' + + notificationClient = NotificationClient() + for host in resultAll['Value']: + if not resultAll['Value'][host]['OK']: + # If the host cannot be contacted, exclude it and send message + excludedHosts.append(host) + + result = notificationClient.sendMail( + Operations().getValue( + 'EMail/Production', + []), + 'Unreachable host', + '\ndirac-populate-component-db: Could not fill the database with the components from unreachable host %s\n' % + host) + if not result['OK']: + gLogger.error('Can not send unreachable host notification mail: %s' % result['Message']) + + if not resultAll['OK']: + gLogger.error(resultAll['Message']) + DIRACexit(-1) + resultHosts = client.getHostInfo() + if not resultHosts['OK']: + gLogger.error(resultHosts['Message']) + DIRACexit(-1) + resultInfo = client.getInfo() + if not resultInfo['OK']: + gLogger.error(resultInfo['Message']) + DIRACexit(-1) + resultMySQL = client.getMySQLStatus() + if not resultMySQL['OK']: + gLogger.error(resultMySQL['Message']) + DIRACexit(-1) + resultAllDB = client.getDatabases() + if not resultAllDB['OK']: + gLogger.error(resultAllDB['Message']) + DIRACexit(-1) + resultAvailableDB = client.getAvailableDatabases() + if not resultAvailableDB['OK']: + gLogger.error(resultAvailableDB['Message']) + DIRACexit(-1) + + records = [] + finalSet = list(set(resultAll['Value']) - set(excludedHosts)) + for host in finalSet: + hasMySQL = True + result = resultAll['Value'][host] + hostResult = resultHosts['Value'][host] + infoResult = resultInfo['Value'][host] + mySQLResult = resultMySQL['Value'][host] + allDBResult = resultAllDB['Value'][host] + availableDBResult = resultAvailableDB['Value'][host] + + if not result['OK']: + gLogger.error('Host %s: %s' % (host, result['Message'])) + continue + if not hostResult['OK']: + gLogger.error('Host %s: %s' % (host, hostResult['Message'])) + continue + if not infoResult['OK']: + gLogger.error('Host %s: %s' % (host, infoResult['Message'])) + continue + if mySQLResult['OK']: + if not allDBResult['OK']: + gLogger.error('Host %s: %s' % (host, allDBResult['Message'])) + continue + if not availableDBResult['OK']: + gLogger.error('Host %s: %s' % (host, availableDBResult['Message'])) + continue + else: + hasMySQL = False + + setup = infoResult['Value']['Setup'] + if setup != mySetup: + continue + + cpu = hostResult['Value']['CPUModel'].strip() + rDict = result['Value'] + # Components other than databases + for compType in rDict: + if componentType and componentType != compType: + continue + for system in rDict[compType]: + components = sorted(rDict[compType][system]) + for component in components: + record = {'Installation': {}, 'Component': {}, 'Host': {}} + if rDict[compType][system][component]['Installed'] and \ + component != 'ComponentMonitoring': + runitStatus = \ + str(rDict[compType][system][component]['RunitStatus']) + if runitStatus != 'Unknown': + module = \ + str(rDict[compType][system][component]['Module']) + record['Component']['System'] = system + record['Component']['Module'] = module + # Transform 'Services' into 'service', 'Agents' into 'agent' ... + record['Component']['Type'] = compType.lower()[:-1] + record['Host']['HostName'] = host + record['Host']['CPU'] = cpu + record['Installation']['Instance'] = component + record['Installation']['InstallationTime'] = datetime.utcnow() + record['Installation']['InstalledBy'] = user + records.append(record) + + # Databases + csClient = CSAPI() + cfg = csClient.getCurrentCFG()['Value'] + + if hasMySQL: + allDB = allDBResult['Value'] + availableDB = availableDBResult['Value'] + + for db in allDB: + # Check for DIRAC only databases + if db in availableDB and db != 'InstalledComponentsDB': + # Check for 'installed' databases + isSection = cfg.isSection('Systems/' + availableDB[db]['System'] + '/' + + cfg.getOption('DIRAC/Setups/' + setup + '/' + + availableDB[db]['System']) + '/Databases/' + db + + '/') + if isSection: + record = {'Installation': {}, 'Component': {}, 'Host': {}} + record['Component']['System'] = availableDB[db]['System'] + record['Component']['Module'] = db + record['Component']['Type'] = 'DB' + record['Host']['HostName'] = host + record['Host']['CPU'] = cpu + record['Installation']['Instance'] = db + record['Installation']['InstallationTime'] = datetime.utcnow() + record['Installation']['InstalledBy'] = user + records.append(record) + + monitoringClient = ComponentMonitoringClient() + + # Add the installations to the database + for record in records: + result = MonitoringUtilities.monitorInstallation( + record['Component']['Type'], record['Component']['System'], + record['Installation']['Instance'], record['Component']['Module'], + record['Host']['CPU'], record['Host']['HostName']) + if not result['OK']: + gLogger.error(result['Message']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-destroy.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_destroy.py similarity index 97% rename from src/DIRAC/FrameworkSystem/scripts/dirac-proxy-destroy.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_proxy_destroy.py index 50239d8b114..8b109a63cbb 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-destroy.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_destroy.py @@ -13,6 +13,7 @@ from DIRAC import gLogger, S_OK from DIRAC.Core.Security import Locations from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager @@ -126,7 +127,7 @@ def deleteLocalProxy(proxyLoc): gLogger.notice('Local proxy deleted.') -def main(): +def run(): """ main program entry point """ @@ -180,10 +181,15 @@ def main(): return 0 -if __name__ == "__main__": +@DIRACScript() +def main(): try: - retval = main() + retval = run() DIRAC.exit(retval) except RuntimeError as rtError: gLogger.error('Operation failed: %s' % str(rtError)) DIRAC.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_get_uploaded_info.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_get_uploaded_info.py new file mode 100755 index 00000000000..3a68a55b366 --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_get_uploaded_info.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-proxy-init.py +# Author : Adrian Casajus +######################################################################## +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +import sys + +from DIRAC import gLogger, S_OK +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient +from DIRAC.Core.Security import Properties +from DIRAC.Core.Security.ProxyInfo import getProxyInfo +from DIRAC.ConfigurationSystem.Client.Helpers import Registry + +__RCSID__ = "$Id:" + +userName = False + + +def setUser(arg): + """ Set user + + :param str arg: user name + + :return: S_OK() + """ + global userName + userName = arg + return S_OK() + + +@DIRACScript() +def main(): + global userName + Script.registerSwitch("u:", "user=", "User to query (by default oneself)", setUser) + + Script.parseCommandLine() + + result = getProxyInfo() + if not result['OK']: + gLogger.notice("Do you have a valid proxy?") + gLogger.notice(result['Message']) + sys.exit(1) + proxyProps = result['Value'] + + userName = userName or proxyProps.get('username') + if not userName: + gLogger.notice("Your proxy don`t have username extension") + sys.exit(1) + + if userName in Registry.getAllUsers(): + if Properties.PROXY_MANAGEMENT not in proxyProps['groupProperties']: + if userName != proxyProps['username'] and userName != proxyProps['issuer']: + gLogger.notice("You can only query info about yourself!") + sys.exit(1) + result = Registry.getDNForUsername(userName) + if not result['OK']: + gLogger.notice("Oops %s" % result['Message']) + dnList = result['Value'] + if not dnList: + gLogger.notice("User %s has no DN defined!" % userName) + sys.exit(1) + userDNs = dnList + else: + userDNs = [userName] + + gLogger.notice("Checking for DNs %s" % " | ".join(userDNs)) + pmc = ProxyManagerClient() + result = pmc.getDBContents({'UserDN': userDNs}) + if not result['OK']: + gLogger.notice("Could not retrieve the proxy list: %s" % result['Value']) + sys.exit(1) + + data = result['Value'] + colLengths = [] + for pN in data['ParameterNames']: + colLengths.append(len(pN)) + for row in data['Records']: + for i in range(len(row)): + colLengths[i] = max(colLengths[i], len(str(row[i]))) + + lines = [""] + for i in range(len(data['ParameterNames'])): + pN = data['ParameterNames'][i] + lines[0] += "| %s " % pN.ljust(colLengths[i]) + lines[0] += "|" + tL = len(lines[0]) + lines.insert(0, "-" * tL) + lines.append("-" * tL) + for row in data['Records']: + nL = "" + for i in range(len(row)): + nL += "| %s " % str(row[i]).ljust(colLengths[i]) + nL += "|" + lines.append(nL) + lines.append("-" * tL) + + gLogger.notice("\n".join(lines)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py new file mode 100755 index 00000000000..bdf07caf4af --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-proxy-info.py +# Author : Adrian Casajus +######################################################################## + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +import sys + +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.Core.Utilities.ReturnValues import S_OK + +__RCSID__ = "$Id$" + + +class Params(object): + + proxyLoc = False + vomsEnabled = True + csEnabled = True + steps = False + checkValid = False + checkClock = True + uploadedInfo = False + + def showVersion(self, arg): + print("Version:") + print(" ", __RCSID__) + sys.exit(0) + return S_OK() + + def setProxyLocation(self, arg): + self.proxyLoc = arg + return S_OK() + + def disableVOMS(self, arg): + self.vomsEnabled = False + return S_OK() + + def disableCS(self, arg): + self.csEnabled = False + return S_OK() + + def showSteps(self, arg): + self.steps = True + return S_OK() + + def validityCheck(self, arg): + self.checkValid = True + return S_OK() + + def disableClockCheck(self, arg): + self.checkClock = False + return S_OK() + + def setManagerInfo(self, arg): + self.uploadedInfo = True + return S_OK() + + +@DIRACScript() +def main(): + params = Params() + + from DIRAC.Core.Base import Script + Script.registerSwitch("f:", "file=", "File to use as user key", params.setProxyLocation) + Script.registerSwitch("i", "version", "Print version", params.showVersion) + Script.registerSwitch("n", "novoms", "Disable VOMS", params.disableVOMS) + Script.registerSwitch("v", "checkvalid", "Return error if the proxy is invalid", params.validityCheck) + Script.registerSwitch("x", "nocs", "Disable CS", params.disableCS) + Script.registerSwitch("e", "steps", "Show steps info", params.showSteps) + Script.registerSwitch("j", "noclockcheck", "Disable checking if time is ok", params.disableClockCheck) + Script.registerSwitch("m", "uploadedinfo", "Show uploaded proxies info", params.setManagerInfo) + + Script.disableCS() + Script.parseCommandLine() + + from DIRAC.Core.Utilities.NTP import getClockDeviation + from DIRAC import gLogger + from DIRAC.Core.Security.ProxyInfo import getProxyInfo, getProxyStepsInfo + from DIRAC.Core.Security.ProxyInfo import formatProxyInfoAsString, formatProxyStepsInfoAsString + from DIRAC.Core.Security import VOMS + from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager + from DIRAC.ConfigurationSystem.Client.Helpers import Registry + + if params.csEnabled: + retVal = Script.enableCS() + if not retVal['OK']: + print("Cannot contact CS to get user list") + + if params.checkClock: + result = getClockDeviation() + if result['OK']: + deviation = result['Value'] + if deviation > 600: + gLogger.error("Your host clock seems to be off by more than TEN MINUTES! Thats really bad.") + elif deviation > 180: + gLogger.error("Your host clock seems to be off by more than THREE minutes! Thats bad.") + elif deviation > 60: + gLogger.error("Your host clock seems to be off by more than a minute! Thats not good.") + + result = getProxyInfo(params.proxyLoc, not params.vomsEnabled) + if not result['OK']: + gLogger.error(result['Message']) + sys.exit(1) + infoDict = result['Value'] + gLogger.notice(formatProxyInfoAsString(infoDict)) + if not infoDict['isProxy']: + gLogger.error('==============================\n!!! The proxy is not valid !!!') + + if params.steps: + gLogger.notice("== Steps extended info ==") + chain = infoDict['chain'] + stepInfo = getProxyStepsInfo(chain)['Value'] + gLogger.notice(formatProxyStepsInfoAsString(stepInfo)) + + def invalidProxy(msg): + gLogger.error("Invalid proxy:", msg) + sys.exit(1) + + if params.uploadedInfo: + result = gProxyManager.getUserProxiesInfo() + if not result['OK']: + gLogger.error("Could not retrieve the uploaded proxies info", result['Message']) + else: + uploadedInfo = result['Value'] + if not uploadedInfo: + gLogger.notice("== No proxies uploaded ==") + if uploadedInfo: + gLogger.notice("== Proxies uploaded ==") + maxDNLen = 0 + maxGroupLen = 0 + for userDN in uploadedInfo: + maxDNLen = max(maxDNLen, len(userDN)) + for group in uploadedInfo[userDN]: + maxGroupLen = max(maxGroupLen, len(group)) + gLogger.notice(" %s | %s | Until (GMT)" % ("DN".ljust(maxDNLen), "Group".ljust(maxGroupLen))) + for userDN in uploadedInfo: + for group in uploadedInfo[userDN]: + gLogger.notice(" %s | %s | %s" % (userDN.ljust(maxDNLen), + group.ljust(maxGroupLen), + uploadedInfo[userDN][group].strftime("%Y/%m/%d %H:%M"))) + + if params.checkValid: + if infoDict['secondsLeft'] == 0: + invalidProxy("Proxy is expired") + if params.csEnabled and not infoDict['validGroup']: + invalidProxy("Group %s is not valid" % infoDict['group']) + if 'hasVOMS' in infoDict and infoDict['hasVOMS']: + requiredVOMS = Registry.getVOMSAttributeForGroup(infoDict['group']) + if 'VOMS' not in infoDict or not infoDict['VOMS']: + invalidProxy("Unable to retrieve VOMS extension") + if len(infoDict['VOMS']) > 1: + invalidProxy("More than one voms attribute found") + if requiredVOMS not in infoDict['VOMS']: + invalidProxy("Unexpected VOMS extension %s. Extension expected for DIRAC group is %s" % ( + infoDict['VOMS'][0], + requiredVOMS)) + result = VOMS.VOMS().getVOMSProxyInfo(infoDict['chain'], 'actimeleft') + if not result['OK']: + invalidProxy("Cannot determine life time of VOMS attributes: %s" % result['Message']) + if int(result['Value'].strip()) == 0: + invalidProxy("VOMS attributes are expired") + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-init.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py similarity index 98% rename from src/DIRAC/FrameworkSystem/scripts/dirac-proxy-init.py rename to src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py index d5bcf00a2f2..693ad998b7b 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac-proxy-init.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py @@ -17,6 +17,7 @@ from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.FrameworkSystem.Client import ProxyGeneration, ProxyUpload from DIRAC.Core.Security import X509Chain, ProxyInfo, Properties, VOMS from DIRAC.ConfigurationSystem.Client.Helpers import Registry @@ -232,7 +233,9 @@ def doTheMagic(self): return S_OK() -if __name__ == "__main__": +@DIRACScript() +def main(): + global piParams, pI piParams = Params() piParams.registerCLISwitches() @@ -249,3 +252,7 @@ def doTheMagic(self): pI.printInfo() sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_restart_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_restart_component.py new file mode 100755 index 00000000000..ba4e2fa16d7 --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_restart_component.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +""" + Restart DIRAC component using runsvctrl utility +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" +# +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.disableCS() + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... [System [Service|Agent]]' % Script.scriptName, + 'Arguments:', + ' System: Name of the system for the component (default *: all)', + ' Service|Agent: Name of the particular component (default *: all)'])) + Script.parseCommandLine() + args = Script.getPositionalArgs() + if len(args) > 2: + Script.showHelp(exitCode=1) + + system = '*' + component = '*' + if args: + system = args[0] + if system != '*': + if len(args) > 1: + component = args[1] + + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + + gComponentInstaller.exitOnError = True + + result = gComponentInstaller.runsvctrlComponent(system, component, 't') + if not result['OK']: + print('ERROR:', result['Message']) + exit(-1) + + gComponentInstaller.printStartupStatus(result['Value']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_start_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_start_component.py new file mode 100755 index 00000000000..5a3e03ecafe --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_start_component.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +""" +Start DIRAC component using runsvctrl utility +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.disableCS() + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... [system [service|agent]]' % Script.scriptName, + 'Arguments:', + ' system: Name of the system for the component (default *: all)', + ' service|agent: Name of the particular component (default *: all)'])) + Script.parseCommandLine() + args = Script.getPositionalArgs() + if len(args) > 2: + Script.showHelp(exitCode=1) + + system = '*' + component = '*' + if len(args) > 0: + system = args[0] + if system != '*': + if len(args) > 1: + component = args[1] + # + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + # + gComponentInstaller.exitOnError = True + # + result = gComponentInstaller.runsvctrlComponent(system, component, 'u') + if not result['OK']: + print('ERROR:', result['Message']) + exit(-1) + + gComponentInstaller.printStartupStatus(result['Value']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_status_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_status_component.py new file mode 100755 index 00000000000..d202db7329e --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_status_component.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +""" +Status of DIRAC components using runsvstat utility +""" +# +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.disableCS() + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... [system [service|agent]]' % Script.scriptName, + 'Arguments:', + ' system: Name of the system for the component (default *: all)', + ' service|agent: Name of the particular component (default *: all)'])) + Script.parseCommandLine() + args = Script.getPositionalArgs() + + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + + __RCSID__ = "$Id$" + + if len(args) > 2: + Script.showHelp(exitCode=1) + + system = '*' + component = '*' + if len(args) > 0: + system = args[0] + if system != '*': + if len(args) > 1: + component = args[1] + # + gComponentInstaller.exitOnError = True + # + result = gComponentInstaller.getStartupComponentStatus([system, component]) + if not result['OK']: + print('ERROR:', result['Message']) + exit(-1) + + gComponentInstaller.printStartupStatus(result['Value']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_stop_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_stop_component.py new file mode 100755 index 00000000000..a4992d27614 --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_stop_component.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +""" +Do the initial installation and configuration of the DIRAC MySQL server +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.disableCS() + Script.setUsageMessage('\n'.join(['Stop DIRAC component using runsvctrl utility', + 'Usage:', + ' %s [option|cfgfile] ... [system [service|agent]]' % Script.scriptName, + 'Arguments:', + ' system: Name of the system for the component (default *: all)', + ' service|agent: Name of the particular component (default *: all)'])) + Script.parseCommandLine() + args = Script.getPositionalArgs() + + from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + + __RCSID__ = "$Id$" + + if len(args) > 2: + Script.showHelp(exitCode=1) + + system = '*' + component = '*' + if len(args) > 0: + system = args[0] + if system != '*': + if len(args) > 1: + component = args[1] + # + # + gComponentInstaller.exitOnError = True + # + result = gComponentInstaller.runsvctrlComponent(system, component, 'd') + if not result['OK']: + print('ERROR:', result['Message']) + exit(-1) + + gComponentInstaller.printStartupStatus(result['Value']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_sys_sendmail.py b/src/DIRAC/FrameworkSystem/scripts/dirac_sys_sendmail.py new file mode 100755 index 00000000000..f0f99c2b84c --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_sys_sendmail.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-sys-sendmail +# Author : Matvey Sapunov +######################################################################## + +""" + Utility to send an e-mail using DIRAC notification service. + + Arguments: + Formated text message. The message consists of e-mail headers and e-mail body + separated by two newline characters. Headers are key : value pairs separated + by newline character. Meaningful headers are "To:", "From:", "Subject:". + Other keys will be ommited. + Message body is an arbitrary string. + + Options: + There are no options. + + Examples: + dirac-sys-sendmail "From: source@email.com\\nTo: destination@email.com\\nSubject: Test\\n\\nMessage body" + echo "From: source@email.com\\nSubject: Test\\n\\nMessage body" | dirac-sys-sendmail destination@email.com +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +import socket +import sys +import os + +from DIRAC import gLogger, exit as DIRACexit +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient + + +@DIRACScript() +def main(): + Script.setUsageMessage(''.join(__doc__)) + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + arg = "".join(args) + + if not len(arg) > 0: + gLogger.error("Missing argument") + DIRACexit(2) + + try: + head, body = arg.split("\\n\\n") + except Exception as x: + head = "To: %s" % arg + body = sys.stdin.read() + + try: + tmp, body = body.split("\\n\\n") + head = tmp + "\\n" + head + except Exception as x: + pass + + body = "".join(body.strip()) + + try: + headers = dict((i.strip(), j.strip()) for i, j in + (item.split(':') for item in head.split('\\n'))) + except BaseException: + gLogger.error("Failed to convert string: %s to email headers" % head) + DIRACexit(4) + + if "To" not in headers: + gLogger.error("Failed to get 'To:' field from headers %s" % head) + DIRACexit(5) + to = headers["To"] + + origin = "%s@%s" % (os.getenv("LOGNAME", "dirac"), socket.getfqdn()) + if "From" in headers: + origin = headers["From"] + + subject = "Sent from %s" % socket.getfqdn() + if "Subject" in headers: + subject = headers["Subject"] + + ntc = NotificationClient() + print("sendMail(%s,%s,%s,%s,%s)" % (to, subject, body, origin, False)) + result = ntc.sendMail(to, subject, body, origin, localAttempt=False) + if not result["OK"]: + gLogger.error(result["Message"]) + DIRACexit(6) + + DIRACexit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_uninstall_component.py b/src/DIRAC/FrameworkSystem/scripts/dirac_uninstall_component.py new file mode 100755 index 00000000000..c4d2b4ceb81 --- /dev/null +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_uninstall_component.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python + +""" +Uninstallation of a DIRAC component +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import socket +from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient +from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities +from DIRAC import gLogger, S_OK +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.Core.Utilities.PromptUser import promptUser +from DIRAC import exit as DIRACexit +from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + +__RCSID__ = "$Id$" + +gComponentInstaller.exitOnError = True + +force = False + + +def setForce(opVal): + global force + force = True + return S_OK() + + +@DIRACScript() +def main(): + global force + Script.registerSwitch("f", "force", "Forces the removal of the logs", setForce) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, + 'Arguments:', + ' System: Name of the DIRAC system (ie: WorkloadManagement)', + ' Component: Name of the DIRAC component (ie: Matcher)'])) + + Script.parseCommandLine() + args = Script.getPositionalArgs() + + if len(args) == 1: + args = args[0].split('/') + + if len(args) < 2: + Script.showHelp(exitCode=1) + + system = args[0] + component = args[1] + + monitoringClient = ComponentMonitoringClient() + result = monitoringClient.getInstallations({'Instance': component, 'UnInstallationTime': None}, + {'System': system}, + {'HostName': socket.getfqdn()}, True) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + if len(result['Value']) < 1: + gLogger.warn('Given component does not exist') + DIRACexit(1) + if len(result['Value']) > 1: + gLogger.error('Too many components match') + DIRACexit(1) + + removeLogs = False + if force: + removeLogs = True + else: + if result['Value'][0]['Component']['Type'] in gComponentInstaller.componentTypes: + result = promptUser('Remove logs?', ['y', 'n'], 'n') + if result['OK']: + removeLogs = result['Value'] == 'y' + else: + gLogger.error(result['Message']) + DIRACexit(1) + + result = gComponentInstaller.uninstallComponent(system, component, removeLogs) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + result = MonitoringUtilities.monitorUninstallation(system, component) + if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + gLogger.notice('Successfully uninstalled component %s/%s' % (system, component)) + DIRACexit() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-add-group.py b/src/DIRAC/Interfaces/scripts/dirac-admin-add-group.py deleted file mode 100755 index a97a8bce590..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-add-group.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -# pylint: disable=wrong-import-position - -import DIRAC -from DIRAC.Core.Base import Script - -groupName = None -groupProperties = [] -userNames = [] - - -def setGroupName(arg): - global groupName - if groupName or not arg: - Script.showHelp(exitCode=1) - groupName = arg - - -def addUserName(arg): - global userNames - if not arg: - Script.showHelp(exitCode=1) - if arg not in userNames: - userNames.append(arg) - - -def addProperty(arg): - global groupProperties - if not arg: - Script.showHelp(exitCode=1) - if arg not in groupProperties: - groupProperties.append(arg) - - -Script.setUsageMessage('\n'.join(['Add or Modify a Group info in DIRAC', - '\nUsage:\n', - ' %s [option|cfgfile] ... Property= ...' % Script.scriptName, - '\nArguments:\n', - ' Property=: Other properties to be added to the User like (VOMSRole=XXXX)', - ])) - -Script.registerSwitch('G:', 'GroupName:', 'Name of the Group (Mandatory)', setGroupName) -Script.registerSwitch( - 'U:', - 'UserName:', - 'Short Name of user to be added to the Group (Allow Multiple instances or None)', - addUserName) -Script.registerSwitch( - 'P:', - 'Property:', - 'Property to be added to the Group (Allow Multiple instances or None)', - addProperty) - -Script.parseCommandLine(ignoreErrors=True) - -if groupName is None: - Script.showHelp(exitCode=1) - -args = Script.getPositionalArgs() - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -groupProps = {} -if userNames: - groupProps['Users'] = ', '.join(userNames) -if groupProperties: - groupProps['Properties'] = ', '.join(groupProperties) - -for prop in args: - pl = prop.split("=") - if len(pl) < 2: - errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) - exitCode = 255 - else: - pName = pl[0] - pValue = "=".join(pl[1:]) - Script.gLogger.info("Setting property %s to %s" % (pName, pValue)) - groupProps[pName] = pValue - -if not diracAdmin.csModifyGroup(groupName, groupProps, createIfNonExistant=True)['OK']: - errorList.append(("add group", "Cannot register group %s" % groupName)) - exitCode = 255 -else: - result = diracAdmin.csCommitChanges() - if not result['OK']: - errorList.append(("commit", result['Message'])) - exitCode = 255 - -for error in errorList: - Script.gLogger.error("%s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-add-host.py b/src/DIRAC/Interfaces/scripts/dirac-admin-add-host.py deleted file mode 100755 index 611913b4ebe..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-add-host.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" -import DIRAC -from DIRAC.Core.Base import Script - -hostName = None -hostDN = None -hostProperties = [] - - -def setHostName(arg): - global hostName - if hostName or not arg: - Script.showHelp(exitCode=1) - hostName = arg - - -def setHostDN(arg): - global hostDN - if hostDN or not arg: - Script.showHelp(exitCode=1) - hostDN = arg - - -def addProperty(arg): - global hostProperties - if not arg: - Script.showHelp(exitCode=1) - if arg not in hostProperties: - hostProperties.append(arg) - - -Script.setUsageMessage('\n'.join(['Add or Modify a Host info in DIRAC', - '\nUsage:\n', - ' %s [option|cfgfile] ... Property= ...' % Script.scriptName, - '\nArguments:\n', - ' Property=: Other properties to be added to the User like (Responsible=XXX)', - ])) - -Script.registerSwitch('H:', 'HostName:', 'Name of the Host (Mandatory)', setHostName) -Script.registerSwitch('D:', 'HostDN:', 'DN of the Host Certificate (Mandatory)', setHostDN) -Script.registerSwitch( - 'P:', - 'Property:', - 'Property to be added to the Host (Allow Multiple instances or None)', - addProperty) - -Script.parseCommandLine(ignoreErrors=True) - -if hostName is None or hostDN is None: - Script.showHelp(exitCode=1) - -args = Script.getPositionalArgs() - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -hostProps = {'DN': hostDN} -if hostProperties: - hostProps['Properties'] = ', '.join(hostProperties) - -for prop in args: - pl = prop.split("=") - if len(pl) < 2: - errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) - exitCode = 255 - else: - pName = pl[0] - pValue = "=".join(pl[1:]) - Script.gLogger.info("Setting property %s to %s" % (pName, pValue)) - hostProps[pName] = pValue - -if not diracAdmin.csModifyHost(hostName, hostProps, createIfNonExistant=True)['OK']: - errorList.append(("add host", "Cannot register host %s" % hostName)) - exitCode = 255 -else: - result = diracAdmin.csCommitChanges() - if not result['OK']: - errorList.append(("commit", result['Message'])) - exitCode = 255 - -if exitCode == 0: - from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient - cmc = ComponentMonitoringClient() - ret = cmc.hostExists(dict(HostName=hostName)) - if not ret['OK']: - Script.gLogger.error('Cannot check if host is registered in ComponentMonitoring', ret['Message']) - elif ret['Value']: - Script.gLogger.info('Host already registered in ComponentMonitoring') - else: - ret = cmc.addHost(dict(HostName=hostName, CPU='TO_COME')) - if not ret['OK']: - Script.gLogger.error('Failed to add Host to ComponentMonitoring', ret['Message']) - -for error in errorList: - Script.gLogger.error("%s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-add-user.py b/src/DIRAC/Interfaces/scripts/dirac-admin-add-user.py deleted file mode 100755 index 83a3931e121..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-add-user.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -userName = None -userDN = None -userMail = None -userGroups = [] - - -def setUserName(arg): - global userName - if userName or not arg: - Script.showHelp(exitCode=1) - userName = arg - - -def setUserDN(arg): - global userDN - if userDN or not arg: - Script.showHelp(exitCode=1) - userDN = arg - - -def setUserMail(arg): - global userMail - if userMail or not arg: - Script.showHelp(exitCode=1) - if not arg.find('@') > 0: - Script.gLogger.error('Not a valid mail address', arg) - DIRAC.exit(-1) - userMail = arg - - -def addUserGroup(arg): - global userGroups - if not arg: - Script.showHelp(exitCode=1) - if arg not in userGroups: - userGroups.append(arg) - - -Script.setUsageMessage('\n'.join(['Add or Modify a User info in DIRAC', - 'Usage:', - ' %s [option|cfgfile] ... Property= ...' % Script.scriptName, - 'Arguments:', - ' Property=: Properties to be added to the User like (Phone=XXXX)', ])) - -Script.registerSwitch('N:', 'UserName:', 'Short Name of the User (Mandatory)', setUserName) -Script.registerSwitch('D:', 'UserDN:', 'DN of the User Certificate (Mandatory)', setUserDN) -Script.registerSwitch('M:', 'UserMail:', 'eMail of the user (Mandatory)', setUserMail) -Script.registerSwitch( - 'G:', - 'UserGroup:', - 'Name of the Group for the User (Allow Multiple instances or None)', - addUserGroup) - -Script.parseCommandLine(ignoreErrors=True) - -if userName is None or userDN is None or userMail is None: - Script.showHelp(exitCode=1) - -args = Script.getPositionalArgs() - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -userProps = {'DN': userDN, 'Email': userMail} -if userGroups: - userProps['Groups'] = userGroups -for prop in args: - pl = prop.split("=") - if len(pl) < 2: - errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) - exitCode = 255 - else: - pName = pl[0] - pValue = "=".join(pl[1:]) - Script.gLogger.info("Setting property %s to %s" % (pName, pValue)) - userProps[pName] = pValue - -if not diracAdmin.csModifyUser(userName, userProps, createIfNonExistant=True)['OK']: - errorList.append(("add user", "Cannot register user %s" % userName)) - exitCode = 255 -else: - result = diracAdmin.csCommitChanges() - if not result['OK']: - errorList.append(("commit", result['Message'])) - exitCode = 255 - -for error in errorList: - Script.gLogger.error("%s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-allow-site.py b/src/DIRAC/Interfaces/scripts/dirac-admin-allow-site.py deleted file mode 100755 index c249925598d..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-allow-site.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-allow-site -# Author : Stuart Paterson -######################################################################## -""" - Add Site to Active mask for current Setup -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import time - -from DIRAC.Core.Base import Script - -Script.registerSwitch("E:", "email=", "Boolean True/False (True by default)") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Site Comment' % Script.scriptName, - 'Arguments:', - ' Site: Name of the Site', - ' Comment: Reason of the action'])) -Script.parseCommandLine(ignoreErrors=True) - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC import exit as DIRACExit, gConfig, gLogger - - -def getBoolean(value): - if value.lower() == 'true': - return True - elif value.lower() == 'false': - return False - else: - Script.showHelp() - - -email = True -for switch in Script.getUnprocessedSwitches(): - if switch[0] == "email": - email = getBoolean(switch[1]) - -args = Script.getPositionalArgs() - -if len(args) < 2: - Script.showHelp() - -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] -setup = gConfig.getValue('/DIRAC/Setup', '') -if not setup: - print('ERROR: Could not contact Configuration Service') - exitCode = 2 - DIRACExit(exitCode) - -#result = promptUser( 'All the elements that are associated with this site will be active, are you sure about this action?' ) -# if not result['OK'] or result['Value'] is 'n': -# print 'Script stopped' -# DIRACExit( 0 ) - -site = args[0] -comment = args[1] -result = diracAdmin.allowSite(site, comment, printOutput=True) -if not result['OK']: - errorList.append((site, result['Message'])) - exitCode = 2 -else: - if email: - userName = diracAdmin._getCurrentUser() - if not userName['OK']: - print('ERROR: Could not obtain current username from proxy') - exitCode = 2 - DIRACExit(exitCode) - userName = userName['Value'] - subject = '%s is added in site mask for %s setup' % (site, setup) - body = 'Site %s is added to the site mask for %s setup by %s on %s.\n\n' % (site, setup, userName, time.asctime()) - body += 'Comment:\n%s' % comment - addressPath = 'EMail/Production' - address = Operations().getValue(addressPath, '') - if not address: - gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) - else: - result = diracAdmin.sendMail(address, subject, body) - else: - print('Automatic email disabled by flag.') - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-ban-site.py b/src/DIRAC/Interfaces/scripts/dirac-admin-ban-site.py deleted file mode 100755 index abaf453bd0f..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-ban-site.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-ban-site -# Author : Stuart Paterson -######################################################################## -""" - Remove Site from Active mask for current Setup -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import time - -from DIRAC import exit as DIRACExit, gConfig, gLogger -from DIRAC.Core.Base import Script - -Script.registerSwitch("E:", "email=", "Boolean True/False (True by default)") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Site Comment' % Script.scriptName, - 'Arguments:', - ' Site: Name of the Site', - ' Comment: Reason of the action'])) -Script.parseCommandLine(ignoreErrors=True) - -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.Core.Utilities.PromptUser import promptUser -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin - - -def getBoolean(value): - if value.lower() == 'true': - return True - elif value.lower() == 'false': - return False - else: - Script.showHelp() - - -email = True -for switch in Script.getUnprocessedSwitches(): - if switch[0] == "email": - email = getBoolean(switch[1]) - -args = Script.getPositionalArgs() - -if len(args) < 2: - Script.showHelp() - -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] -setup = gConfig.getValue('/DIRAC/Setup', '') -if not setup: - print('ERROR: Could not contact Configuration Service') - exitCode = 2 - DIRACExit(exitCode) - -#result = promptUser( 'All the elements that are associated with this site will be banned, are you sure about this action?' ) -# if not result['OK'] or result['Value'] is 'n': -# print 'Script stopped' -# DIRACExit( 0 ) - -site = args[0] -comment = args[1] -result = diracAdmin.banSite(site, comment, printOutput=True) -if not result['OK']: - errorList.append((site, result['Message'])) - exitCode = 2 -else: - if email: - userName = diracAdmin._getCurrentUser() - if not userName['OK']: - print('ERROR: Could not obtain current username from proxy') - exitCode = 2 - DIRACExit(exitCode) - userName = userName['Value'] - subject = '%s is banned for %s setup' % (site, setup) - body = 'Site %s is removed from site mask for %s setup by %s on %s.\n\n' % (site, setup, userName, time.asctime()) - body += 'Comment:\n%s' % comment - - addressPath = 'EMail/Production' - address = Operations().getValue(addressPath, '') - if not address: - gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) - else: - result = diracAdmin.sendMail(address, subject, body) - else: - print('Automatic email disabled by flag.') - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-ce-info.py b/src/DIRAC/Interfaces/scripts/dirac-admin-ce-info.py deleted file mode 100755 index c2138950e6a..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-ce-info.py +++ /dev/null @@ -1,42 +0,0 @@ -#! /usr/bin/env python -######################################################################## -# File : dirac-admin-ce-info -# Author : Vladimir Romanovsky -######################################################################## -""" - Retrieve Site Associated to a given CE -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -from DIRAC import gConfig, gLogger, exit as Dexit -from DIRAC.Core.Base import Script -from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping -from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath - - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... CE ...' % Script.scriptName, - 'Arguments:', - ' CE: Name of the CE (mandatory)'])) - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) - -res = getCESiteMapping(args[0]) -if not res['OK']: - gLogger.error(res['Message']) - Dexit(1) -site = res['Value'][args[0]] - -res = gConfig.getOptionsDict(cfgPath('Resources', 'Sites', site.split('.')[0], site, 'CEs', args[0])) -if not res['OK']: - gLogger.error(res['Message']) - Dexit(1) -gLogger.notice(res['Value']) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-delete-user.py b/src/DIRAC/Interfaces/scripts/dirac-admin-delete-user.py deleted file mode 100755 index a778cb41bae..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-delete-user.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# $HeadURL$ -# File : dirac-admin-delete-user -# Author : Adrian Casajus -######################################################################## -""" - Remove User from Configuration -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... User ...' % Script.scriptName, - 'Arguments:', - ' User: User name'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -from DIRAC import exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -if len(args) < 1: - Script.showHelp() - -choice = raw_input("Are you sure you want to delete user/s %s? yes/no [no]: " % ", ".join(args)) -choice = choice.lower() -if choice not in ("yes", "y"): - print("Delete aborted") - DIRACExit(0) - -for user in args: - if not diracAdmin.csDeleteUser(user): - errorList.append(("delete user", "Cannot delete user %s" % user)) - exitCode = 255 - -if not exitCode: - result = diracAdmin.csCommitChanges() - if not result['OK']: - errorList.append(("commit", result['Message'])) - exitCode = 255 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-banned-sites.py b/src/DIRAC/Interfaces/scripts/dirac-admin-get-banned-sites.py deleted file mode 100755 index 36b505b94a5..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-get-banned-sites.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-get-banned-sites -# Author : Stuart Paterson -######################################################################## - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script -Script.parseCommandLine(ignoreErrors=True) - -from DIRAC import gLogger, exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin - -diracAdmin = DiracAdmin() - -result = diracAdmin.getBannedSites() -if result['OK']: - bannedSites = result['Value'] -else: - gLogger.error(result['Message']) - DIRACExit(2) - -for site in bannedSites: - result = diracAdmin.getSiteMaskLogging(site, printOutput=True) - if not result['OK']: - gLogger.error(result['Message']) - DIRACExit(2) - -DIRACExit(0) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-job-pilot-output.py b/src/DIRAC/Interfaces/scripts/dirac-admin-get-job-pilot-output.py deleted file mode 100755 index 9bfaa1fa193..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-get-job-pilot-output.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-get-job-pilot-output -# Author : Stuart Paterson -######################################################################## -""" - Retrieve the output of the pilot that executed a given job -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC ID of the Job'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC import exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -for job in args: - - try: - job = int(job) - except Exception as x: - errorList.append(('Expected integer for JobID', job)) - exitCode = 2 - continue - - result = diracAdmin.getJobPilotOutput(job) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-job-pilots.py b/src/DIRAC/Interfaces/scripts/dirac-admin-get-job-pilots.py deleted file mode 100755 index 063ca125a26..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-get-job-pilots.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-get-job-pilots -# Author : Stuart Paterson -######################################################################## -""" - Retrieve info about pilots that have matched a given Job -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -# pylint: disable=wrong-import-position - -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC ID of the Job'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC import exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -for job in args: - - try: - job = int(job) - except BaseException as x: - errorList.append((job, 'Expected integer for jobID')) - exitCode = 2 - continue - - result = diracAdmin.getJobPilots(job) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-info.py b/src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-info.py deleted file mode 100755 index 3e4444eb909..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-info.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-get-pilot-output -# Author : Ricardo Graciani -######################################################################## -""" - Retrieve available info about the given pilot -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -# pylint: disable=wrong-import-position - -from DIRAC import exit as DIRACExit -from DIRAC.Core.Base import Script - -extendedPrint = False - - -def setExtendedPrint(_arg): - global extendedPrint - extendedPrint = True - - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, - 'Arguments:', - ' PilotID: Grid ID of the pilot'])) -Script.registerSwitch('e', 'extended', 'Get extended printout', setExtendedPrint) -Script.parseCommandLine(ignoreErrors=True) - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -from DIRAC.Interfaces.API.Dirac import Dirac - -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - - -diracAdmin = DiracAdmin() -dirac = Dirac() -exitCode = 0 -errorList = [] - -for gridID in args: - - result = diracAdmin.getPilotInfo(gridID) - if not result['OK']: - errorList.append((gridID, result['Message'])) - exitCode = 2 - else: - res = result['Value'][gridID] - if extendedPrint: - tab = '' - for key in [ - 'PilotJobReference', - 'Status', - 'OwnerDN', - 'OwnerGroup', - 'SubmissionTime', - 'DestinationSite', - 'GridSite', - ]: - if key in res: - diracAdmin.log.notice('%s%s: %s' % (tab, key, res[key])) - if not tab: - tab = ' ' - diracAdmin.log.notice('') - for jobID in res['Jobs']: - tab = ' ' - result = dirac.getJobAttributes(int(jobID)) - if not result['OK']: - errorList.append((gridID, result['Message'])) - exitCode = 2 - else: - job = result['Value'] - diracAdmin.log.notice('%sJob ID: %s' % (tab, jobID)) - tab += ' ' - for key in ['OwnerDN', 'OwnerGroup', 'JobName', 'Status', 'StartExecTime', 'LastUpdateTime', 'EndExecTime']: - if key in job: - diracAdmin.log.notice('%s%s:' % (tab, key), job[key]) - diracAdmin.log.notice('') - else: - print(diracAdmin.pPrint.pformat({gridID: res})) - - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-logging-info.py b/src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-logging-info.py deleted file mode 100755 index 8b6dfa40c43..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-logging-info.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-get-pilot-logging-info.py -# Author : Stuart Paterson -######################################################################## -""" - Retrieve logging info of a Grid pilot -""" - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -# pylint: disable=wrong-import-position - -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, - 'Arguments:', - ' PilotID: Grid ID of the pilot'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC import exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -for gridID in args: - - result = diracAdmin.getPilotLoggingInfo(gridID) - if not result['OK']: - errorList.append((gridID, result['Message'])) - exitCode = 2 - else: - print('Pilot Reference: %s', gridID) - print(result['Value']) - print() - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-output.py b/src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-output.py deleted file mode 100755 index bb64a6c531b..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-get-pilot-output.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# $HeadURL$ -# File : dirac-admin-get-pilot-output -# Author : Stuart Paterson -######################################################################## -""" - Retrieve output of a Grid pilot -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script - - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, - 'Arguments:', - ' PilotID: Grid ID of the pilot'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC import exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -for gridID in args: - - result = diracAdmin.getPilotOutput(gridID) - if not result['OK']: - errorList.append((gridID, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-get-site-mask.py b/src/DIRAC/Interfaces/scripts/dirac-admin-get-site-mask.py deleted file mode 100755 index 0dc5089a0f4..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-get-site-mask.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-get-site-mask -# Author : Stuart Paterson -######################################################################## - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - - -from DIRAC.Core.Base import Script - -Script.setUsageMessage(""" -Get the list of sites enabled in the mask for job submission - -Usage: - %s [options] -""" % Script.scriptName) - -Script.parseCommandLine(ignoreErrors=True) - -from DIRAC import exit as DIRACExit, gLogger -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin - -diracAdmin = DiracAdmin() - -gLogger.setLevel('ALWAYS') - -result = diracAdmin.getSiteMask(printOutput=True, status="Active") -if result['OK']: - DIRACExit(0) -else: - print(result['Message']) - DIRACExit(2) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-list-hosts.py b/src/DIRAC/Interfaces/scripts/dirac-admin-list-hosts.py deleted file mode 100755 index b95fd7a9d5a..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-list-hosts.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-list-users -# Author : Adrian Casajus -######################################################################## -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script - -Script.registerSwitch("e", "extended", "Show extended info") - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -from DIRAC import exit as DIRACExit -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] -extendedInfo = False - -for unprocSw in Script.getUnprocessedSwitches(): - if unprocSw[0] in ('e', 'extended'): - extendedInfo = True - -if not extendedInfo: - result = diracAdmin.csListHosts() - for host in result['Value']: - print(" %s" % host) -else: - result = diracAdmin.csDescribeHosts() - print(diracAdmin.pPrint.pformat(result['Value'])) - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRACExit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-list-users.py b/src/DIRAC/Interfaces/scripts/dirac-admin-list-users.py deleted file mode 100755 index 4834f52a293..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-list-users.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-list-users -# Author : Adrian Casajus -######################################################################## -""" - Lists the users in the Configuration. If no group is specified return all users. -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" -import DIRAC -from DIRAC.Core.Base import Script - -Script.registerSwitch("e", "extended", "Show extended info") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... [Group] ...' % Script.scriptName, - 'Arguments:', - ' Group: Only users from this group (default: all)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) == 0: - args = ['all'] - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] -extendedInfo = False - -for unprocSw in Script.getUnprocessedSwitches(): - if unprocSw[0] in ('e', 'extended'): - extendedInfo = True - - -def printUsersInGroup(group=False): - result = diracAdmin.csListUsers(group) - if result['OK']: - if group: - print("Users in group %s:" % group) - else: - print("All users registered:") - for username in result['Value']: - print(" %s" % username) - - -def describeUsersInGroup(group=False): - result = diracAdmin.csListUsers(group) - if result['OK']: - if group: - print("Users in group %s:" % group) - else: - print("All users registered:") - result = diracAdmin.csDescribeUsers(result['Value']) - print(diracAdmin.pPrint.pformat(result['Value'])) - - -for group in args: - if 'all' in args: - group = False - if not extendedInfo: - printUsersInGroup(group) - else: - describeUsersInGroup(group) - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-modify-user.py b/src/DIRAC/Interfaces/scripts/dirac-admin-modify-user.py deleted file mode 100755 index c5db766b6ef..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-modify-user.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-modify-user -# Author : Adrian Casajus -######################################################################## -""" - Modify a user in the CS. -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.registerSwitch("p:", "property=", "Add property to the user =") -Script.registerSwitch("f", "force", "create the user if it doesn't exist") - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... user DN group [group] ...' % Script.scriptName, - 'Arguments:', - ' user: User name (mandatory)', - ' DN: DN of the User (mandatory)', - ' group: Add the user to the group (mandatory)'])) -Script.parseCommandLine(ignoreErrors=True) - -args = Script.getPositionalArgs() - -if len(args) < 3: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -forceCreation = False -errorList = [] - -userProps = {} -for unprocSw in Script.getUnprocessedSwitches(): - if unprocSw[0] in ("f", "force"): - forceCreation = True - elif unprocSw[0] in ("p", "property"): - prop = unprocSw[1] - pl = prop.split("=") - if len(pl) < 2: - errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) - exitCode = 255 - else: - pName = pl[0] - pValue = "=".join(pl[1:]) - print("Setting property %s to %s" % (pName, pValue)) - userProps[pName] = pValue - -userName = args[0] -userProps['DN'] = args[1] -userProps['Groups'] = args[2:] - -if not diracAdmin.csModifyUser(userName, userProps, createIfNonExistant=forceCreation): - errorList.append(("modify user", "Cannot modify user %s" % userName)) - exitCode = 255 -else: - result = diracAdmin.csCommitChanges() - if not result['OK']: - errorList.append(("commit", result['Message'])) - exitCode = 255 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-reset-job.py b/src/DIRAC/Interfaces/scripts/dirac-admin-reset-job.py deleted file mode 100755 index 08f14c594d5..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-reset-job.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-reset-job -# Author : Stuart Paterson -######################################################################## -""" - Reset a job or list of jobs in the WMS -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC ID of the Job'])) -Script.parseCommandLine(ignoreErrors=True) - -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -for job in args: - - try: - job = int(job) - except Exception as x: - errorList.append(('Expected integer for jobID', job)) - exitCode = 2 - continue - - result = diracAdmin.resetJob(job) - if result['OK']: - print('Reset Job %s' % (job)) - else: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-service-ports.py b/src/DIRAC/Interfaces/scripts/dirac-admin-service-ports.py deleted file mode 100755 index 07fdd6dee6f..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-service-ports.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-service-ports -# Author : Stuart Paterson -######################################################################## -""" - Print the service ports for the specified setup -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... [Setup]' % Script.scriptName, - 'Arguments:', - ' Setup: Name of the setup'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -setup = '' -if args: - setup = args[0] - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -result = diracAdmin.getServicePorts(setup, printOutput=True) -if result['OK']: - DIRAC.exit(0) -else: - print(result['Message']) - DIRAC.exit(2) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-set-site-protocols.py b/src/DIRAC/Interfaces/scripts/dirac-admin-set-site-protocols.py deleted file mode 100755 index 7b57194c52b..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-set-site-protocols.py +++ /dev/null @@ -1,44 +0,0 @@ -#! /usr/bin/env python -######################################################################## -# File : dirac-admin-set-site-protocols -# Author : Stuart Paterson -######################################################################## -""" - Defined protocols for each SE for a given site. -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.registerSwitch("", "Site=", "Site for which protocols are to be set (mandatory)") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Protocol ...' % Script.scriptName, - 'Arguments:', - ' Protocol: SE access protocol (mandatory)'])) -Script.parseCommandLine(ignoreErrors=True) - -site = None -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "site": - site = switch[1] - -args = Script.getPositionalArgs() - -if not site or not args: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -result = diracAdmin.setSiteProtocols(site, args, printOutput=True) -if not result['OK']: - print('ERROR: %s' % result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-site-info.py b/src/DIRAC/Interfaces/scripts/dirac-admin-site-info.py deleted file mode 100755 index 37cfd46c9bc..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-site-info.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-site-info -# Author : Stuart Paterson -######################################################################## -""" - Print Configuration information for a given Site -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Site ...' % Script.scriptName, - 'Arguments:', - ' Site: Name of the Site'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -for site in args: - - result = diracAdmin.getSiteSection(site, printOutput=True) - if not result['OK']: - errorList.append((site, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-site-mask-logging.py b/src/DIRAC/Interfaces/scripts/dirac-admin-site-mask-logging.py deleted file mode 100755 index f17d6e1c757..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-site-mask-logging.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -""" - Retrieves site mask logging information. -""" - -######################################################################## -# File : dirac-admin-site-mask-logging -# Author : Stuart Paterson -######################################################################## - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Site ...' % Script.scriptName, - 'Arguments:', - ' Site: Name of the Site'])) - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -errorList = [] - -for site in args: - result = diracAdmin.getSiteMaskLogging(site, printOutput=True) - if not result['OK']: - errorList.append((site, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-sync-users-from-file.py b/src/DIRAC/Interfaces/scripts/dirac-admin-sync-users-from-file.py deleted file mode 100755 index f184be0c8fe..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-sync-users-from-file.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-sync-users-from-file -# Author : Adrian Casajus -######################################################################## -""" - Sync users in Configuration with the cfg contents. -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from diraccfg import CFG - -import DIRAC -from DIRAC.Core.Base import Script - -__RCSID__ = "$Id$" - -Script.registerSwitch("t", "test", "Only test. Don't commit changes") -Script.setUsageMessage( - '\n'.join( - [ - __doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... UserCfg' % - Script.scriptName, - 'Arguments:', - ' UserCfg: Cfg FileName with Users as sections containing DN, Groups, and other properties as options'])) -Script.parseCommandLine(ignoreErrors=True) - -args = Script.getExtraCLICFGFiles() - -if len(args) < 1: - Script.showHelp() - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 -testOnly = False -errorList = [] - -for unprocSw in Script.getUnprocessedSwitches(): - if unprocSw[0] in ("t", "test"): - testOnly = True - -try: - usersCFG = CFG().loadFromFile(args[0]) -except Exception as e: - errorList.append("file open", "Can't parse file %s: %s" % (args[0], str(e))) - errorCode = 1 -else: - if not diracAdmin.csSyncUsersWithCFG(usersCFG): - errorList.append(("modify users", "Cannot sync with %s" % args[0])) - exitCode = 255 - -if not exitCode and not testOnly: - result = diracAdmin.csCommitChanges() - if not result['OK']: - errorList.append(("commit", result['Message'])) - exitCode = 255 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-get-file.py b/src/DIRAC/Interfaces/scripts/dirac-dms-get-file.py deleted file mode 100755 index 228e6ca8713..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-dms-get-file.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-dms-get-file -# Author : Stuart Paterson -######################################################################## -""" - Retrieve a single file or list of files from Grid storage to the current directory. -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, - 'Arguments:', - ' LFN: Logical File Name or file containing LFNs'])) -Script.parseCommandLine(ignoreErrors=True) -lfns = Script.getPositionalArgs() - -if len(lfns) < 1: - Script.showHelp() - -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 - -if len(lfns) == 1: - try: - with open(lfns[0], 'r') as f: - lfns = f.read().splitlines() - except Exception: - pass - -result = dirac.getFile(lfns, printOutput=True) -if not result['OK']: - print('ERROR %s' % (result['Message'])) - exitCode = 2 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-lfn-accessURL.py b/src/DIRAC/Interfaces/scripts/dirac-dms-lfn-accessURL.py deleted file mode 100755 index d0acb761a94..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-dms-lfn-accessURL.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-dms-lfn-accessURL -# Author : Stuart Paterson -######################################################################## -""" - Retrieve an access URL for an LFN replica given a valid DIRAC SE. -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... LFN SE [PROTO]' % Script.scriptName, - 'Arguments:', - ' LFN: Logical File Name or file containing LFNs (mandatory)', - ' SE: Valid DIRAC SE (mandatory)', - ' PROTO: Optional protocol for accessURL'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -# pylint: disable=wrong-import-position -from DIRAC.Interfaces.API.Dirac import Dirac - -if len(args) < 2: - Script.showHelp(exitCode=1) - -if len(args) > 3: - print('Only one LFN SE pair will be considered') - -dirac = Dirac() -exitCode = 0 - -lfn = args[0] -seName = args[1] -proto = False -if len(args) > 2: - proto = args[2] - -try: - with open(lfn, 'r') as f: - lfns = f.read().splitlines() -except IOError: - lfns = [lfn] - -for lfn in lfns: - result = dirac.getAccessURL(lfn, seName, protocol=proto, printOutput=True) - if not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-lfn-metadata.py b/src/DIRAC/Interfaces/scripts/dirac-dms-lfn-metadata.py deleted file mode 100755 index fed4db4d6ff..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-dms-lfn-metadata.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-lfn-metadata -# Author : Stuart Paterson -######################################################################## -""" - Obtain replica metadata from file catalogue client. -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, - 'Arguments:', - ' LFN: Logical File Name or file containing LFNs'])) -Script.parseCommandLine(ignoreErrors=True) -lfns = Script.getPositionalArgs() - -if len(lfns) < 1: - Script.showHelp() - -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 -errorList = [] - -if len(lfns) == 1: - try: - with open(lfns[0], 'r') as f: - lfns = f.read().splitlines() - except BaseException: - pass - -result = dirac.getLfnMetadata(lfns, printOutput=True) -if not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-lfn-replicas.py b/src/DIRAC/Interfaces/scripts/dirac-dms-lfn-replicas.py deleted file mode 100755 index 6067d91cf0c..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-dms-lfn-replicas.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-lfn-replicas -# Author : Stuart Paterson -######################################################################## -""" - Obtain replica information from file catalogue client. -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.registerSwitch('a', "All", " Also show inactive replicas") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, - 'Arguments:', - ' LFN: Logical File Name or file containing LFNs'])) -Script.parseCommandLine(ignoreErrors=True) -lfns = Script.getPositionalArgs() -switches = Script.getUnprocessedSwitches() - -active = True -for switch in switches: - opt = switch[0].lower() - if opt in ("a", "all"): - active = False -if len(lfns) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 - -if len(lfns) == 1: - try: - with open(lfns[0], 'r') as f: - lfns = f.read().splitlines() - except BaseException: - pass - -result = dirac.getReplicas(lfns, active=active, printOutput=True) -if not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-pfn-accessURL.py b/src/DIRAC/Interfaces/scripts/dirac-dms-pfn-accessURL.py deleted file mode 100755 index 805725dc573..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-dms-pfn-accessURL.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-dms-pfn-accessURL -# Author : Stuart Paterson -######################################################################## -""" - Retrieve an access URL for a PFN given a valid DIRAC SE -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... PFN SE' % Script.scriptName, - 'Arguments:', - ' PFN: Physical File Name or file containing PFNs (mandatory)', - ' SE: Valid DIRAC SE (mandatory)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 2: - Script.showHelp(exitCode=1) - -if len(args) > 2: - print('Only one PFN SE pair will be considered') - -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 - -pfn = args[0] -seName = args[1] -try: - with open(pfn, 'r') as f: - pfns = f.read().splitlines() -except BaseException: - pfns = [pfn] - -for pfn in pfns: - result = dirac.getPhysicalFileAccessURL(pfn, seName, printOutput=True) - if not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-pfn-metadata.py b/src/DIRAC/Interfaces/scripts/dirac-dms-pfn-metadata.py deleted file mode 100755 index c2be191f7b9..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-dms-pfn-metadata.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-dms-pfn-metadata -# Author : Stuart Paterson -######################################################################## -""" - Retrieve metadata for a PFN given a valid DIRAC SE -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... PFN SE' % Script.scriptName, - 'Arguments:', - ' PFN: Physical File Name or file containing PFNs (mandatory)', - ' SE: Valid DIRAC SE (mandatory)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 2: - Script.showHelp(exitCode=1) - -if len(args) > 2: - print('Only one PFN SE pair will be considered') - -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 - -pfn = args[0] -seName = args[1] -try: - f = open(pfn, 'r') - pfns = f.read().splitlines() - f.close() -except BaseException: - pfns = [pfn] - -for pfn in pfns: - result = dirac.getPhysicalFileMetadata(pfn, seName, printOutput=True) - if not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-dms-replicate-lfn.py b/src/DIRAC/Interfaces/scripts/dirac-dms-replicate-lfn.py deleted file mode 100755 index 1fe1ad70b00..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-dms-replicate-lfn.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-dms-replicate-lfn -# Author : Stuart Paterson -######################################################################## -""" - Replicate an existing LFN to another Storage Element -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... LFN Dest [Source [Cache]]' % Script.scriptName, - 'Arguments:', - ' LFN: Logical File Name or file containing LFNs (mandatory)', - ' Dest: Valid DIRAC SE (mandatory)', - ' Source: Valid DIRAC SE', - ' Cache: Local directory to be used as cache'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 2 or len(args) > 4: - Script.showHelp(exitCode=1) - -lfn = args[0] -seName = args[1] -sourceSE = '' -localCache = '' -if len(args) > 2: - sourceSE = args[2] -if len(args) == 4: - localCache = args[3] - -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 - -try: - f = open(lfn, 'r') - lfns = f.read().splitlines() - f.close() -except BaseException: - lfns = [lfn] - -finalResult = {"Failed": [], "Successful": []} -for lfn in lfns: - result = dirac.replicateFile(lfn, seName, sourceSE, localCache, printOutput=True) - if not result['OK']: - finalResult["Failed"].append(lfn) - print('ERROR %s' % (result['Message'])) - exitCode = 2 - else: - finalResult["Successful"].append(lfn) - -if len(lfns) > 1: - print(finalResult) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-framework-ping-service.py b/src/DIRAC/Interfaces/scripts/dirac-framework-ping-service.py deleted file mode 100755 index c809d16498f..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-framework-ping-service.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-framework-ping-service -# Author : Stuart Paterson -######################################################################## -""" - Ping the given DIRAC Service -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... System Service|System/Agent' % Script.scriptName, - 'Arguments:', - ' System: Name of the DIRAC system (ie: WorkloadManagement) (mandatory)', - ' Service: Name of the DIRAC service (ie: Matcher) (mandatory)', - ' url: URL of the service to ping (instead of System and Service)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -system = None -service = None -url = None -if len(args) == 1: - # it is a URL - if args[0].startswith('dips://'): - url = args[0] - # It is System/Service - else: - sys_serv = args[0].split('/') - if len(sys_serv) != 2: - Script.showHelp(exitCode=1) - else: - system, service = sys_serv - -elif len(args) == 2: - system, service = args[0], args[1] -else: - Script.showHelp(exitCode=1) - - -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac() -exitCode = 0 - -result = dirac.pingService(system, service, printOutput=True, url=url) - -if not result: - print('ERROR: Null result from ping()') - exitCode = 2 -elif not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-framework-self-ping.py b/src/DIRAC/Interfaces/scripts/dirac-framework-self-ping.py deleted file mode 100755 index 54027ccb28a..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-framework-self-ping.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python - -""" Performs a DIPS ping on a given target and exit with the return code. -It uses the local host certificate -The target is specified as ""/System/Service" -The script does not print anything, and just exists with 0 in case of success, -or 1 in case of error """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - - -import sys -import os -import time - -with open(os.devnull, 'w') as redirectStdout, open(os.devnull, 'w') as redirectStderr: - from DIRAC import gLogger - from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData - gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true') - gLogger.setLevel('FATAL') - from DIRAC.Core.DISET.RPCClient import RPCClient - - rpc = RPCClient('dips://localhost:%s' % sys.argv[1]) - res = rpc.ping() - time.sleep(0.1) - if not res['OK']: - sys.exit(1) - else: - sys.exit(0) diff --git a/src/DIRAC/Interfaces/scripts/dirac-repo-monitor.py b/src/DIRAC/Interfaces/scripts/dirac-repo-monitor.py deleted file mode 100755 index ee799e3b9c4..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-repo-monitor.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -""" - Monitor the jobs present in the repository -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... RepoDir' % Script.scriptName, - 'Arguments:', - ' RepoDir: Location of Job Repository'])) -Script.parseCommandLine(ignoreErrors=False) -args = Script.getPositionalArgs() - -if len(args) != 1: - Script.showHelp() - -repoLocation = args[0] -from DIRAC.Interfaces.API.Dirac import Dirac -dirac = Dirac(withRepo=True, repoLocation=repoLocation) - -exitCode = 0 -result = dirac.monitorRepository(printOutput=True) -if not result['OK']: - print('ERROR: ', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-utils-file-adler.py b/src/DIRAC/Interfaces/scripts/dirac-utils-file-adler.py deleted file mode 100755 index 8feec1f20f0..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-utils-file-adler.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-utils-file-adler -######################################################################## -""" - Calculate alder32 of the supplied file -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Utilities.Adler import fileAdler -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... File ...' % Script.scriptName, - 'Arguments:', - ' File: File Name'])) -Script.parseCommandLine(ignoreErrors=False) -files = Script.getPositionalArgs() -if len(files) == 0: - Script.showHelp() - -exitCode = 0 - -for fa in files: - adler = fileAdler(fa) - if adler: - print(fa.rjust(100), adler.ljust(10)) # pylint: disable=no-member - else: - print('ERROR %s: Failed to get adler' % fa) - exitCode = 2 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-utils-file-md5.py b/src/DIRAC/Interfaces/scripts/dirac-utils-file-md5.py deleted file mode 100755 index 7ff4399fcf2..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-utils-file-md5.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-utils-file-adler -# Author : -######################################################################## -""" - Calculate md5 of the supplied file -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Utilities.File import makeGuid -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... File ...' % Script.scriptName, - 'Arguments:', - ' File: File Name'])) -Script.parseCommandLine(ignoreErrors=False) -files = Script.getPositionalArgs() -if len(files) == 0: - Script.showHelp() - -exitCode = 0 - -for file in files: - try: - md5 = makeGuid(file) - if md5: - print(file.rjust(100), md5.ljust(10)) - else: - print('ERROR %s: Failed to get md5' % file) - exitCode = 2 - except Exception as x: - print('ERROR %s: Failed to get md5' % file, str(x)) - exitCode = 2 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-get-normalized-queue-length.py b/src/DIRAC/Interfaces/scripts/dirac-wms-get-normalized-queue-length.py deleted file mode 100755 index e2684d72d74..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-get-normalized-queue-length.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-get-normalized-queue-length.py -# Author : Ricardo Graciani -######################################################################## -""" - Report Normalized CPU length of queue - - This script was used by the dirac-pilot script to set the CPUTime limit for the matching but now this is no more the case -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script -from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import queueNormalizedCPU - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Queue ...' % Script.scriptName, - 'Arguments:', - ' Queue: GlueCEUniqueID of the Queue (ie, juk.nikhef.nl:8443/cream-pbs-lhcb)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -exitCode = 0 - -for ceUniqueID in args: - - normCPU = queueNormalizedCPU(ceUniqueID) - - if not normCPU['OK']: - print('ERROR %s:' % ceUniqueID, normCPU['Message']) - exitCode = 2 - continue - print(ceUniqueID, normCPU['Value']) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-get-queue-normalization.py b/src/DIRAC/Interfaces/scripts/dirac-wms-get-queue-normalization.py deleted file mode 100755 index 313653d1508..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-get-queue-normalization.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-get-queue-normalization.py -# Author : Ricardo Graciani -######################################################################## -""" - Report Normalization Factor applied by Site to the given Queue -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script -from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import getQueueNormalization - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Queue ...' % Script.scriptName, - 'Arguments:', - ' Queue: GlueCEUniqueID of the Queue (ie, juk.nikhef.nl:8443/cream-pbs-lhcb)'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -exitCode = 0 - -for ceUniqueID in args: - - cpuNorm = getQueueNormalization(ceUniqueID) - - if not cpuNorm['OK']: - print('ERROR %s:' % ceUniqueID, cpuNorm['Message']) - exitCode = 2 - continue - print(ceUniqueID, cpuNorm['Value']) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-attributes.py b/src/DIRAC/Interfaces/scripts/dirac-wms-job-attributes.py deleted file mode 100755 index 11b3e89b191..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-attributes.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-job-attributes -# Author : Stuart Paterson -######################################################################## -""" - Retrieve attributes associated with the given DIRAC job -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] - -for job in parseArguments(args): - - result = dirac.getJobAttributes(job, printOutput=True) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-input.py b/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-input.py deleted file mode 100755 index d6c1229b011..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-input.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-production-job-get-input -# Author : Stuart Paterson -######################################################################## -""" - Retrieve input sandbox for DIRAC Job -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import os - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.registerSwitch("D:", "Dir=", "Store the output in this directory") -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] - -outputDir = None -for sw, v in Script.getUnprocessedSwitches(): - if sw in ('D', 'Dir'): - outputDir = v - -for job in parseArguments(args): - - result = dirac.getInputSandbox(job, outputDir=outputDir) - if result['OK']: - if os.path.exists('InputSandbox%s' % job): - print('Job input sandbox retrieved in InputSandbox%s/' % (job)) - else: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-jdl.py b/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-jdl.py deleted file mode 100755 index 0872c0bbc3e..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-jdl.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-job-get-jdl -# Author : Stuart Paterson -######################################################################## -""" - Retrieve the current JDL of a DIRAC job -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -original = False -Script.registerSwitch('O', 'Original', 'Gets the original JDL') -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -for switch in Script.getUnprocessedSwitches(): - if switch[0] == 'Original' or switch[0] == 'O': - original = True - -for switch in Script.getUnprocessedSwitches(): - if switch[0] == 'Original': - original = True - -if len(args) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] - -for job in parseArguments(args): - - result = dirac.getJobJDL(job, original=original, printOutput=True) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-output-data.py b/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-output-data.py deleted file mode 100755 index f1b2ac6c1b7..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-output-data.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-job-get-output-data -# Author : Stuart Paterson -######################################################################## -""" - Retrieve the output data files of a DIRAC job -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.registerSwitch("D:", "Dir=", "Store the output in this directory") -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] - -outputDir = '' -for sw, v in Script.getUnprocessedSwitches(): - if sw in ('D', 'Dir'): - outputDir = v - -for job in parseArguments(args): - - result = dirac.getJobOutputData(job, destinationDir=outputDir) - if result['OK']: - print('Job %s output data retrieved' % (job)) - else: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-output.py b/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-output.py deleted file mode 100755 index bb92070909f..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-get-output.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-job-get-output -# Author : Stuart Paterson -######################################################################## -""" - Retrieve output sandbox for a DIRAC job -""" - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import os -import shutil - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID or a name of the file with JobID per line'])) - -Script.registerSwitch("D:", "Dir=", "Store the output in this directory") -Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file") -Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group") - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -from DIRAC.Core.Utilities.Time import toString, date, day -from DIRAC.Core.Utilities.File import mkDir - -dirac = Dirac() -exitCode = 0 -errorList = [] - -outputDir = None -group = None -jobs = [] -for sw, value in Script.getUnprocessedSwitches(): - if sw in ('D', 'Dir'): - outputDir = value - elif sw.lower() in ('f', 'file'): - if os.path.exists(value): - jFile = open(value) - jobs += jFile.read().split() - jFile.close() - elif sw.lower() in ('g', 'jobgroup'): - group = value - jobDate = toString(date() - 30 * day) - - # Choose jobs in final state, no more than 30 days old - result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Done') - if not result['OK']: - if "No jobs selected" not in result['Message']: - print("Error:", result['Message']) - DIRAC.exit(-1) - else: - jobs += result['Value'] - result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Failed') - if not result['OK']: - if "No jobs selected" not in result['Message']: - print("Error:", result['Message']) - DIRAC.exit(-1) - else: - jobs += result['Value'] - -for arg in parseArguments(args): - if os.path.isdir(arg): - print("Output for job %s already retrieved, remove the output directory to redownload" % arg) - else: - jobs.append(arg) - -if not jobs: - print("No jobs selected") - DIRAC.exit(0) - -if group: - if outputDir: - outputDir = os.path.join(outputDir, group) - else: - outputDir = group - -if outputDir: - mkDir(outputDir) -else: - outputDir = os.getcwd() - -jobs = [str(job) for job in jobs] -doneJobs = os.listdir(outputDir) -todoJobs = [job for job in jobs if job not in doneJobs] - -for job in todoJobs: - - result = dirac.getOutputSandbox(job, outputDir=outputDir) - - jobDir = str(job) - if outputDir: - jobDir = os.path.join(outputDir, job) - if result['OK']: - if os.path.exists(jobDir): - print('Job output sandbox retrieved in %s/' % (jobDir)) - else: - if os.path.exists('%s' % jobDir): - shutil.rmtree(jobDir) - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-kill.py b/src/DIRAC/Interfaces/scripts/dirac-wms-job-kill.py deleted file mode 100755 index 9180d137e3c..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-kill.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-job-kil -# Author : Stuart Paterson -######################################################################## -""" - Issue a kill signal to a running DIRAC job -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments - -result = Dirac().killJob(parseArguments(args)) -if result['OK']: - print('Killed jobs %s' % ','.join([str(j) for j in result['Value']])) - exitCode = 0 -else: - print('ERROR', result['Message']) - exitCode = 2 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-logging-info.py b/src/DIRAC/Interfaces/scripts/dirac-wms-job-logging-info.py deleted file mode 100755 index 8bfb170ae09..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-logging-info.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-job-logging-info -# Author : Stuart Paterson -######################################################################## -""" - Retrieve history of transitions for a DIRAC job -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] - -for job in parseArguments(args): - - result = dirac.getJobLoggingInfo(job, printOutput=True) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-parameters.py b/src/DIRAC/Interfaces/scripts/dirac-wms-job-parameters.py deleted file mode 100755 index 95a43e6f54a..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-parameters.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-job-parameters -# Author : Stuart Paterson -######################################################################## -""" - Retrieve parameters associated to the given DIRAC job -""" - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC - -from DIRAC import gLogger -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] - -for job in parseArguments(args): - - result = dirac.getJobParameters(job, printOutput=True) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-peek.py b/src/DIRAC/Interfaces/scripts/dirac-wms-job-peek.py deleted file mode 100755 index a80c5e86136..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-peek.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-job-delete -# Author : Stuart Paterson -######################################################################## -""" - Peek StdOut of the given DIRAC job -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] - -for job in parseArguments(args): - - result = dirac.peekJob(job, printOutput=True) - if not result['OK']: - errorList.append((job, result['Message'])) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-reschedule.py b/src/DIRAC/Interfaces/scripts/dirac-wms-job-reschedule.py deleted file mode 100755 index 1dd2731e8cf..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-reschedule.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-job-delete -# Author : Stuart Paterson -######################################################################## -""" - Reschedule the given DIRAC job -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 -errorList = [] - -result = dirac.rescheduleJob(parseArguments(args)) -if result['OK']: - print('Rescheduled job %s' % ','.join([str(j) for j in result['Value']])) -else: - errorList.append((j, result['Message'])) - print(result['Message']) - exitCode = 2 - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-status.py b/src/DIRAC/Interfaces/scripts/dirac-wms-job-status.py deleted file mode 100755 index 04290da1397..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-status.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-job-status -# Author : Stuart Paterson -######################################################################## -""" - Retrieve status of the given DIRAC job -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import os -import DIRAC -from DIRAC import exit as DIRACExit -from DIRAC.Core.Base import Script -from DIRAC.Core.Utilities.Time import toString, date, day - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - '\nUsage:\n', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - '\nArguments:\n', - ' JobID: DIRAC Job ID'])) - -Script.registerSwitch("f:", "File=", "Get status for jobs with IDs from the file") -Script.registerSwitch("g:", "JobGroup=", "Get status for jobs in the given group") - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments -dirac = Dirac() -exitCode = 0 - -jobs = [] -for key, value in Script.getUnprocessedSwitches(): - if key.lower() in ('f', 'file'): - if os.path.exists(value): - jFile = open(value) - jobs += jFile.read().split() - jFile.close() - elif key.lower() in ('g', 'jobgroup'): - jobDate = toString(date() - 30 * day) - # Choose jobs no more than 30 days old - result = dirac.selectJobs(jobGroup=value, date=jobDate) - if not result['OK']: - print("Error:", result['Message']) - DIRACExit(-1) - jobs += result['Value'] - -if len(args) < 1 and not jobs: - Script.showHelp(exitCode=1) - -if len(args) > 0: - jobs += parseArguments(args) - -result = dirac.getJobStatus(jobs) -if result['OK']: - for job in result['Value']: - print('JobID=' + str(job), end=' ') - for status in result['Value'][job].items(): - print('%s=%s;' % status, end=' ') - print() -else: - exitCode = 2 - print("ERROR: %s" % result['Message']) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-submit.py b/src/DIRAC/Interfaces/scripts/dirac-wms-job-submit.py deleted file mode 100755 index cd0d8a81b6b..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-submit.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-job-submit -# Author : Stuart Paterson -######################################################################## -""" - Submit jobs to DIRAC WMS -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -import os - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - - - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JDL ...' % Script.scriptName, - 'Arguments:', - ' JDL: Path to JDL file'])) - -Script.registerSwitch("f:", "File=", "Writes job ids to file ") -Script.registerSwitch("r:", "UseJobRepo=", "Use the job repository") -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC.Interfaces.API.Dirac import Dirac -unprocessed_switches = Script.getUnprocessedSwitches() -use_repo = False -repo_name = "" -for sw, value in unprocessed_switches: - if sw.lower() in ["r", "usejobrepo"]: - use_repo = True - repo_name = value - repo_name = repo_name.replace(".cfg", ".repo") -dirac = Dirac(use_repo, repo_name) -exitCode = 0 -errorList = [] - -jFile = None -for sw, value in unprocessed_switches: - if sw.lower() in ('f', 'file'): - if os.path.isfile(value): - print('Appending job ids to existing logfile: %s' % value) - if not os.access(value, os.W_OK): - print('Existing logfile %s must be writable by user.' % value) - jFile = open(value, 'a') - -for jdl in args: - - result = dirac.submitJob(jdl) - if result['OK']: - print('JobID = %s' % (result['Value'])) - if jFile is not None: - # parametric jobs - if isinstance(result['Value'], list): - jFile.write('\n'.join(str(p) for p in result['Value'])) - jFile.write('\n') - else: - jFile.write(str(result['Value']) + '\n') - else: - errorList.append((jdl, result['Message'])) - exitCode = 2 - -if jFile is not None: - jFile.close() - -for error in errorList: - print("ERROR %s: %s" % error) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-jobs-select-output-search.py b/src/DIRAC/Interfaces/scripts/dirac-wms-jobs-select-output-search.py deleted file mode 100755 index 9b6daa53357..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-jobs-select-output-search.py +++ /dev/null @@ -1,113 +0,0 @@ -#! /usr/bin/env python -######################################################################## -# File : dirac-wms-jobs-select-output-search -# Author : Vladimir Romanovsky -######################################################################## -""" - Retrieve output sandbox for DIRAC Jobs for the given selection and search for a string in their std.out -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import os -from shutil import rmtree - -import DIRAC -from DIRAC.Core.Base import Script - -Script.registerSwitch("", "Status=", "Primary status") -Script.registerSwitch("", "MinorStatus=", "Secondary status") -Script.registerSwitch("", "ApplicationStatus=", "Application status") -Script.registerSwitch("", "Site=", "Execution site") -Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)") -Script.registerSwitch("", "JobGroup=", "Select jobs for specified job group") -Script.registerSwitch("", "Date=", "Date in YYYY-MM-DD format, if not specified default is today") -Script.registerSwitch("", "File=", "File name,if not specified default is std.out ") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... String ...' % Script.scriptName, - 'Arguments:', - ' String: string to search for'])) - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -# Default values -status = None -minorStatus = None -appStatus = None -site = None -owner = None -jobGroup = None -date = None -filename = 'std.out' - -if len(args) != 1: - Script.showHelp() - -searchstring = str(args[0]) - -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "status": - status = switch[1] - elif switch[0].lower() == "minorstatus": - minorStatus = switch[1] - elif switch[0].lower() == "applicationstatus": - appStatus = switch[1] - elif switch[0].lower() == "site": - site = switch[1] - elif switch[0].lower() == "owner": - owner = switch[1] - elif switch[0].lower() == "jobgroup": - jobGroup = switch[1] - elif switch[0].lower() == "date": - date = switch[1] - elif switch[0].lower() == "file": - filename = switch[1] - -selDate = date -if not date: - selDate = 'Today' - -from DIRAC.Interfaces.API.Dirac import Dirac - -dirac = Dirac() -exitCode = 0 -errorList = [] -resultDict = {} - -result = dirac.selectJobs(status=status, minorStatus=minorStatus, applicationStatus=appStatus, - site=site, owner=owner, jobGroup=jobGroup, date=date) -if result['OK']: - jobs = result['Value'] -else: - print("Error in selectJob", result['Message']) - DIRAC.exit(2) - -for job in jobs: - - result = dirac.getOutputSandbox(job) - if result['OK']: - if os.path.exists('%s' % job): - - lines = [] - try: - lines = open(os.path.join(job, filename)).readlines() - except Exception as x: - errorList.append((job, x)) - for line in lines: - if line.count(searchstring): - resultDict[job] = line - rmtree("%s" % (job)) - else: - errorList.append((job, result['Message'])) - exitCode = 2 - -for result in resultDict.items(): - print(result) - - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-select-jobs.py b/src/DIRAC/Interfaces/scripts/dirac-wms-select-jobs.py deleted file mode 100755 index 145ea6c7cd5..00000000000 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-select-jobs.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-select-jobs -# Author : Stuart Paterson -######################################################################## -""" - Select DIRAC jobs matching the given conditions -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC import gLogger -from DIRAC.Core.Base import Script - -maxJobs = 100 -Script.registerSwitch("", "Status=", "Primary status") -Script.registerSwitch("", "MinorStatus=", "Secondary status") -Script.registerSwitch("", "ApplicationStatus=", "Application status") -Script.registerSwitch("", "Site=", "Execution site") -Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)") -Script.registerSwitch("", "JobGroup=", "Select jobs for specified job group") -Script.registerSwitch("", "Date=", "Date in YYYY-MM-DD format, if not specified default is today") -Script.registerSwitch("", "Maximum=", "Maximum number of jobs shown (default %d, 0 means all)" % maxJobs) -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName])) -Script.parseCommandLine(ignoreErrors=True) - -args = Script.getPositionalArgs() - -# Default values -status = None -minorStatus = None -appStatus = None -site = None -owner = None -jobGroups = [] -date = None - -if args: - Script.showHelp() - -exitCode = 0 - -for switch in Script.getUnprocessedSwitches(): - if switch[0].lower() == "status": - status = switch[1] - elif switch[0].lower() == "minorstatus": - minorStatus = switch[1] - elif switch[0].lower() == "applicationstatus": - appStatus = switch[1] - elif switch[0].lower() == "site": - site = switch[1] - elif switch[0].lower() == "owner": - owner = switch[1] - elif switch[0].lower() == "jobgroup": - for jg in switch[1].split(','): - if jg.isdigit(): - jobGroups.append('%08d' % int(jg)) - else: - jobGroups.append(jg) - elif switch[0].lower() == "date": - date = switch[1] - elif switch[0] == 'Maximum': - try: - maxJobs = int(switch[1]) - except TypeError: - gLogger.fatal("Invalid max number of jobs", switch[1]) - DIRAC.exit(1) - -selDate = date -if not date: - selDate = 'Today' -conditions = {'Status': status, - 'MinorStatus': minorStatus, - 'ApplicationStatus': appStatus, - 'Owner': owner, - 'JobGroup': ','.join(str(jg) for jg in jobGroups), - 'Date': selDate} - - -from DIRAC.Interfaces.API.Dirac import Dirac - -dirac = Dirac() -jobs = [] - -if jobGroups: - for jobGroup in jobGroups: - res = dirac.selectJobs(status=status, - minorStatus=minorStatus, - applicationStatus=appStatus, - site=site, - owner=owner, - jobGroup=jobGroup, - date=date, - printErrors=False) - if res['OK']: - jobs.extend(res['Value']) - else: - gLogger.error("Can't select jobs: ", res['Message']) -else: - res = dirac.selectJobs(status=status, - minorStatus=minorStatus, - applicationStatus=appStatus, - site=site, - owner=owner, - date=date, - printErrors=False) - if res['OK']: - jobs.extend(res['Value']) - else: - gLogger.error("Can't select jobs: ", res['Message']) - - -conds = ['%s = %s' % (n, v) for n, v in conditions.items() if v] -if maxJobs and len(jobs) > maxJobs: - jobs = jobs[:maxJobs] - constrained = ' (first %d shown) ' % maxJobs -else: - constrained = ' ' - -if jobs: - gLogger.notice('==> Selected %s jobs%swith conditions: %s\n%s' % (len(jobs), - constrained, - ', '.join(conds), - ','.join(jobs))) -else: - gLogger.notice('No jobs were selected with conditions:', ', '.join(conds)) - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_add_group.py b/src/DIRAC/Interfaces/scripts/dirac_admin_add_group.py new file mode 100755 index 00000000000..7fde384cb34 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_add_group.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +# pylint: disable=wrong-import-position + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + +groupName = None +groupProperties = [] +userNames = [] + + +def setGroupName(arg): + global groupName + if groupName or not arg: + Script.showHelp(exitCode=1) + groupName = arg + + +def addUserName(arg): + global userNames + if not arg: + Script.showHelp(exitCode=1) + if arg not in userNames: + userNames.append(arg) + + +def addProperty(arg): + global groupProperties + if not arg: + Script.showHelp(exitCode=1) + if arg not in groupProperties: + groupProperties.append(arg) + + +@DIRACScript() +def main(): + global groupName + global groupProperties + global userNames + Script.setUsageMessage('\n'.join(['Add or Modify a Group info in DIRAC', + '\nUsage:\n', + ' %s [option|cfgfile] ... Property= ...' % Script.scriptName, + '\nArguments:\n', + ' Property=: Other properties to be added to the User like (VOMSRole=XXXX)', + ])) + + Script.registerSwitch('G:', 'GroupName:', 'Name of the Group (Mandatory)', setGroupName) + Script.registerSwitch( + 'U:', + 'UserName:', + 'Short Name of user to be added to the Group (Allow Multiple instances or None)', + addUserName) + Script.registerSwitch( + 'P:', + 'Property:', + 'Property to be added to the Group (Allow Multiple instances or None)', + addProperty) + + Script.parseCommandLine(ignoreErrors=True) + + if groupName is None: + Script.showHelp(exitCode=1) + + args = Script.getPositionalArgs() + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + groupProps = {} + if userNames: + groupProps['Users'] = ', '.join(userNames) + if groupProperties: + groupProps['Properties'] = ', '.join(groupProperties) + + for prop in args: + pl = prop.split("=") + if len(pl) < 2: + errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) + exitCode = 255 + else: + pName = pl[0] + pValue = "=".join(pl[1:]) + Script.gLogger.info("Setting property %s to %s" % (pName, pValue)) + groupProps[pName] = pValue + + if not diracAdmin.csModifyGroup(groupName, groupProps, createIfNonExistant=True)['OK']: + errorList.append(("add group", "Cannot register group %s" % groupName)) + exitCode = 255 + else: + result = diracAdmin.csCommitChanges() + if not result['OK']: + errorList.append(("commit", result['Message'])) + exitCode = 255 + + for error in errorList: + Script.gLogger.error("%s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_add_host.py b/src/DIRAC/Interfaces/scripts/dirac_admin_add_host.py new file mode 100755 index 00000000000..8b989ab1a0d --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_add_host.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + +hostName = None +hostDN = None +hostProperties = [] + + +def setHostName(arg): + global hostName + if hostName or not arg: + Script.showHelp(exitCode=1) + hostName = arg + + +def setHostDN(arg): + global hostDN + if hostDN or not arg: + Script.showHelp(exitCode=1) + hostDN = arg + + +def addProperty(arg): + global hostProperties + if not arg: + Script.showHelp(exitCode=1) + if arg not in hostProperties: + hostProperties.append(arg) + + +@DIRACScript() +def main(): + global hostName + global hostDN + global hostProperties + Script.setUsageMessage('\n'.join([ + 'Add or Modify a Host info in DIRAC', + '\nUsage:\n', + ' %s [option|cfgfile] ... Property= ...' % Script.scriptName, + '\nArguments:\n', + ' Property=: Other properties to be added to the User like (Responsible=XXX)', + ])) + + Script.registerSwitch('H:', 'HostName:', 'Name of the Host (Mandatory)', setHostName) + Script.registerSwitch('D:', 'HostDN:', 'DN of the Host Certificate (Mandatory)', setHostDN) + Script.registerSwitch( + 'P:', + 'Property:', + 'Property to be added to the Host (Allow Multiple instances or None)', + addProperty) + + Script.parseCommandLine(ignoreErrors=True) + + if hostName is None or hostDN is None: + Script.showHelp(exitCode=1) + + args = Script.getPositionalArgs() + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + hostProps = {'DN': hostDN} + if hostProperties: + hostProps['Properties'] = ', '.join(hostProperties) + + for prop in args: + pl = prop.split("=") + if len(pl) < 2: + errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) + exitCode = 255 + else: + pName = pl[0] + pValue = "=".join(pl[1:]) + Script.gLogger.info("Setting property %s to %s" % (pName, pValue)) + hostProps[pName] = pValue + + if not diracAdmin.csModifyHost(hostName, hostProps, createIfNonExistant=True)['OK']: + errorList.append(("add host", "Cannot register host %s" % hostName)) + exitCode = 255 + else: + result = diracAdmin.csCommitChanges() + if not result['OK']: + errorList.append(("commit", result['Message'])) + exitCode = 255 + + if exitCode == 0: + from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient + cmc = ComponentMonitoringClient() + ret = cmc.hostExists(dict(HostName=hostName)) + if not ret['OK']: + Script.gLogger.error('Cannot check if host is registered in ComponentMonitoring', ret['Message']) + elif ret['Value']: + Script.gLogger.info('Host already registered in ComponentMonitoring') + else: + ret = cmc.addHost(dict(HostName=hostName, CPU='TO_COME')) + if not ret['OK']: + Script.gLogger.error('Failed to add Host to ComponentMonitoring', ret['Message']) + + for error in errorList: + Script.gLogger.error("%s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_add_user.py b/src/DIRAC/Interfaces/scripts/dirac_admin_add_user.py new file mode 100755 index 00000000000..8c0ef418a22 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_add_user.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + +userName = None +userDN = None +userMail = None +userGroups = [] + + +def setUserName(arg): + global userName + if userName or not arg: + Script.showHelp(exitCode=1) + userName = arg + + +def setUserDN(arg): + global userDN + if userDN or not arg: + Script.showHelp(exitCode=1) + userDN = arg + + +def setUserMail(arg): + global userMail + if userMail or not arg: + Script.showHelp(exitCode=1) + if not arg.find('@') > 0: + Script.gLogger.error('Not a valid mail address', arg) + DIRAC.exit(-1) + userMail = arg + + +def addUserGroup(arg): + global userGroups + if not arg: + Script.showHelp(exitCode=1) + if arg not in userGroups: + userGroups.append(arg) + + +@DIRACScript() +def main(): + global userName + global userDN + global userMail + global userGroups + Script.setUsageMessage('\n'.join(['Add or Modify a User info in DIRAC', + 'Usage:', + ' %s [option|cfgfile] ... Property= ...' % Script.scriptName, + 'Arguments:', + ' Property=: Properties to be added to the User like (Phone=XXXX)', ])) + + Script.registerSwitch('N:', 'UserName:', 'Short Name of the User (Mandatory)', setUserName) + Script.registerSwitch('D:', 'UserDN:', 'DN of the User Certificate (Mandatory)', setUserDN) + Script.registerSwitch('M:', 'UserMail:', 'eMail of the user (Mandatory)', setUserMail) + Script.registerSwitch( + 'G:', + 'UserGroup:', + 'Name of the Group for the User (Allow Multiple instances or None)', + addUserGroup) + + Script.parseCommandLine(ignoreErrors=True) + + if userName is None or userDN is None or userMail is None: + Script.showHelp(exitCode=1) + + args = Script.getPositionalArgs() + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + userProps = {'DN': userDN, 'Email': userMail} + if userGroups: + userProps['Groups'] = userGroups + for prop in args: + pl = prop.split("=") + if len(pl) < 2: + errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) + exitCode = 255 + else: + pName = pl[0] + pValue = "=".join(pl[1:]) + Script.gLogger.info("Setting property %s to %s" % (pName, pValue)) + userProps[pName] = pValue + + if not diracAdmin.csModifyUser(userName, userProps, createIfNonExistant=True)['OK']: + errorList.append(("add user", "Cannot register user %s" % userName)) + exitCode = 255 + else: + result = diracAdmin.csCommitChanges() + if not result['OK']: + errorList.append(("commit", result['Message'])) + exitCode = 255 + + for error in errorList: + Script.gLogger.error("%s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_allow_site.py b/src/DIRAC/Interfaces/scripts/dirac_admin_allow_site.py new file mode 100755 index 00000000000..1be789f6094 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_allow_site.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-allow-site +# Author : Stuart Paterson +######################################################################## +""" + Add Site to Active mask for current Setup +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import time + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch("E:", "email=", "Boolean True/False (True by default)") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Site Comment' % Script.scriptName, + 'Arguments:', + ' Site: Name of the Site', + ' Comment: Reason of the action'])) + Script.parseCommandLine(ignoreErrors=True) + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + from DIRAC import exit as DIRACExit, gConfig, gLogger + + def getBoolean(value): + if value.lower() == 'true': + return True + elif value.lower() == 'false': + return False + else: + Script.showHelp() + + email = True + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "email": + email = getBoolean(switch[1]) + + args = Script.getPositionalArgs() + + if len(args) < 2: + Script.showHelp() + + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + setup = gConfig.getValue('/DIRAC/Setup', '') + if not setup: + print('ERROR: Could not contact Configuration Service') + exitCode = 2 + DIRACExit(exitCode) + + # result = promptUser( + # 'All the elements that are associated with this site will be active, ' + # 'are you sure about this action?' + # ) + # if not result['OK'] or result['Value'] is 'n': + # print 'Script stopped' + # DIRACExit( 0 ) + + site = args[0] + comment = args[1] + result = diracAdmin.allowSite(site, comment, printOutput=True) + if not result['OK']: + errorList.append((site, result['Message'])) + exitCode = 2 + else: + if email: + userName = diracAdmin._getCurrentUser() + if not userName['OK']: + print('ERROR: Could not obtain current username from proxy') + exitCode = 2 + DIRACExit(exitCode) + userName = userName['Value'] + subject = '%s is added in site mask for %s setup' % (site, setup) + body = 'Site %s is added to the site mask for %s setup by %s on %s.\n\n' % (site, setup, userName, time.asctime()) + body += 'Comment:\n%s' % comment + addressPath = 'EMail/Production' + address = Operations().getValue(addressPath, '') + if not address: + gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) + else: + result = diracAdmin.sendMail(address, subject, body) + else: + print('Automatic email disabled by flag.') + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_ban_site.py b/src/DIRAC/Interfaces/scripts/dirac_admin_ban_site.py new file mode 100755 index 00000000000..55217e3a78d --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_ban_site.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-ban-site +# Author : Stuart Paterson +######################################################################## +""" + Remove Site from Active mask for current Setup +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import time + +from DIRAC import exit as DIRACExit, gConfig, gLogger +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch("E:", "email=", "Boolean True/False (True by default)") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Site Comment' % Script.scriptName, + 'Arguments:', + ' Site: Name of the Site', + ' Comment: Reason of the action'])) + Script.parseCommandLine(ignoreErrors=True) + + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + from DIRAC.Core.Utilities.PromptUser import promptUser + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + + def getBoolean(value): + if value.lower() == 'true': + return True + elif value.lower() == 'false': + return False + else: + Script.showHelp() + + email = True + for switch in Script.getUnprocessedSwitches(): + if switch[0] == "email": + email = getBoolean(switch[1]) + + args = Script.getPositionalArgs() + + if len(args) < 2: + Script.showHelp() + + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + setup = gConfig.getValue('/DIRAC/Setup', '') + if not setup: + print('ERROR: Could not contact Configuration Service') + exitCode = 2 + DIRACExit(exitCode) + + # result = promptUser( + # 'All the elements that are associated with this site will be banned,' + # 'are you sure about this action?' + # ) + # if not result['OK'] or result['Value'] is 'n': + # print 'Script stopped' + # DIRACExit( 0 ) + + site = args[0] + comment = args[1] + result = diracAdmin.banSite(site, comment, printOutput=True) + if not result['OK']: + errorList.append((site, result['Message'])) + exitCode = 2 + else: + if email: + userName = diracAdmin._getCurrentUser() + if not userName['OK']: + print('ERROR: Could not obtain current username from proxy') + exitCode = 2 + DIRACExit(exitCode) + userName = userName['Value'] + subject = '%s is banned for %s setup' % (site, setup) + body = 'Site %s is removed from site mask for %s setup by %s on %s.\n\n' % (site, setup, userName, time.asctime()) + body += 'Comment:\n%s' % comment + + addressPath = 'EMail/Production' + address = Operations().getValue(addressPath, '') + if not address: + gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) + else: + result = diracAdmin.sendMail(address, subject, body) + else: + print('Automatic email disabled by flag.') + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_ce_info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_ce_info.py new file mode 100755 index 00000000000..3b094358a09 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_ce_info.py @@ -0,0 +1,49 @@ +#! /usr/bin/env python +######################################################################## +# File : dirac-admin-ce-info +# Author : Vladimir Romanovsky +######################################################################## +""" + Retrieve Site Associated to a given CE +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +from DIRAC import gConfig, gLogger, exit as Dexit +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping +from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... CE ...' % Script.scriptName, + 'Arguments:', + ' CE: Name of the CE (mandatory)'])) + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + + res = getCESiteMapping(args[0]) + if not res['OK']: + gLogger.error(res['Message']) + Dexit(1) + site = res['Value'][args[0]] + + res = gConfig.getOptionsDict(cfgPath('Resources', 'Sites', site.split('.')[0], site, 'CEs', args[0])) + if not res['OK']: + gLogger.error(res['Message']) + Dexit(1) + gLogger.notice(res['Value']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_delete_user.py b/src/DIRAC/Interfaces/scripts/dirac_admin_delete_user.py new file mode 100755 index 00000000000..4818d9a1a47 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_delete_user.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +######################################################################## +# $HeadURL$ +# File : dirac-admin-delete-user +# Author : Adrian Casajus +######################################################################## +""" + Remove User from Configuration +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... User ...' % Script.scriptName, + 'Arguments:', + ' User: User name'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + from DIRAC import exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + if len(args) < 1: + Script.showHelp() + + choice = raw_input("Are you sure you want to delete user/s %s? yes/no [no]: " % ", ".join(args)) + choice = choice.lower() + if choice not in ("yes", "y"): + print("Delete aborted") + DIRACExit(0) + + for user in args: + if not diracAdmin.csDeleteUser(user): + errorList.append(("delete user", "Cannot delete user %s" % user)) + exitCode = 255 + + if not exitCode: + result = diracAdmin.csCommitChanges() + if not result['OK']: + errorList.append(("commit", result['Message'])) + exitCode = 255 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_banned_sites.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_banned_sites.py new file mode 100755 index 00000000000..d3b5ab5e1f1 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_banned_sites.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-get-banned-sites +# Author : Stuart Paterson +######################################################################## + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.parseCommandLine(ignoreErrors=True) + + from DIRAC import gLogger, exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + + diracAdmin = DiracAdmin() + + result = diracAdmin.getBannedSites() + if result['OK']: + bannedSites = result['Value'] + else: + gLogger.error(result['Message']) + DIRACExit(2) + + for site in bannedSites: + result = diracAdmin.getSiteMaskLogging(site, printOutput=True) + if not result['OK']: + gLogger.error(result['Message']) + DIRACExit(2) + + DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py new file mode 100755 index 00000000000..6c0883599cf --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-get-job-pilot-output +# Author : Stuart Paterson +######################################################################## +""" + Retrieve the output of the pilot that executed a given job +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC ID of the Job'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC import exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + for job in args: + + try: + job = int(job) + except Exception as x: + errorList.append(('Expected integer for JobID', job)) + exitCode = 2 + continue + + result = diracAdmin.getJobPilotOutput(job) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py new file mode 100755 index 00000000000..1148123c886 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-get-job-pilots +# Author : Stuart Paterson +######################################################################## +""" + Retrieve info about pilots that have matched a given Job +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +# pylint: disable=wrong-import-position + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC ID of the Job'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC import exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + for job in args: + + try: + job = int(job) + except BaseException as x: + errorList.append((job, 'Expected integer for jobID')) + exitCode = 2 + continue + + result = diracAdmin.getJobPilots(job) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py new file mode 100755 index 00000000000..b757719da57 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-get-pilot-output +# Author : Ricardo Graciani +######################################################################## +""" + Retrieve available info about the given pilot +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +# pylint: disable=wrong-import-position + +from DIRAC import exit as DIRACExit +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + +extendedPrint = False + + +def setExtendedPrint(_arg): + global extendedPrint + extendedPrint = True + + +@DIRACScript() +def main(): + global extendedPrint + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, + 'Arguments:', + ' PilotID: Grid ID of the pilot'])) + Script.registerSwitch('e', 'extended', 'Get extended printout', setExtendedPrint) + Script.parseCommandLine(ignoreErrors=True) + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + from DIRAC.Interfaces.API.Dirac import Dirac + + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + diracAdmin = DiracAdmin() + dirac = Dirac() + exitCode = 0 + errorList = [] + + for gridID in args: + result = diracAdmin.getPilotInfo(gridID) + if not result['OK']: + errorList.append((gridID, result['Message'])) + exitCode = 2 + else: + res = result['Value'][gridID] + if extendedPrint: + tab = '' + for key in [ + 'PilotJobReference', + 'Status', + 'OwnerDN', + 'OwnerGroup', + 'SubmissionTime', + 'DestinationSite', + 'GridSite', + ]: + if key in res: + diracAdmin.log.notice('%s%s: %s' % (tab, key, res[key])) + if not tab: + tab = ' ' + diracAdmin.log.notice('') + for jobID in res['Jobs']: + tab = ' ' + result = dirac.getJobAttributes(int(jobID)) + if not result['OK']: + errorList.append((gridID, result['Message'])) + exitCode = 2 + else: + job = result['Value'] + diracAdmin.log.notice('%sJob ID: %s' % (tab, jobID)) + tab += ' ' + for key in ['OwnerDN', 'OwnerGroup', 'JobName', 'Status', 'StartExecTime', 'LastUpdateTime', 'EndExecTime']: + if key in job: + diracAdmin.log.notice('%s%s:' % (tab, key), job[key]) + diracAdmin.log.notice('') + else: + print(diracAdmin.pPrint.pformat({gridID: res})) + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py new file mode 100755 index 00000000000..8fe5ae43da8 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-get-pilot-logging-info.py +# Author : Stuart Paterson +######################################################################## +""" + Retrieve logging info of a Grid pilot +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +# pylint: disable=wrong-import-position + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, + 'Arguments:', + ' PilotID: Grid ID of the pilot'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC import exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + for gridID in args: + + result = diracAdmin.getPilotLoggingInfo(gridID) + if not result['OK']: + errorList.append((gridID, result['Message'])) + exitCode = 2 + else: + print('Pilot Reference: %s', gridID) + print(result['Value']) + print() + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py new file mode 100755 index 00000000000..13b8009aca0 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +######################################################################## +# $HeadURL$ +# File : dirac-admin-get-pilot-output +# Author : Stuart Paterson +######################################################################## +""" + Retrieve output of a Grid pilot +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, + 'Arguments:', + ' PilotID: Grid ID of the pilot'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC import exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + for gridID in args: + + result = diracAdmin.getPilotOutput(gridID) + if not result['OK']: + errorList.append((gridID, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py new file mode 100755 index 00000000000..4713e4e053c --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_site_mask.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-get-site-mask +# Author : Stuart Paterson +######################################################################## + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" + Get the list of sites enabled in the mask for job submission + + Usage: + %s [options] + """ % Script.scriptName) + + Script.parseCommandLine(ignoreErrors=True) + + from DIRAC import exit as DIRACExit, gLogger + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + + diracAdmin = DiracAdmin() + + gLogger.setLevel('ALWAYS') + + result = diracAdmin.getSiteMask(printOutput=True, status="Active") + if result['OK']: + DIRACExit(0) + else: + print(result['Message']) + DIRACExit(2) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_list_hosts.py b/src/DIRAC/Interfaces/scripts/dirac_admin_list_hosts.py new file mode 100755 index 00000000000..cb34b525f48 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_list_hosts.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-list-users +# Author : Adrian Casajus +######################################################################## +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch("e", "extended", "Show extended info") + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + from DIRAC import exit as DIRACExit + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + extendedInfo = False + + for unprocSw in Script.getUnprocessedSwitches(): + if unprocSw[0] in ('e', 'extended'): + extendedInfo = True + + if not extendedInfo: + result = diracAdmin.csListHosts() + for host in result['Value']: + print(" %s" % host) + else: + result = diracAdmin.csDescribeHosts() + print(diracAdmin.pPrint.pformat(result['Value'])) + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRACExit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_list_users.py b/src/DIRAC/Interfaces/scripts/dirac_admin_list_users.py new file mode 100755 index 00000000000..ff01b65eb04 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_list_users.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-list-users +# Author : Adrian Casajus +######################################################################## +""" + Lists the users in the Configuration. If no group is specified return all users. +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch("e", "extended", "Show extended info") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... [Group] ...' % Script.scriptName, + 'Arguments:', + ' Group: Only users from this group (default: all)'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) == 0: + args = ['all'] + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + extendedInfo = False + + for unprocSw in Script.getUnprocessedSwitches(): + if unprocSw[0] in ('e', 'extended'): + extendedInfo = True + + def printUsersInGroup(group=False): + result = diracAdmin.csListUsers(group) + if result['OK']: + if group: + print("Users in group %s:" % group) + else: + print("All users registered:") + for username in result['Value']: + print(" %s" % username) + + def describeUsersInGroup(group=False): + result = diracAdmin.csListUsers(group) + if result['OK']: + if group: + print("Users in group %s:" % group) + else: + print("All users registered:") + result = diracAdmin.csDescribeUsers(result['Value']) + print(diracAdmin.pPrint.pformat(result['Value'])) + + for group in args: + if 'all' in args: + group = False + if not extendedInfo: + printUsersInGroup(group) + else: + describeUsersInGroup(group) + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_modify_user.py b/src/DIRAC/Interfaces/scripts/dirac_admin_modify_user.py new file mode 100755 index 00000000000..259eaf6b253 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_modify_user.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-modify-user +# Author : Adrian Casajus +######################################################################## +""" + Modify a user in the CS. +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch("p:", "property=", "Add property to the user =") + Script.registerSwitch("f", "force", "create the user if it doesn't exist") + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... user DN group [group] ...' % Script.scriptName, + 'Arguments:', + ' user: User name (mandatory)', + ' DN: DN of the User (mandatory)', + ' group: Add the user to the group (mandatory)'])) + Script.parseCommandLine(ignoreErrors=True) + + args = Script.getPositionalArgs() + + if len(args) < 3: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + forceCreation = False + errorList = [] + + userProps = {} + for unprocSw in Script.getUnprocessedSwitches(): + if unprocSw[0] in ("f", "force"): + forceCreation = True + elif unprocSw[0] in ("p", "property"): + prop = unprocSw[1] + pl = prop.split("=") + if len(pl) < 2: + errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop)) + exitCode = 255 + else: + pName = pl[0] + pValue = "=".join(pl[1:]) + print("Setting property %s to %s" % (pName, pValue)) + userProps[pName] = pValue + + userName = args[0] + userProps['DN'] = args[1] + userProps['Groups'] = args[2:] + + if not diracAdmin.csModifyUser(userName, userProps, createIfNonExistant=forceCreation): + errorList.append(("modify user", "Cannot modify user %s" % userName)) + exitCode = 255 + else: + result = diracAdmin.csCommitChanges() + if not result['OK']: + errorList.append(("commit", result['Message'])) + exitCode = 255 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac-admin-pilot-summary.py b/src/DIRAC/Interfaces/scripts/dirac_admin_pilot_summary.py similarity index 51% rename from src/DIRAC/Interfaces/scripts/dirac-admin-pilot-summary.py rename to src/DIRAC/Interfaces/scripts/dirac_admin_pilot_summary.py index 3d98f135156..cb82d5540f2 100755 --- a/src/DIRAC/Interfaces/scripts/dirac-admin-pilot-summary.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_pilot_summary.py @@ -13,16 +13,24 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() +@DIRACScript() +def main(): + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -result = diracAdmin.getPilotSummary() -if result['OK']: - DIRAC.exit(0) -else: - print(result['Message']) - DIRAC.exit(2) + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + + result = diracAdmin.getPilotSummary() + if result['OK']: + DIRAC.exit(0) + else: + print(result['Message']) + DIRAC.exit(2) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py b/src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py new file mode 100755 index 00000000000..affbb31801f --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-reset-job +# Author : Stuart Paterson +######################################################################## +""" + Reset a job or list of jobs in the WMS +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC ID of the Job'])) + Script.parseCommandLine(ignoreErrors=True) + + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + for job in args: + + try: + job = int(job) + except Exception as x: + errorList.append(('Expected integer for jobID', job)) + exitCode = 2 + continue + + result = diracAdmin.resetJob(job) + if result['OK']: + print('Reset Job %s' % (job)) + else: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_service_ports.py b/src/DIRAC/Interfaces/scripts/dirac_admin_service_ports.py new file mode 100755 index 00000000000..150215eafa3 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_service_ports.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-service-ports +# Author : Stuart Paterson +######################################################################## +""" + Print the service ports for the specified setup +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... [Setup]' % Script.scriptName, + 'Arguments:', + ' Setup: Name of the setup'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + setup = '' + if args: + setup = args[0] + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + result = diracAdmin.getServicePorts(setup, printOutput=True) + if result['OK']: + DIRAC.exit(0) + else: + print(result['Message']) + DIRAC.exit(2) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_set_site_protocols.py b/src/DIRAC/Interfaces/scripts/dirac_admin_set_site_protocols.py new file mode 100755 index 00000000000..fdeb0a2a90a --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_set_site_protocols.py @@ -0,0 +1,52 @@ +#! /usr/bin/env python +######################################################################## +# File : dirac-admin-set-site-protocols +# Author : Stuart Paterson +######################################################################## +""" + Defined protocols for each SE for a given site. +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch("", "Site=", "Site for which protocols are to be set (mandatory)") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Protocol ...' % Script.scriptName, + 'Arguments:', + ' Protocol: SE access protocol (mandatory)'])) + Script.parseCommandLine(ignoreErrors=True) + + site = None + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "site": + site = switch[1] + + args = Script.getPositionalArgs() + + if not site or not args: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + result = diracAdmin.setSiteProtocols(site, args, printOutput=True) + if not result['OK']: + print('ERROR: %s' % result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py new file mode 100755 index 00000000000..b632b915d81 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-site-info +# Author : Stuart Paterson +######################################################################## +""" + Print Configuration information for a given Site +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Site ...' % Script.scriptName, + 'Arguments:', + ' Site: Name of the Site'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + for site in args: + + result = diracAdmin.getSiteSection(site, printOutput=True) + if not result['OK']: + errorList.append((site, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_site_mask_logging.py b/src/DIRAC/Interfaces/scripts/dirac_admin_site_mask_logging.py new file mode 100755 index 00000000000..f70d9566a4e --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_site_mask_logging.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +""" + Retrieves site mask logging information. +""" + +######################################################################## +# File : dirac-admin-site-mask-logging +# Author : Stuart Paterson +######################################################################## + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Site ...' % Script.scriptName, + 'Arguments:', + ' Site: Name of the Site'])) + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + errorList = [] + + for site in args: + result = diracAdmin.getSiteMaskLogging(site, printOutput=True) + if not result['OK']: + errorList.append((site, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_sync_users_from_file.py b/src/DIRAC/Interfaces/scripts/dirac_admin_sync_users_from_file.py new file mode 100755 index 00000000000..4640befe3c0 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_sync_users_from_file.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-sync-users-from-file +# Author : Adrian Casajus +######################################################################## +""" + Sync users in Configuration with the cfg contents. +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from diraccfg import CFG + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + +__RCSID__ = "$Id$" + + +@DIRACScript() +def main(): + Script.registerSwitch("t", "test", "Only test. Don't commit changes") + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... UserCfg' % + Script.scriptName, + 'Arguments:', + ' UserCfg: Cfg FileName with Users as sections containing DN, Groups, and other properties as options' + ])) + Script.parseCommandLine(ignoreErrors=True) + + args = Script.getExtraCLICFGFiles() + + if len(args) < 1: + Script.showHelp() + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + testOnly = False + errorList = [] + + for unprocSw in Script.getUnprocessedSwitches(): + if unprocSw[0] in ("t", "test"): + testOnly = True + + try: + usersCFG = CFG().loadFromFile(args[0]) + except Exception as e: + errorList.append("file open", "Can't parse file %s: %s" % (args[0], str(e))) + errorCode = 1 + else: + if not diracAdmin.csSyncUsersWithCFG(usersCFG): + errorList.append(("modify users", "Cannot sync with %s" % args[0])) + exitCode = 255 + + if not exitCode and not testOnly: + result = diracAdmin.csCommitChanges() + if not result['OK']: + errorList.append(("commit", result['Message'])) + exitCode = 255 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_get_file.py b/src/DIRAC/Interfaces/scripts/dirac_dms_get_file.py new file mode 100755 index 00000000000..735d546353c --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_get_file.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-dms-get-file +# Author : Stuart Paterson +######################################################################## +""" + Retrieve a single file or list of files from Grid storage to the current directory. +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, + 'Arguments:', + ' LFN: Logical File Name or file containing LFNs'])) + Script.parseCommandLine(ignoreErrors=True) + lfns = Script.getPositionalArgs() + + if len(lfns) < 1: + Script.showHelp() + + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + + if len(lfns) == 1: + try: + with open(lfns[0], 'r') as f: + lfns = f.read().splitlines() + except Exception: + pass + + result = dirac.getFile(lfns, printOutput=True) + if not result['OK']: + print('ERROR %s' % (result['Message'])) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_accessURL.py b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_accessURL.py new file mode 100755 index 00000000000..289f881a11c --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_accessURL.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-dms-lfn-accessURL +# Author : Stuart Paterson +######################################################################## +""" + Retrieve an access URL for an LFN replica given a valid DIRAC SE. +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... LFN SE [PROTO]' % Script.scriptName, + 'Arguments:', + ' LFN: Logical File Name or file containing LFNs (mandatory)', + ' SE: Valid DIRAC SE (mandatory)', + ' PROTO: Optional protocol for accessURL'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + # pylint: disable=wrong-import-position + from DIRAC.Interfaces.API.Dirac import Dirac + + if len(args) < 2: + Script.showHelp(exitCode=1) + + if len(args) > 3: + print('Only one LFN SE pair will be considered') + + dirac = Dirac() + exitCode = 0 + + lfn = args[0] + seName = args[1] + proto = False + if len(args) > 2: + proto = args[2] + + try: + with open(lfn, 'r') as f: + lfns = f.read().splitlines() + except IOError: + lfns = [lfn] + + for lfn in lfns: + result = dirac.getAccessURL(lfn, seName, protocol=proto, printOutput=True) + if not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_metadata.py b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_metadata.py new file mode 100755 index 00000000000..39e534022c6 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_metadata.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-lfn-metadata +# Author : Stuart Paterson +######################################################################## +""" + Obtain replica metadata from file catalogue client. +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, + 'Arguments:', + ' LFN: Logical File Name or file containing LFNs'])) + Script.parseCommandLine(ignoreErrors=True) + lfns = Script.getPositionalArgs() + + if len(lfns) < 1: + Script.showHelp() + + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + errorList = [] + + if len(lfns) == 1: + try: + with open(lfns[0], 'r') as f: + lfns = f.read().splitlines() + except BaseException: + pass + + result = dirac.getLfnMetadata(lfns, printOutput=True) + if not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_replicas.py b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_replicas.py new file mode 100755 index 00000000000..1e4f142f200 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_lfn_replicas.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-lfn-replicas +# Author : Stuart Paterson +######################################################################## +""" + Obtain replica information from file catalogue client. +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch('a', "All", " Also show inactive replicas") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, + 'Arguments:', + ' LFN: Logical File Name or file containing LFNs'])) + Script.parseCommandLine(ignoreErrors=True) + lfns = Script.getPositionalArgs() + switches = Script.getUnprocessedSwitches() + + active = True + for switch in switches: + opt = switch[0].lower() + if opt in ("a", "all"): + active = False + if len(lfns) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + + if len(lfns) == 1: + try: + with open(lfns[0], 'r') as f: + lfns = f.read().splitlines() + except BaseException: + pass + + result = dirac.getReplicas(lfns, active=active, printOutput=True) + if not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_accessURL.py b/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_accessURL.py new file mode 100755 index 00000000000..dce18d093bb --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_accessURL.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-dms-pfn-accessURL +# Author : Stuart Paterson +######################################################################## +""" + Retrieve an access URL for a PFN given a valid DIRAC SE +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... PFN SE' % Script.scriptName, + 'Arguments:', + ' PFN: Physical File Name or file containing PFNs (mandatory)', + ' SE: Valid DIRAC SE (mandatory)'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 2: + Script.showHelp(exitCode=1) + + if len(args) > 2: + print('Only one PFN SE pair will be considered') + + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + + pfn = args[0] + seName = args[1] + try: + with open(pfn, 'r') as f: + pfns = f.read().splitlines() + except BaseException: + pfns = [pfn] + + for pfn in pfns: + result = dirac.getPhysicalFileAccessURL(pfn, seName, printOutput=True) + if not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_metadata.py b/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_metadata.py new file mode 100755 index 00000000000..db31f525148 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_pfn_metadata.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-dms-pfn-metadata +# Author : Stuart Paterson +######################################################################## +""" + Retrieve metadata for a PFN given a valid DIRAC SE +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... PFN SE' % Script.scriptName, + 'Arguments:', + ' PFN: Physical File Name or file containing PFNs (mandatory)', + ' SE: Valid DIRAC SE (mandatory)'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 2: + Script.showHelp(exitCode=1) + + if len(args) > 2: + print('Only one PFN SE pair will be considered') + + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + + pfn = args[0] + seName = args[1] + try: + f = open(pfn, 'r') + pfns = f.read().splitlines() + f.close() + except BaseException: + pfns = [pfn] + + for pfn in pfns: + result = dirac.getPhysicalFileMetadata(pfn, seName, printOutput=True) + if not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_dms_replicate_lfn.py b/src/DIRAC/Interfaces/scripts/dirac_dms_replicate_lfn.py new file mode 100755 index 00000000000..ac6cd2c2ae3 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_dms_replicate_lfn.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-dms-replicate-lfn +# Author : Stuart Paterson +######################################################################## +""" + Replicate an existing LFN to another Storage Element +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... LFN Dest [Source [Cache]]' % Script.scriptName, + 'Arguments:', + ' LFN: Logical File Name or file containing LFNs (mandatory)', + ' Dest: Valid DIRAC SE (mandatory)', + ' Source: Valid DIRAC SE', + ' Cache: Local directory to be used as cache'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 2 or len(args) > 4: + Script.showHelp(exitCode=1) + + lfn = args[0] + seName = args[1] + sourceSE = '' + localCache = '' + if len(args) > 2: + sourceSE = args[2] + if len(args) == 4: + localCache = args[3] + + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + + try: + f = open(lfn, 'r') + lfns = f.read().splitlines() + f.close() + except BaseException: + lfns = [lfn] + + finalResult = {"Failed": [], "Successful": []} + for lfn in lfns: + result = dirac.replicateFile(lfn, seName, sourceSE, localCache, printOutput=True) + if not result['OK']: + finalResult["Failed"].append(lfn) + print('ERROR %s' % (result['Message'])) + exitCode = 2 + else: + finalResult["Successful"].append(lfn) + + if len(lfns) > 1: + print(finalResult) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_framework_ping_service.py b/src/DIRAC/Interfaces/scripts/dirac_framework_ping_service.py new file mode 100755 index 00000000000..2e3184ad3c5 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_framework_ping_service.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-framework-ping-service +# Author : Stuart Paterson +######################################################################## +""" + Ping the given DIRAC Service +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... System Service|System/Agent' % Script.scriptName, + 'Arguments:', + ' System: Name of the DIRAC system (ie: WorkloadManagement) (mandatory)', + ' Service: Name of the DIRAC service (ie: Matcher) (mandatory)', + ' url: URL of the service to ping (instead of System and Service)'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + system = None + service = None + url = None + if len(args) == 1: + # it is a URL + if args[0].startswith('dips://'): + url = args[0] + # It is System/Service + else: + sys_serv = args[0].split('/') + if len(sys_serv) != 2: + Script.showHelp(exitCode=1) + else: + system, service = sys_serv + + elif len(args) == 2: + system, service = args[0], args[1] + else: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() + exitCode = 0 + + result = dirac.pingService(system, service, printOutput=True, url=url) + + if not result: + print('ERROR: Null result from ping()') + exitCode = 2 + elif not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_framework_self_ping.py b/src/DIRAC/Interfaces/scripts/dirac_framework_self_ping.py new file mode 100755 index 00000000000..613b6342084 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_framework_self_ping.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +""" Performs a DIPS ping on a given target and exit with the return code. +It uses the local host certificate +The target is specified as ""/System/Service" +The script does not print anything, and just exists with 0 in case of success, +or 1 in case of error """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + + +import sys +import os +import time +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + with open(os.devnull, 'w') as redirectStdout, open(os.devnull, 'w') as redirectStderr: + from DIRAC import gLogger + from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true') + gLogger.setLevel('FATAL') + from DIRAC.Core.DISET.RPCClient import RPCClient + + rpc = RPCClient('dips://localhost:%s' % sys.argv[1]) + res = rpc.ping() + time.sleep(0.1) + if not res['OK']: + sys.exit(1) + else: + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_repo_monitor.py b/src/DIRAC/Interfaces/scripts/dirac_repo_monitor.py new file mode 100755 index 00000000000..1804946127f --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_repo_monitor.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +""" + Monitor the jobs present in the repository +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... RepoDir' % Script.scriptName, + 'Arguments:', + ' RepoDir: Location of Job Repository'])) + Script.parseCommandLine(ignoreErrors=False) + args = Script.getPositionalArgs() + + if len(args) != 1: + Script.showHelp() + + repoLocation = args[0] + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac(withRepo=True, repoLocation=repoLocation) + + exitCode = 0 + result = dirac.monitorRepository(printOutput=True) + if not result['OK']: + print('ERROR: ', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_utils_file_adler.py b/src/DIRAC/Interfaces/scripts/dirac_utils_file_adler.py new file mode 100755 index 00000000000..620fe5b2ff2 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_utils_file_adler.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-utils-file-adler +######################################################################## +""" + Calculate alder32 of the supplied file +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Utilities.Adler import fileAdler +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... File ...' % Script.scriptName, + 'Arguments:', + ' File: File Name'])) + Script.parseCommandLine(ignoreErrors=False) + files = Script.getPositionalArgs() + if len(files) == 0: + Script.showHelp() + + exitCode = 0 + + for fa in files: + adler = fileAdler(fa) + if adler: + print(fa.rjust(100), adler.ljust(10)) # pylint: disable=no-member + else: + print('ERROR %s: Failed to get adler' % fa) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_utils_file_md5.py b/src/DIRAC/Interfaces/scripts/dirac_utils_file_md5.py new file mode 100755 index 00000000000..cdb70483909 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_utils_file_md5.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-utils-file-adler +# Author : +######################################################################## +""" + Calculate md5 of the supplied file +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Utilities.File import makeGuid +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... File ...' % Script.scriptName, + 'Arguments:', + ' File: File Name'])) + Script.parseCommandLine(ignoreErrors=False) + files = Script.getPositionalArgs() + if len(files) == 0: + Script.showHelp() + + exitCode = 0 + + for file in files: + try: + md5 = makeGuid(file) + if md5: + print(file.rjust(100), md5.ljust(10)) + else: + print('ERROR %s: Failed to get md5' % file) + exitCode = 2 + except Exception as x: + print('ERROR %s: Failed to get md5' % file, str(x)) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py b/src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py new file mode 100755 index 00000000000..72a42c1799d --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-get-normalized-queue-length.py +# Author : Ricardo Graciani +######################################################################## +""" + Report Normalized CPU length of queue + + This script was used by the dirac-pilot script to set the CPUTime limit for + the matching but now this is no more the case. +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import queueNormalizedCPU +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Queue ...' % Script.scriptName, + 'Arguments:', + ' Queue: GlueCEUniqueID of the Queue (ie, juk.nikhef.nl:8443/cream-pbs-lhcb)' + ])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + exitCode = 0 + + for ceUniqueID in args: + + normCPU = queueNormalizedCPU(ceUniqueID) + + if not normCPU['OK']: + print('ERROR %s:' % ceUniqueID, normCPU['Message']) + exitCode = 2 + continue + print(ceUniqueID, normCPU['Value']) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py b/src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py new file mode 100755 index 00000000000..4a3e2f4627a --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-get-queue-normalization.py +# Author : Ricardo Graciani +######################################################################## +""" + Report Normalization Factor applied by Site to the given Queue +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import getQueueNormalization +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Queue ...' % Script.scriptName, + 'Arguments:', + ' Queue: GlueCEUniqueID of the Queue (ie, juk.nikhef.nl:8443/cream-pbs-lhcb)', + ])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + exitCode = 0 + + for ceUniqueID in args: + + cpuNorm = getQueueNormalization(ceUniqueID) + + if not cpuNorm['OK']: + print('ERROR %s:' % ceUniqueID, cpuNorm['Message']) + exitCode = 2 + continue + print(ceUniqueID, cpuNorm['Value']) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py new file mode 100755 index 00000000000..89fdf413266 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-job-attributes +# Author : Stuart Paterson +######################################################################## +""" + Retrieve attributes associated with the given DIRAC job +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] + + for job in parseArguments(args): + + result = dirac.getJobAttributes(job, printOutput=True) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac-wms-job-delete.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_delete.py similarity index 71% rename from src/DIRAC/Interfaces/scripts/dirac-wms-job-delete.py rename to src/DIRAC/Interfaces/scripts/dirac_wms_job_delete.py index 7d02057e3bc..7f69e3a538a 100755 --- a/src/DIRAC/Interfaces/scripts/dirac-wms-job-delete.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_delete.py @@ -17,21 +17,22 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC Job ID'])) -Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file") -Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group") +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() + Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file") + Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group") - -if __name__ == "__main__": + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments from DIRAC.Core.Utilities.Time import toString, date, day @@ -72,3 +73,7 @@ exitCode = 2 DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py new file mode 100755 index 00000000000..3759ac87a2e --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-production-job-get-input +# Author : Stuart Paterson +######################################################################## +""" + Retrieve input sandbox for DIRAC Job +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import os + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.registerSwitch("D:", "Dir=", "Store the output in this directory") + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] + + outputDir = None + for sw, v in Script.getUnprocessedSwitches(): + if sw in ('D', 'Dir'): + outputDir = v + + for job in parseArguments(args): + + result = dirac.getInputSandbox(job, outputDir=outputDir) + if result['OK']: + if os.path.exists('InputSandbox%s' % job): + print('Job input sandbox retrieved in InputSandbox%s/' % (job)) + else: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py new file mode 100755 index 00000000000..d655b7a15e2 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-job-get-jdl +# Author : Stuart Paterson +######################################################################## +""" + Retrieve the current JDL of a DIRAC job +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + original = False + Script.registerSwitch('O', 'Original', 'Gets the original JDL') + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + for switch in Script.getUnprocessedSwitches(): + if switch[0] == 'Original' or switch[0] == 'O': + original = True + + for switch in Script.getUnprocessedSwitches(): + if switch[0] == 'Original': + original = True + + if len(args) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] + + for job in parseArguments(args): + + result = dirac.getJobJDL(job, original=original, printOutput=True) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py new file mode 100755 index 00000000000..3f860b2031e --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-job-get-output +# Author : Stuart Paterson +######################################################################## +""" + Retrieve output sandbox for a DIRAC job +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import os +import shutil + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID or a name of the file with JobID per line'])) + + Script.registerSwitch("D:", "Dir=", "Store the output in this directory") + Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file") + Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group") + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + from DIRAC.Core.Utilities.Time import toString, date, day + from DIRAC.Core.Utilities.File import mkDir + + dirac = Dirac() + exitCode = 0 + errorList = [] + + outputDir = None + group = None + jobs = [] + for sw, value in Script.getUnprocessedSwitches(): + if sw in ('D', 'Dir'): + outputDir = value + elif sw.lower() in ('f', 'file'): + if os.path.exists(value): + jFile = open(value) + jobs += jFile.read().split() + jFile.close() + elif sw.lower() in ('g', 'jobgroup'): + group = value + jobDate = toString(date() - 30 * day) + + # Choose jobs in final state, no more than 30 days old + result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Done') + if not result['OK']: + if "No jobs selected" not in result['Message']: + print("Error:", result['Message']) + DIRAC.exit(-1) + else: + jobs += result['Value'] + result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Failed') + if not result['OK']: + if "No jobs selected" not in result['Message']: + print("Error:", result['Message']) + DIRAC.exit(-1) + else: + jobs += result['Value'] + + for arg in parseArguments(args): + if os.path.isdir(arg): + print("Output for job %s already retrieved, remove the output directory to redownload" % arg) + else: + jobs.append(arg) + + if not jobs: + print("No jobs selected") + DIRAC.exit(0) + + if group: + if outputDir: + outputDir = os.path.join(outputDir, group) + else: + outputDir = group + + if outputDir: + mkDir(outputDir) + else: + outputDir = os.getcwd() + + jobs = [str(job) for job in jobs] + doneJobs = os.listdir(outputDir) + todoJobs = [job for job in jobs if job not in doneJobs] + + for job in todoJobs: + + result = dirac.getOutputSandbox(job, outputDir=outputDir) + + jobDir = str(job) + if outputDir: + jobDir = os.path.join(outputDir, job) + if result['OK']: + if os.path.exists(jobDir): + print('Job output sandbox retrieved in %s/' % (jobDir)) + else: + if os.path.exists('%s' % jobDir): + shutil.rmtree(jobDir) + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py new file mode 100755 index 00000000000..60d79d8c88c --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-job-get-output-data +# Author : Stuart Paterson +######################################################################## +""" + Retrieve the output data files of a DIRAC job +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.registerSwitch("D:", "Dir=", "Store the output in this directory") + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] + + outputDir = '' + for sw, v in Script.getUnprocessedSwitches(): + if sw in ('D', 'Dir'): + outputDir = v + + for job in parseArguments(args): + + result = dirac.getJobOutputData(job, destinationDir=outputDir) + if result['OK']: + print('Job %s output data retrieved' % (job)) + else: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_kill.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_kill.py new file mode 100755 index 00000000000..e5ce171f569 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_kill.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-job-kil +# Author : Stuart Paterson +######################################################################## +""" + Issue a kill signal to a running DIRAC job +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + + result = Dirac().killJob(parseArguments(args)) + if result['OK']: + print('Killed jobs %s' % ','.join([str(j) for j in result['Value']])) + exitCode = 0 + else: + print('ERROR', result['Message']) + exitCode = 2 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py new file mode 100755 index 00000000000..581b0dd882f --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-job-logging-info +# Author : Stuart Paterson +######################################################################## +""" + Retrieve history of transitions for a DIRAC job +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] + + for job in parseArguments(args): + + result = dirac.getJobLoggingInfo(job, printOutput=True) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py new file mode 100755 index 00000000000..938abf9eeaa --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-job-parameters +# Author : Stuart Paterson +######################################################################## +""" + Retrieve parameters associated to the given DIRAC job +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC + +from DIRAC import gLogger +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] + + for job in parseArguments(args): + + result = dirac.getJobParameters(job, printOutput=True) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py new file mode 100755 index 00000000000..fe7b6dd3d93 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-job-delete +# Author : Stuart Paterson +######################################################################## +""" + Peek StdOut of the given DIRAC job +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] + + for job in parseArguments(args): + + result = dirac.peekJob(job, printOutput=True) + if not result['OK']: + errorList.append((job, result['Message'])) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_reschedule.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_reschedule.py new file mode 100755 index 00000000000..40483652e07 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_reschedule.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-job-delete +# Author : Stuart Paterson +######################################################################## +""" + Reschedule the given DIRAC job +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC Job ID'])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + errorList = [] + + result = dirac.rescheduleJob(parseArguments(args)) + if result['OK']: + print('Rescheduled job %s' % ','.join([str(j) for j in result['Value']])) + else: + errorList.append((j, result['Message'])) + print(result['Message']) + exitCode = 2 + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_status.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_status.py new file mode 100755 index 00000000000..1cbb2f53070 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_status.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-job-status +# Author : Stuart Paterson +######################################################################## +""" + Retrieve status of the given DIRAC job +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import os +import DIRAC +from DIRAC import exit as DIRACExit +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.Core.Utilities.Time import toString, date, day + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + '\nUsage:\n', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, + '\nArguments:\n', + ' JobID: DIRAC Job ID'])) + + Script.registerSwitch("f:", "File=", "Get status for jobs with IDs from the file") + Script.registerSwitch("g:", "JobGroup=", "Get status for jobs in the given group") + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments + dirac = Dirac() + exitCode = 0 + + jobs = [] + for key, value in Script.getUnprocessedSwitches(): + if key.lower() in ('f', 'file'): + if os.path.exists(value): + jFile = open(value) + jobs += jFile.read().split() + jFile.close() + elif key.lower() in ('g', 'jobgroup'): + jobDate = toString(date() - 30 * day) + # Choose jobs no more than 30 days old + result = dirac.selectJobs(jobGroup=value, date=jobDate) + if not result['OK']: + print("Error:", result['Message']) + DIRACExit(-1) + jobs += result['Value'] + + if len(args) < 1 and not jobs: + Script.showHelp(exitCode=1) + + if len(args) > 0: + jobs += parseArguments(args) + + result = dirac.getJobStatus(jobs) + if result['OK']: + for job in result['Value']: + print('JobID=' + str(job), end=' ') + for status in result['Value'][job].items(): + print('%s=%s;' % status, end=' ') + print() + else: + exitCode = 2 + print("ERROR: %s" % result['Message']) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py new file mode 100755 index 00000000000..c5e40fb7284 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-job-submit +# Author : Stuart Paterson +######################################################################## +""" + Submit jobs to DIRAC WMS +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +import os + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JDL ...' % Script.scriptName, + 'Arguments:', + ' JDL: Path to JDL file'])) + + Script.registerSwitch("f:", "File=", "Writes job ids to file ") + Script.registerSwitch("r:", "UseJobRepo=", "Use the job repository") + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC.Interfaces.API.Dirac import Dirac + unprocessed_switches = Script.getUnprocessedSwitches() + use_repo = False + repo_name = "" + for sw, value in unprocessed_switches: + if sw.lower() in ["r", "usejobrepo"]: + use_repo = True + repo_name = value + repo_name = repo_name.replace(".cfg", ".repo") + dirac = Dirac(use_repo, repo_name) + exitCode = 0 + errorList = [] + + jFile = None + for sw, value in unprocessed_switches: + if sw.lower() in ('f', 'file'): + if os.path.isfile(value): + print('Appending job ids to existing logfile: %s' % value) + if not os.access(value, os.W_OK): + print('Existing logfile %s must be writable by user.' % value) + jFile = open(value, 'a') + + for jdl in args: + + result = dirac.submitJob(jdl) + if result['OK']: + print('JobID = %s' % (result['Value'])) + if jFile is not None: + # parametric jobs + if isinstance(result['Value'], list): + jFile.write('\n'.join(str(p) for p in result['Value'])) + jFile.write('\n') + else: + jFile.write(str(result['Value']) + '\n') + else: + errorList.append((jdl, result['Message'])) + exitCode = 2 + + if jFile is not None: + jFile.close() + + for error in errorList: + print("ERROR %s: %s" % error) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py b/src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py new file mode 100755 index 00000000000..fb6569930c8 --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py @@ -0,0 +1,120 @@ +#! /usr/bin/env python +######################################################################## +# File : dirac-wms-jobs-select-output-search +# Author : Vladimir Romanovsky +######################################################################## +""" + Retrieve output sandbox for DIRAC Jobs for the given selection and search for a string in their std.out +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import os +from shutil import rmtree + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch("", "Status=", "Primary status") + Script.registerSwitch("", "MinorStatus=", "Secondary status") + Script.registerSwitch("", "ApplicationStatus=", "Application status") + Script.registerSwitch("", "Site=", "Execution site") + Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)") + Script.registerSwitch("", "JobGroup=", "Select jobs for specified job group") + Script.registerSwitch("", "Date=", "Date in YYYY-MM-DD format, if not specified default is today") + Script.registerSwitch("", "File=", "File name,if not specified default is std.out ") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... String ...' % Script.scriptName, + 'Arguments:', + ' String: string to search for'])) + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + # Default values + status = None + minorStatus = None + appStatus = None + site = None + owner = None + jobGroup = None + date = None + filename = 'std.out' + + if len(args) != 1: + Script.showHelp() + + searchstring = str(args[0]) + + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "status": + status = switch[1] + elif switch[0].lower() == "minorstatus": + minorStatus = switch[1] + elif switch[0].lower() == "applicationstatus": + appStatus = switch[1] + elif switch[0].lower() == "site": + site = switch[1] + elif switch[0].lower() == "owner": + owner = switch[1] + elif switch[0].lower() == "jobgroup": + jobGroup = switch[1] + elif switch[0].lower() == "date": + date = switch[1] + elif switch[0].lower() == "file": + filename = switch[1] + + selDate = date + if not date: + selDate = 'Today' + + from DIRAC.Interfaces.API.Dirac import Dirac + + dirac = Dirac() + exitCode = 0 + errorList = [] + resultDict = {} + + result = dirac.selectJobs(status=status, minorStatus=minorStatus, applicationStatus=appStatus, + site=site, owner=owner, jobGroup=jobGroup, date=date) + if result['OK']: + jobs = result['Value'] + else: + print("Error in selectJob", result['Message']) + DIRAC.exit(2) + + for job in jobs: + + result = dirac.getOutputSandbox(job) + if result['OK']: + if os.path.exists('%s' % job): + + lines = [] + try: + lines = open(os.path.join(job, filename)).readlines() + except Exception as x: + errorList.append((job, x)) + for line in lines: + if line.count(searchstring): + resultDict[job] = line + rmtree("%s" % (job)) + else: + errorList.append((job, result['Message'])) + exitCode = 2 + + for result in resultDict.items(): + print(result) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_select_jobs.py b/src/DIRAC/Interfaces/scripts/dirac_wms_select_jobs.py new file mode 100755 index 00000000000..ac9f9199a2f --- /dev/null +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_select_jobs.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-select-jobs +# Author : Stuart Paterson +######################################################################## +""" + Select DIRAC jobs matching the given conditions +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC import gLogger +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + maxJobs = 100 + Script.registerSwitch("", "Status=", "Primary status") + Script.registerSwitch("", "MinorStatus=", "Secondary status") + Script.registerSwitch("", "ApplicationStatus=", "Application status") + Script.registerSwitch("", "Site=", "Execution site") + Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)") + Script.registerSwitch("", "JobGroup=", "Select jobs for specified job group") + Script.registerSwitch("", "Date=", "Date in YYYY-MM-DD format, if not specified default is today") + Script.registerSwitch("", "Maximum=", "Maximum number of jobs shown (default %d, 0 means all)" % maxJobs) + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... JobID ...' % Script.scriptName])) + Script.parseCommandLine(ignoreErrors=True) + + args = Script.getPositionalArgs() + + # Default values + status = None + minorStatus = None + appStatus = None + site = None + owner = None + jobGroups = [] + date = None + + if args: + Script.showHelp() + + exitCode = 0 + + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "status": + status = switch[1] + elif switch[0].lower() == "minorstatus": + minorStatus = switch[1] + elif switch[0].lower() == "applicationstatus": + appStatus = switch[1] + elif switch[0].lower() == "site": + site = switch[1] + elif switch[0].lower() == "owner": + owner = switch[1] + elif switch[0].lower() == "jobgroup": + for jg in switch[1].split(','): + if jg.isdigit(): + jobGroups.append('%08d' % int(jg)) + else: + jobGroups.append(jg) + elif switch[0].lower() == "date": + date = switch[1] + elif switch[0] == 'Maximum': + try: + maxJobs = int(switch[1]) + except TypeError: + gLogger.fatal("Invalid max number of jobs", switch[1]) + DIRAC.exit(1) + + selDate = date + if not date: + selDate = 'Today' + conditions = {'Status': status, + 'MinorStatus': minorStatus, + 'ApplicationStatus': appStatus, + 'Owner': owner, + 'JobGroup': ','.join(str(jg) for jg in jobGroups), + 'Date': selDate} + + from DIRAC.Interfaces.API.Dirac import Dirac + + dirac = Dirac() + jobs = [] + + if jobGroups: + for jobGroup in jobGroups: + res = dirac.selectJobs(status=status, + minorStatus=minorStatus, + applicationStatus=appStatus, + site=site, + owner=owner, + jobGroup=jobGroup, + date=date, + printErrors=False) + if res['OK']: + jobs.extend(res['Value']) + else: + gLogger.error("Can't select jobs: ", res['Message']) + else: + res = dirac.selectJobs(status=status, + minorStatus=minorStatus, + applicationStatus=appStatus, + site=site, + owner=owner, + date=date, + printErrors=False) + if res['OK']: + jobs.extend(res['Value']) + else: + gLogger.error("Can't select jobs: ", res['Message']) + + conds = ['%s = %s' % (n, v) for n, v in conditions.items() if v] + if maxJobs and len(jobs) > maxJobs: + jobs = jobs[:maxJobs] + constrained = ' (first %d shown) ' % maxJobs + else: + constrained = ' ' + + if jobs: + gLogger.notice('==> Selected %s jobs%swith conditions: %s\n%s' % (len(jobs), + constrained, + ', '.join(conds), + ','.join(jobs))) + else: + gLogger.notice('No jobs were selected with conditions:', ', '.join(conds)) + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/__init__.py b/src/DIRAC/ProductionSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-add-trans.py b/src/DIRAC/ProductionSystem/scripts/dirac-prod-add-trans.py deleted file mode 100755 index 76e7f5ede1a..00000000000 --- a/src/DIRAC/ProductionSystem/scripts/dirac-prod-add-trans.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -""" - Add an existing transformation to an existing production. - Transformations already belonging to another production cannot be added. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID transID [parentTransID]' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID', - ' transID: Transformation ID', - ' parentTransID: Parent Transformation ID' - ])) - - -Script.parseCommandLine() - -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient - -prodClient = ProductionClient() -transClient = TransformationClient() - -# get arguments -args = Script.getPositionalArgs() -if len(args) == 3: - parentTransID = args[2] -elif len(args) == 2: - parentTransID = '' -else: - Script.showHelp(exitCode=1) - -prodID = args[0] -transID = args[1] - -res = transClient.getTransformation(transID) -if not res['OK']: - DIRAC.gLogger.error('Failed to get transformation %s: %s' % (transID, res['Message'])) - DIRAC.exit(-1) - -transID = res['Value']['TransformationID'] - -if parentTransID: - res = transClient.getTransformation(parentTransID) - if not res['OK']: - DIRAC.gLogger.error('Failed to get transformation %s: %s' % (parentTransID, res['Message'])) - DIRAC.exit(-1) - parentTransID = res['Value']['TransformationID'] - -res = prodClient.getProduction(prodID) -if not res['OK']: - DIRAC.gLogger.error('Failed to get production %s: %s' % (prodID, res['Message'])) - DIRAC.exit(-1) - -prodID = res['Value']['ProductionID'] -res = prodClient.addTransformationsToProduction(prodID, transID, parentTransID) -if not res['OK']: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) - -if parentTransID: - msg = 'Transformation %s successfully added to production %s with parent transformation %s' % \ - (transID, prodID, parentTransID) -else: - msg = 'Transformation %s successfully added to production %s with no parent transformation' % \ - (transID, prodID) - -DIRAC.gLogger.notice(msg) - - -DIRAC.exit(0) diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-clean.py b/src/DIRAC/ProductionSystem/scripts/dirac-prod-clean.py deleted file mode 100755 index c2de670411e..00000000000 --- a/src/DIRAC/ProductionSystem/scripts/dirac-prod-clean.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -""" - Clean a given production -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)' - ])) - - -Script.parseCommandLine() - -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient - -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) - -# get arguments -prodID = args[0] - -res = ProductionClient().setProductionStatus(prodID, 'Cleaned') -if not res['OK']: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(1) - -DIRAC.gLogger.notice('Production %s successully cleaned' % prodID) -DIRAC.exit(0) diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-delete.py b/src/DIRAC/ProductionSystem/scripts/dirac-prod-delete.py deleted file mode 100755 index c75d6d992c1..00000000000 --- a/src/DIRAC/ProductionSystem/scripts/dirac-prod-delete.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python - -""" - Delete a given production -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)' - ])) - - -Script.parseCommandLine() - -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient - -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) - -# get arguments -prodID = args[0] - -prodClient = ProductionClient() - -res = prodClient.deleteProduction(prodID) -if res['OK']: - DIRAC.gLogger.notice('Production %s successully deleted' % prodID) -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) - -DIRAC.exit(0) diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-get-all.py b/src/DIRAC/ProductionSystem/scripts/dirac-prod-get-all.py deleted file mode 100755 index 637cf7f776e..00000000000 --- a/src/DIRAC/ProductionSystem/scripts/dirac-prod-get-all.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python - -""" - Get summary informations of all productions -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script -from DIRAC.Core.Utilities.PrettyPrint import printTable - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1]])) - -Script.parseCommandLine() - -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient - -prodClient = ProductionClient() -res = prodClient.getProductions() - -fields = [ - 'ProductionName', - 'Status', - 'ProductionID', - 'CreationDate', - 'LastUpdate', - 'AuthorDN', - 'AuthorGroup'] -records = [] - -if res['OK']: - prodList = res['Value'] - if not isinstance(res['Value'], list): - prodList = [res['Value']] - for prod in prodList: - records.append( - [ - str( - prod['ProductionName']), str( - prod['Status']), str( - prod['ProductionID']), str( - prod['CreationDate']), str( - prod['LastUpdate']), str( - prod['AuthorDN']), str( - prod['AuthorGroup'])]) -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) - -printTable(fields, records) - -DIRAC.exit(0) diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-get-description.py b/src/DIRAC/ProductionSystem/scripts/dirac-prod-get-description.py deleted file mode 100755 index be937db62ec..00000000000 --- a/src/DIRAC/ProductionSystem/scripts/dirac-prod-get-description.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python - -""" - Get the description of a given production -""" - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)' - ])) - - -Script.parseCommandLine() - -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient - -prodClient = ProductionClient() - -# get arguments -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) -else: - prodID = args[0] - res = prodClient.getProduction(prodID) - -if res['OK']: - prod = res['Value'] -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) - -print('Description for production %s:\n' % prodID) -print(prod['Description']) - -DIRAC.exit(0) diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-get-trans.py b/src/DIRAC/ProductionSystem/scripts/dirac-prod-get-trans.py deleted file mode 100755 index 4e9a276b5bb..00000000000 --- a/src/DIRAC/ProductionSystem/scripts/dirac-prod-get-trans.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python - -""" - Get the transformations belonging to a given production -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script -from DIRAC.Core.Utilities.PrettyPrint import printTable - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)', - '\ne.g: %s 381' % Script.scriptName, - ])) - - -Script.parseCommandLine() - -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient - -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) - -# get arguments -prodID = args[0] - -prodClient = ProductionClient() -transClient = TransformationClient() - -res = prodClient.getProductionTransformations(prodID) -transIDs = [] - -if res['OK']: - transList = res['Value'] - if not transList: - DIRAC.gLogger.notice('No transformation associated with production %s' % prodID) - DIRAC.exit(-1) - for trans in transList: - transIDs.append(trans['TransformationID']) -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) - - -fields = [ - 'TransformationName', - 'Status', - 'F_Proc.', - 'F_Proc.(%)', - 'TransformationID', - 'ProductionID', - 'Prod_LastUpdate', - 'Prod_InsertedTime'] - -records = [] - -paramShowNames = ['TransformationID', 'TransformationName', 'Type', 'Status', 'Files_Total', 'Files_PercentProcessed', - 'Files_Processed', 'Files_Unused', 'Jobs_TotalCreated', 'Jobs_Waiting', - 'Jobs_Running', 'Jobs_Done', 'Jobs_Failed', 'Jobs_Stalled'] -resList = [] - - -res = transClient.getTransformationSummaryWeb({'TransformationID': transIDs}, [], 0, len(transIDs)) - -if not res['OK']: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) - -if res['Value']['TotalRecords'] > 0: - paramNames = res['Value']['ParameterNames'] - for paramValues in res['Value']['Records']: - paramShowValues = map(lambda pname: paramValues[paramNames.index(pname)], paramShowNames) - showDict = dict(zip(paramShowNames, paramShowValues)) - resList.append(showDict) - -for res in resList: - files_Processed = res['Files_Processed'] - files_PercentProcessed = res['Files_PercentProcessed'] - status = res['Status'] - type = res['Type'] - transName = res['TransformationName'] - transID = res['TransformationID'] - records.append([transName, status, str(files_Processed), str(files_PercentProcessed), str(transID), str(prodID), - str(trans['LastUpdate']), str(trans['InsertedTime'])]) - - -printTable(fields, records) - -DIRAC.exit(0) diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-get.py b/src/DIRAC/ProductionSystem/scripts/dirac-prod-get.py deleted file mode 100755 index 4abca1c93f8..00000000000 --- a/src/DIRAC/ProductionSystem/scripts/dirac-prod-get.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python - -""" - Get informations for a given production -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script -from DIRAC.Core.Utilities.PrettyPrint import printTable - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)' - ])) - - -Script.parseCommandLine() - -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient - -prodClient = ProductionClient() - -# get arguments -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) -else: - prodID = args[0] - res = prodClient.getProduction(prodID) - -fields = [ - 'ProductionName', - 'Status', - 'ProductionID', - 'CreationDate', - 'LastUpdate', - 'AuthorDN', - 'AuthorGroup'] -records = [] - -if res['OK']: - prodList = res['Value'] - if not isinstance(res['Value'], list): - prodList = [res['Value']] - for prod in prodList: - records.append( - [ - str( - prod['ProductionName']), str( - prod['Status']), str( - prod['ProductionID']), str( - prod['CreationDate']), str( - prod['LastUpdate']), str( - prod['AuthorDN']), str( - prod['AuthorGroup'])]) -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) - -printTable(fields, records) - -DIRAC.exit(0) diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-start.py b/src/DIRAC/ProductionSystem/scripts/dirac-prod-start.py deleted file mode 100755 index 32abec0037e..00000000000 --- a/src/DIRAC/ProductionSystem/scripts/dirac-prod-start.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python - -""" - Start a given production -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)' - ])) - - -Script.parseCommandLine() - -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient - -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) - -# get arguments -prodID = args[0] - -prodClient = ProductionClient() - -res = prodClient.setProductionStatus(prodID, 'Active') -if res['OK']: - DIRAC.gLogger.notice('Production %s successully started' % prodID) -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) - -DIRAC.exit(0) diff --git a/src/DIRAC/ProductionSystem/scripts/dirac-prod-stop.py b/src/DIRAC/ProductionSystem/scripts/dirac-prod-stop.py deleted file mode 100755 index 00be7945736..00000000000 --- a/src/DIRAC/ProductionSystem/scripts/dirac-prod-stop.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python - -""" - Stop a given production -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s prodID' % Script.scriptName, - 'Arguments:', - ' prodID: Production ID (mandatory)' - ])) - - -Script.parseCommandLine() - -from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient - -args = Script.getPositionalArgs() -if len(args) < 1: - Script.showHelp(exitCode=1) - -# get arguments -prodID = args[0] - -prodClient = ProductionClient() - -res = prodClient.setProductionStatus(prodID, 'Stopped') -if res['OK']: - DIRAC.gLogger.notice('Production %s successully stopped' % prodID) -else: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(-1) - -DIRAC.exit(0) diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_add_trans.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_add_trans.py new file mode 100755 index 00000000000..35bccf4a045 --- /dev/null +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_add_trans.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +""" + Add an existing transformation to an existing production. + Transformations already belonging to another production cannot be added. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID transID [parentTransID]' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID', + ' transID: Transformation ID', + ' parentTransID: Parent Transformation ID' + ])) + + Script.parseCommandLine() + + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + prodClient = ProductionClient() + transClient = TransformationClient() + + # get arguments + args = Script.getPositionalArgs() + if len(args) == 3: + parentTransID = args[2] + elif len(args) == 2: + parentTransID = '' + else: + Script.showHelp(exitCode=1) + + prodID = args[0] + transID = args[1] + + res = transClient.getTransformation(transID) + if not res['OK']: + DIRAC.gLogger.error('Failed to get transformation %s: %s' % (transID, res['Message'])) + DIRAC.exit(-1) + + transID = res['Value']['TransformationID'] + + if parentTransID: + res = transClient.getTransformation(parentTransID) + if not res['OK']: + DIRAC.gLogger.error('Failed to get transformation %s: %s' % (parentTransID, res['Message'])) + DIRAC.exit(-1) + parentTransID = res['Value']['TransformationID'] + + res = prodClient.getProduction(prodID) + if not res['OK']: + DIRAC.gLogger.error('Failed to get production %s: %s' % (prodID, res['Message'])) + DIRAC.exit(-1) + + prodID = res['Value']['ProductionID'] + res = prodClient.addTransformationsToProduction(prodID, transID, parentTransID) + if not res['OK']: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) + + if parentTransID: + msg = 'Transformation %s successfully added to production %s with parent transformation %s' % \ + (transID, prodID, parentTransID) + else: + msg = 'Transformation %s successfully added to production %s with no parent transformation' % \ + (transID, prodID) + + DIRAC.gLogger.notice(msg) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_clean.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_clean.py new file mode 100755 index 00000000000..77e37bf3754 --- /dev/null +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_clean.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +""" + Clean a given production +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)' + ])) + + Script.parseCommandLine() + + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) + + # get arguments + prodID = args[0] + + res = ProductionClient().setProductionStatus(prodID, 'Cleaned') + if not res['OK']: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(1) + + DIRAC.gLogger.notice('Production %s successully cleaned' % prodID) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_delete.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_delete.py new file mode 100755 index 00000000000..1a14f95bdaf --- /dev/null +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_delete.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +""" + Delete a given production +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)' + ])) + + Script.parseCommandLine() + + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) + + # get arguments + prodID = args[0] + + prodClient = ProductionClient() + + res = prodClient.deleteProduction(prodID) + if res['OK']: + DIRAC.gLogger.notice('Production %s successully deleted' % prodID) + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_get.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get.py new file mode 100755 index 00000000000..b312e701711 --- /dev/null +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +""" + Get informations for a given production +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC.Core.Utilities.PrettyPrint import printTable + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)' + ])) + + Script.parseCommandLine() + + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + + prodClient = ProductionClient() + + # get arguments + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) + else: + prodID = args[0] + res = prodClient.getProduction(prodID) + + fields = [ + 'ProductionName', + 'Status', + 'ProductionID', + 'CreationDate', + 'LastUpdate', + 'AuthorDN', + 'AuthorGroup'] + records = [] + + if res['OK']: + prodList = res['Value'] + if not isinstance(res['Value'], list): + prodList = [res['Value']] + for prod in prodList: + records.append( + [ + str( + prod['ProductionName']), str( + prod['Status']), str( + prod['ProductionID']), str( + prod['CreationDate']), str( + prod['LastUpdate']), str( + prod['AuthorDN']), str( + prod['AuthorGroup'])]) + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) + + printTable(fields, records) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py new file mode 100755 index 00000000000..2966fcfb415 --- /dev/null +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_all.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +""" + Get summary informations of all productions +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.PrettyPrint import printTable +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1]])) + + Script.parseCommandLine() + + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + + prodClient = ProductionClient() + res = prodClient.getProductions() + + fields = [ + 'ProductionName', + 'Status', + 'ProductionID', + 'CreationDate', + 'LastUpdate', + 'AuthorDN', + 'AuthorGroup'] + records = [] + + if res['OK']: + prodList = res['Value'] + if not isinstance(res['Value'], list): + prodList = [res['Value']] + for prod in prodList: + records.append( + [ + str( + prod['ProductionName']), str( + prod['Status']), str( + prod['ProductionID']), str( + prod['CreationDate']), str( + prod['LastUpdate']), str( + prod['AuthorDN']), str( + prod['AuthorGroup'])]) + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) + + printTable(fields, records) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_description.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_description.py new file mode 100755 index 00000000000..fe45903153b --- /dev/null +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_description.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +""" + Get the description of a given production +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)' + ])) + + Script.parseCommandLine() + + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + + prodClient = ProductionClient() + + # get arguments + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) + else: + prodID = args[0] + res = prodClient.getProduction(prodID) + + if res['OK']: + prod = res['Value'] + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) + + print('Description for production %s:\n' % prodID) + print(prod['Description']) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py new file mode 100755 index 00000000000..c4335998215 --- /dev/null +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_get_trans.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +""" + Get the transformations belonging to a given production +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.PrettyPrint import printTable +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)', + '\ne.g: %s 381' % Script.scriptName, + ])) + + Script.parseCommandLine() + + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) + + # get arguments + prodID = args[0] + + prodClient = ProductionClient() + transClient = TransformationClient() + + res = prodClient.getProductionTransformations(prodID) + transIDs = [] + + if res['OK']: + transList = res['Value'] + if not transList: + DIRAC.gLogger.notice('No transformation associated with production %s' % prodID) + DIRAC.exit(-1) + for trans in transList: + transIDs.append(trans['TransformationID']) + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) + + fields = [ + 'TransformationName', + 'Status', + 'F_Proc.', + 'F_Proc.(%)', + 'TransformationID', + 'ProductionID', + 'Prod_LastUpdate', + 'Prod_InsertedTime'] + + records = [] + + paramShowNames = ['TransformationID', 'TransformationName', 'Type', 'Status', 'Files_Total', 'Files_PercentProcessed', + 'Files_Processed', 'Files_Unused', 'Jobs_TotalCreated', 'Jobs_Waiting', + 'Jobs_Running', 'Jobs_Done', 'Jobs_Failed', 'Jobs_Stalled'] + resList = [] + + res = transClient.getTransformationSummaryWeb({'TransformationID': transIDs}, [], 0, len(transIDs)) + + if not res['OK']: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) + + if res['Value']['TotalRecords'] > 0: + paramNames = res['Value']['ParameterNames'] + for paramValues in res['Value']['Records']: + paramShowValues = map(lambda pname: paramValues[paramNames.index(pname)], paramShowNames) + showDict = dict(zip(paramShowNames, paramShowValues)) + resList.append(showDict) + + for res in resList: + files_Processed = res['Files_Processed'] + files_PercentProcessed = res['Files_PercentProcessed'] + status = res['Status'] + type = res['Type'] + transName = res['TransformationName'] + transID = res['TransformationID'] + records.append([transName, status, str(files_Processed), str(files_PercentProcessed), str(transID), str(prodID), + str(trans['LastUpdate']), str(trans['InsertedTime'])]) + + printTable(fields, records) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_start.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_start.py new file mode 100755 index 00000000000..b55f2656767 --- /dev/null +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_start.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +""" + Start a given production +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)' + ])) + + Script.parseCommandLine() + + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) + + # get arguments + prodID = args[0] + + prodClient = ProductionClient() + + res = prodClient.setProductionStatus(prodID, 'Active') + if res['OK']: + DIRAC.gLogger.notice('Production %s successully started' % prodID) + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_stop.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_stop.py new file mode 100755 index 00000000000..03b163a0d0a --- /dev/null +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_stop.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +""" + Stop a given production +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s prodID' % Script.scriptName, + 'Arguments:', + ' prodID: Production ID (mandatory)' + ])) + + Script.parseCommandLine() + + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + + args = Script.getPositionalArgs() + if len(args) < 1: + Script.showHelp(exitCode=1) + + # get arguments + prodID = args[0] + + prodClient = ProductionClient() + + res = prodClient.setProductionStatus(prodID, 'Stopped') + if res['OK']: + DIRAC.gLogger.notice('Production %s successully stopped' % prodID) + else: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(-1) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/RequestManagementSystem/scripts/__init__.py b/src/DIRAC/RequestManagementSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/RequestManagementSystem/scripts/dirac-rms-list-req-cache.py b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_list_req_cache.py similarity index 73% rename from src/DIRAC/RequestManagementSystem/scripts/dirac-rms-list-req-cache.py rename to src/DIRAC/RequestManagementSystem/scripts/dirac_rms_list_req_cache.py index 8a2a1fb8372..6e50f8bd881 100755 --- a/src/DIRAC/RequestManagementSystem/scripts/dirac-rms-list-req-cache.py +++ b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_list_req_cache.py @@ -7,15 +7,16 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Base.Script import parseCommandLine -Script.registerSwitch('', 'Full', ' Print full list of requests') -Script.setUsageMessage('\n'.join([__doc__, 'Usage:', ' %s [option|cfgfile]' % Script.scriptName])) -parseCommandLine() -from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient - -if __name__ == "__main__": +@DIRACScript() +def main(): + Script.registerSwitch('', 'Full', ' Print full list of requests') + Script.setUsageMessage('\n'.join([__doc__, 'Usage:', ' %s [option|cfgfile]' % Script.scriptName])) + parseCommandLine() + from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient fullPrint = False @@ -39,3 +40,7 @@ DIRAC.gLogger.always("Number of requests in the cache", len(reqCache)) DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/RequestManagementSystem/scripts/dirac-rms-reqdb-summary.py b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_reqdb_summary.py similarity index 83% rename from src/DIRAC/RequestManagementSystem/scripts/dirac-rms-reqdb-summary.py rename to src/DIRAC/RequestManagementSystem/scripts/dirac_rms_reqdb_summary.py index 7b6c200d5a7..68c7ddfe33b 100755 --- a/src/DIRAC/RequestManagementSystem/scripts/dirac-rms-reqdb-summary.py +++ b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_reqdb_summary.py @@ -6,11 +6,14 @@ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s [option|cfgfile]' % Script.scriptName])) +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if __name__ == "__main__": + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__, + 'Usage:', + ' %s [option|cfgfile]' % Script.scriptName])) from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() @@ -47,3 +50,7 @@ DIRAC.gLogger.always("- '%s' %s" % (fState, fCount)) DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/RequestManagementSystem/scripts/dirac-rms-request.py b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_request.py similarity index 79% rename from src/DIRAC/RequestManagementSystem/scripts/dirac-rms-request.py rename to src/DIRAC/RequestManagementSystem/scripts/dirac_rms_request.py index 7361a18e522..bc6488f49e2 100755 --- a/src/DIRAC/RequestManagementSystem/scripts/dirac-rms-request.py +++ b/src/DIRAC/RequestManagementSystem/scripts/dirac_rms_request.py @@ -7,6 +7,7 @@ import datetime import os +from DIRAC.Core.Utilities.DIRACScript import DIRACScript def convertDate(date): @@ -18,39 +19,40 @@ def convertDate(date): try: value = datetime.datetime.utcnow() - datetime.timedelta(hours=int(24 * float(date))) except Exception: + from DIRAC import gLogger gLogger.fatal("Invalid date", date) value = None return value -from DIRAC.Core.Base import Script -Script.registerSwitch('', 'Job=', ' JobID[,jobID2,...]') -Script.registerSwitch('', 'Transformation=', ' transformation ID') -Script.registerSwitch('', 'Tasks=', ' Associated to --Transformation, list of taskIDs') -Script.registerSwitch('', 'Verbose', ' Print more information') -Script.registerSwitch('', 'Terse', ' Only print request status') -Script.registerSwitch('', 'Full', ' Print full request content') -Script.registerSwitch('', 'Status=', ' Select all requests in a given status') -Script.registerSwitch('', 'Since=', - ' Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day') -Script.registerSwitch('', 'Until=', ' Associated to --Status, end date (default= now') -Script.registerSwitch('', 'Maximum=', ' Associated to --Status, max number of requests ') -Script.registerSwitch('', 'Reset', ' Reset Failed files to Waiting if any') -Script.registerSwitch('', 'Force', ' Force reset even if not Failed') -Script.registerSwitch('', 'All', ' (if --Status Failed) all requests, otherwise exclude irrecoverable failures') -Script.registerSwitch('', 'FixJob', ' Set job Done if the request is Done') -Script.registerSwitch('', 'Cancel', ' Cancel the request') -Script.registerSwitch('', 'ListJobs', ' List the corresponding jobs') -Script.registerSwitch('', 'TargetSE=', ' Select request only if that SE is in the targetSEs') -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s [option|cfgfile] [request[,request1,...]|' % Script.scriptName, - 'Arguments:', - ' request: a request ID or a unique request name', - ' : a file containing a list of requests (Comma-separated on each line)'])) +@DIRACScript() +def main(): + from DIRAC.Core.Base import Script + Script.registerSwitch('', 'Job=', ' JobID[,jobID2,...]') + Script.registerSwitch('', 'Transformation=', ' transformation ID') + Script.registerSwitch('', 'Tasks=', ' Associated to --Transformation, list of taskIDs') + Script.registerSwitch('', 'Verbose', ' Print more information') + Script.registerSwitch('', 'Terse', ' Only print request status') + Script.registerSwitch('', 'Full', ' Print full request content') + Script.registerSwitch('', 'Status=', ' Select all requests in a given status') + Script.registerSwitch('', 'Since=', + ' Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day') + Script.registerSwitch('', 'Until=', ' Associated to --Status, end date (default= now') + Script.registerSwitch('', 'Maximum=', ' Associated to --Status, max number of requests ') + Script.registerSwitch('', 'Reset', ' Reset Failed files to Waiting if any') + Script.registerSwitch('', 'Force', ' Force reset even if not Failed') + Script.registerSwitch('', 'All', ' (if --Status Failed) all requests, otherwise exclude irrecoverable failures') + Script.registerSwitch('', 'FixJob', ' Set job Done if the request is Done') + Script.registerSwitch('', 'Cancel', ' Cancel the request') + Script.registerSwitch('', 'ListJobs', ' List the corresponding jobs') + Script.registerSwitch('', 'TargetSE=', ' Select request only if that SE is in the targetSEs') + Script.setUsageMessage('\n'.join([__doc__, + 'Usage:', + ' %s [option|cfgfile] [request[,request1,...]|' % Script.scriptName, + 'Arguments:', + ' request: a request ID or a unique request name', + ' : a file containing a list of requests (Comma-separated on each line)'])) -# # execution -if __name__ == "__main__": from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() @@ -274,3 +276,7 @@ def convertDate(date): gLogger.notice('\nList of %d selected requests:' % len(okRequests)) for reqs in breakListIntoChunks(okRequests, 100): gLogger.notice(','.join(reqs)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ResourceStatusSystem/scripts/__init__.py b/src/DIRAC/ResourceStatusSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-list-status.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py similarity index 98% rename from src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-list-status.py rename to src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py index 93405d5af58..2be8149d4b2 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-list-status.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py @@ -15,6 +15,7 @@ from DIRAC import gLogger, exit as DIRACExit, version from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.ResourceStatusSystem.Client import ResourceStatusClient from DIRAC.Core.Utilities.PrettyPrint import printTable @@ -153,8 +154,8 @@ def run(): tabularPrint(elements) -if __name__ == "__main__": - +@DIRACScript() +def main(): subLogger = gLogger.getSubLogger(__file__) # Script initialization @@ -167,3 +168,7 @@ def run(): # Bye DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-query-db.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py similarity index 99% rename from src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-query-db.py rename to src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py index 257679020df..4183046b2f0 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-query-db.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py @@ -23,6 +23,7 @@ import datetime from DIRAC import gLogger, exit as DIRACExit, S_OK, version from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.ResourceStatusSystem.Client import ResourceStatusClient from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Security.ProxyInfo import getProxyInfo @@ -30,7 +31,6 @@ from DIRAC.Core.Utilities.PrettyPrint import printTable subLogger = None -switchDict = {} def registerSwitches(): @@ -387,7 +387,6 @@ def run(args, switchDictSet): """ Main function of the script """ - query = args[0] matches = 0 @@ -410,7 +409,9 @@ def run(args, switchDictSet): confirm(query, matches) -if __name__ == "__main__": +@DIRACScript() +def main(): + global subLogger subLogger = gLogger.getSubLogger(__file__) @@ -427,3 +428,7 @@ def run(args, switchDictSet): # Bye DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-query-dtcache.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py similarity index 98% rename from src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-query-dtcache.py rename to src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py index 4ec9ff2f2f2..7f5b0e97c40 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-query-dtcache.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py @@ -21,13 +21,13 @@ from DIRAC import gLogger, exit as DIRACExit, version from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Utilities import Time from DIRAC.Core.Utilities.PrettyPrint import printTable from DIRAC.ResourceStatusSystem.Utilities import Utils subLogger = None -switchDict = {} def registerSwitches(): @@ -338,7 +338,10 @@ def run(args, switchDict): error(result['Message']) -if __name__ == "__main__": +@DIRACScript() +def main(): + global subLogger + global ResourceManagementClient subLogger = gLogger.getSubLogger(__file__) @@ -346,12 +349,18 @@ def run(args, switchDict): registerSwitches() registerUsageMessage() args, switchDict = parseSwitches() + ResourceManagementClient = getattr( Utils.voimport('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient'), - 'ResourceManagementClient') + 'ResourceManagementClient', + ) # Run script run(args, switchDict) # Bye DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-set-status.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py similarity index 97% rename from src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-set-status.py rename to src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py index 9fba2a9166a..45125e5205d 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-set-status.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py @@ -14,6 +14,7 @@ from DIRAC import gLogger, exit as DIRACExit, S_OK, version from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.ResourceStatusSystem.Client import ResourceStatusClient from DIRAC.ResourceStatusSystem.PolicySystem import StateMachine @@ -21,7 +22,6 @@ subLogger = None -switchDict = {} def registerSwitches(): @@ -232,7 +232,10 @@ def run(switchDict): DIRACExit(1) -if __name__ == "__main__": +@DIRACScript() +def main(): + global subLogger + global registerUsageMessage subLogger = gLogger.getSubLogger(__file__) @@ -249,5 +252,6 @@ def run(switchDict): # Bye DIRACExit(0) -################################################################################ -# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-set-token.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py similarity index 97% rename from src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-set-token.py rename to src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py index cd447543343..05fd2afa032 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-set-token.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py @@ -21,6 +21,7 @@ # DIRAC from DIRAC import gLogger, exit as DIRACExit, S_OK, version from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient @@ -175,11 +176,22 @@ def setToken(user): return S_OK() +@DIRACScript() def main(): """ Main function of the script. Gets the username from the proxy loaded and sets the token taking into account that user and the switchDict parameters. """ + global subLogger + global switchDict + + # Logger initialization + subLogger = gLogger.getSubLogger(__file__) + + # Script initialization + registerSwitches() + registerUsageMessage() + switchDict = parseSwitches() user = proxyUser() if not user['OK']: @@ -192,17 +204,8 @@ def main(): subLogger.error(res['Message']) DIRACExit(1) + DIRACExit(0) -if __name__ == '__main__': - - # Logger initialization - subLogger = gLogger.getSubLogger(__file__) - - # Script initialization - registerSwitches() - registerUsageMessage() - switchDict = parseSwitches() +if __name__ == "__main__": main() - - DIRACExit(0) diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-sync.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py similarity index 98% rename from src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-sync.py rename to src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py index d92f3c0c011..78817687abf 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac-rss-sync.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py @@ -20,6 +20,7 @@ from DIRAC import version, gLogger, exit as DIRACExit, S_OK from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript subLogger = None switchDict = {} @@ -256,10 +257,14 @@ def run(): DIRACExit(1) -if __name__ == "__main__": - +@DIRACScript() +def main(): # Run script run() # Bye DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Resources/scripts/__init__.py b/src/DIRAC/Resources/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/Resources/scripts/dirac-resource-get-parameters.py b/src/DIRAC/Resources/scripts/dirac_resource_get_parameters.py similarity index 100% rename from src/DIRAC/Resources/scripts/dirac-resource-get-parameters.py rename to src/DIRAC/Resources/scripts/dirac_resource_get_parameters.py diff --git a/src/DIRAC/Resources/scripts/dirac-resource-info.py b/src/DIRAC/Resources/scripts/dirac_resource_info.py similarity index 100% rename from src/DIRAC/Resources/scripts/dirac-resource-info.py rename to src/DIRAC/Resources/scripts/dirac_resource_info.py diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-file.py b/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-file.py deleted file mode 100755 index d692338660a..00000000000 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-file.py +++ /dev/null @@ -1,108 +0,0 @@ -#! /usr/bin/env python -######################################################################## -# $HeadURL$ -# File : dirac-stager-monitor-file -# Author : Daniela Remenska -######################################################################## -""" -Give monitoring information regarding a staging file uniquely identified with (LFN,SE) - -- status -- last update -- jobs requesting this file to be staged -- SRM requestID -- pin expiry time -- pin length - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__, - 'Usage:', - ' %s LFN SE ...' % Script.scriptName, - 'Arguments:', - ' LFN: LFN of the staging file', - ' SE: Storage Element for the staging file' - ])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -if len(args) < 2: - Script.showHelp() - -from DIRAC import exit as DIRACExit, gLogger - -lfn = args[0] -se = args[1] - -from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient -client = StorageManagerClient() -res = client.getCacheReplicas({'LFN': lfn, 'SE': se}) -if not res['OK']: - gLogger.error(res['Message']) -cacheReplicaInfo = res['Value'] -if cacheReplicaInfo: - replicaID = list(cacheReplicaInfo)[0] - outStr = "\n--------------------" - outStr += "\n%s: %s" % ('LFN'.ljust(8), cacheReplicaInfo[replicaID]['LFN'].ljust(100)) - outStr += "\n%s: %s" % ('SE'.ljust(8), cacheReplicaInfo[replicaID]['SE'].ljust(100)) - outStr += "\n%s: %s" % ('PFN'.ljust(8), cacheReplicaInfo[replicaID]['PFN'].ljust(100)) - outStr += "\n%s: %s" % ('Status'.ljust(8), cacheReplicaInfo[replicaID]['Status'].ljust(100)) - outStr += "\n%s: %s" % ('LastUpdate'.ljust(8), str(cacheReplicaInfo[replicaID]['LastUpdate']).ljust(100)) - outStr += "\n%s: %s" % ('Reason'.ljust(8), str(cacheReplicaInfo[replicaID]['Reason']).ljust(100)) - - resTasks = client.getTasks({'ReplicaID': replicaID}) - - if resTasks['OK']: - # print resTasks['Message'] - outStr += '\nJob IDs requesting this file to be staged:'.ljust(8) - tasks = resTasks['Value'] - for tid in tasks.keys(): - outStr += ' %s ' % (tasks[tid]['SourceTaskID']) - - resStageRequests = client.getStageRequests({'ReplicaID': replicaID}) - - if not resStageRequests['OK']: - gLogger.error(resStageRequests['Message']) - - if resStageRequests['Records']: - stageRequests = resStageRequests['Value'] - outStr += "\n------SRM staging request info--------------" - for info in stageRequests.values(): - outStr += "\n%s: %s" % ('SRM RequestID'.ljust(8), info['RequestID'].ljust(100)) - outStr += "\n%s: %s" % ('SRM StageStatus'.ljust(8), info['StageStatus'].ljust(100)) - outStr += "\n%s: %s" % ('SRM StageRequestSubmitTime'.ljust(8), str(info['StageRequestSubmitTime']).ljust(100)) - outStr += "\n%s: %s" % ('SRM StageRequestCompletedTime'.ljust(8), - str(info['StageRequestCompletedTime']).ljust(100)) - outStr += "\n%s: %s" % ('SRM PinExpiryTime'.ljust(8), str(info['PinExpiryTime']).ljust(100)) - outStr += "\n%s: %s sec" % ('SRM PinLength'.ljust(8), str(info['PinLength']).ljust(100)) - else: - outStr += '\nThere are no staging requests submitted to the site yet.'.ljust(8) -else: - outStr = "\nThere is no such file requested for staging. Check for typo's!" - # Script.showHelp() -gLogger.notice(outStr) - -DIRACExit(0) - -''' Example: -dirac-stager-monitor-file.py /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst GRIDKA-RDST --------------------- -LFN : /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst -SE : GRIDKA-RDST -PFN : srm://gridka-dCache.fzk.de/pnfs/gridka.de/lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst -Status : StageSubmitted -LastUpdate: 2013-06-11 18:13:40 -Reason : None -Jobs requesting this file to be staged: 48518896 -------SRM staging request info-------------- -SRM RequestID: -1768636375 -SRM StageStatus: StageSubmitted -SRM StageRequestSubmitTime: 2013-06-11 18:13:38 -SRM StageRequestCompletedTime: None -SRM PinExpiryTime: None -SRM PinLength: 43200 -''' diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-request.py b/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-request.py deleted file mode 100755 index 4d6c10a8986..00000000000 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-request.py +++ /dev/null @@ -1,61 +0,0 @@ -#! /usr/bin/env python -######################################################################## -# $HeadURL$ -# File : dirac-stager-monitor -# Author : Andrew C. Smith -######################################################################## -""" - Report the summary of the stage task from the DB. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... Request ...' % Script.scriptName, - 'Arguments:', - ' Request: ID of the Stage request in the StorageManager'])) -Script.parseCommandLine(ignoreErrors=False) - -args = Script.getPositionalArgs() - -if not len(args) == 1: - Script.showHelp() - -from DIRAC import exit as DIRACExit, gLogger - -try: - taskID = int(args[0]) -except BaseException: - gLogger.fatal('Stage requestID must be an integer') - DIRACExit(2) - -from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient -client = StorageManagerClient() - -res = client.getTaskSummary(taskID) -if not res['OK']: - gLogger.error(res['Message']) - DIRACExit(2) -taskInfo = res['Value']['TaskInfo'] -replicaInfo = res['Value']['ReplicaInfo'] -outStr = "%s: %s" % ('TaskID'.ljust(20), taskID) -outStr += "\n%s: %s" % ('Status'.ljust(20), taskInfo[taskID]['Status']) -outStr += "\n%s: %s" % ('Source'.ljust(20), taskInfo[taskID]['Source']) -outStr += "\n%s: %s" % ('SourceTaskID'.ljust(20), taskInfo[taskID]['SourceTaskID']) -outStr += "\n%s: %s" % ('CallBackMethod'.ljust(20), taskInfo[taskID]['CallBackMethod']) -outStr += "\n%s: %s" % ('SubmitTime'.ljust(20), taskInfo[taskID]['SubmitTime']) -outStr += "\n%s: %s" % ('CompleteTime'.ljust(20), taskInfo[taskID]['CompleteTime']) -for lfn, metadata in replicaInfo.items(): - outStr += "\n" - outStr += "\n\t%s: %s" % ('LFN'.ljust(8), lfn.ljust(100)) - outStr += "\n\t%s: %s" % ('SE'.ljust(8), metadata['StorageElement'].ljust(100)) - outStr += "\n\t%s: %s" % ('PFN'.ljust(8), str(metadata['PFN']).ljust(100)) - outStr += "\n\t%s: %s" % ('Size'.ljust(8), str(metadata['FileSize']).ljust(100)) - outStr += "\n\t%s: %s" % ('Status'.ljust(8), metadata['Status'].ljust(100)) - outStr += "\n\t%s: %s" % ('Reason'.ljust(8), str(metadata['Reason']).ljust(100)) -gLogger.notice(outStr) diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-requests.py b/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-requests.py deleted file mode 100755 index f8a91e5f729..00000000000 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-requests.py +++ /dev/null @@ -1,183 +0,0 @@ -#! /usr/bin/env python -######################################################################## -# $HeadURL$ -# File : dirac-stager-monitor-requests -# Author : Daniela Remenska -######################################################################## -""" - Report the details of file staging requests, based on selection filters -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -_RCSID__ = "$Id$" -from DIRAC.Core.Base import Script -from DIRAC import gConfig, gLogger, exit as DIRACExit, S_OK, version - -subLogger = None - -Script.setUsageMessage( - '\n'.join( - [ - __doc__.split('\n')[1], - 'Usage:', - ' %s [--status=] [--se=] [--limit=] [--showJobs=YES] ...' % - Script.scriptName, - 'Arguments:', - ' status: file status=(New, Offline, Waiting, Failed, StageSubmitted, Staged).', - ' se: storage element', - ' showJobs: whether to ALSO list the jobs asking for these files to be staged', - ' WARNING: Query may be heavy, please use --limit switch!'])) - - -def registerSwitches(): - ''' - Registers all switches that can be used while calling the script from the - command line interface. - ''' - - switches = (('status=', 'Filter per file status=(New, Offline, Waiting, Failed, StageSubmitted, Staged).' - '\n If not used, all status values will be taken into account'), - ('se=', 'Filter per Storage Element. If not used, all storage elements will be taken into account.'), - ('limit=', 'Limit the number of entries returned.'), - ('showJobs=', 'Whether to ALSO list the jobs asking for these files to be staged'), - ) - - for switch in switches: - Script.registerSwitch('', switch[0], switch[1]) - - -def parseSwitches(): - ''' - Parses the arguments passed by the user - ''' - - Script.parseCommandLine(ignoreErrors=True) - args = Script.getPositionalArgs() - if args: - subLogger.error("Found the following positional args '%s', but we only accept switches" % args) - subLogger.error("Please, check documentation below") - Script.showHelp(exitCode=1) - - switches = dict(Script.getUnprocessedSwitches()) - - for key in ('status', 'se', 'limit'): - if key not in switches: - subLogger.warn("You're not using switch --%s, query may take long!" % key) - - if 'status' in switches and switches['status'] not in ( - 'New', 'Offline', 'Waiting', 'Failed', 'StageSubmitted', 'Staged'): - subLogger.error("Found \"%s\" as Status value. Incorrect value used!" % switches['status']) - subLogger.error("Please, check documentation below") - Script.showHelp(exitCode=1) - - subLogger.debug("The switches used are:") - map(subLogger.debug, switches.items()) - - return switches - -# ............................................................................... - - -def run(): - - from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient - client = StorageManagerClient() - queryDict = {} - - if 'status' in switchDict: - queryDict['Status'] = str(switchDict['status']) - - if 'se' in switchDict: - queryDict['SE'] = str(switchDict['se']) - - # weird: if there are no switches (dictionary is empty), then the --limit is ignored!! - # must FIX that in StorageManagementDB.py! - # ugly fix: - newer = '1903-08-02 06:24:38' # select newer than - if 'limit' in switchDict: - gLogger.notice("Query limited to %s entries" % switchDict['limit']) - res = client.getCacheReplicas(queryDict, None, newer, None, None, int(switchDict['limit'])) - else: - res = client.getCacheReplicas(queryDict) - - if not res['OK']: - gLogger.error(res['Message']) - outStr = "\n" - if res['Records']: - replicas = res['Value'] - outStr += " %s" % ("Status".ljust(15)) - outStr += " %s" % ("LastUpdate".ljust(20)) - outStr += " %s" % ("LFN".ljust(80)) - outStr += " %s" % ("SE".ljust(10)) - outStr += " %s" % ("Reason".ljust(10)) - if 'showJobs' in switchDict: - outStr += " %s" % ("Jobs".ljust(10)) - outStr += " %s" % ("PinExpiryTime".ljust(15)) - outStr += " %s" % ("PinLength(sec)".ljust(15)) - outStr += "\n" - - for crid, info in replicas.items(): - outStr += " %s" % (info['Status'].ljust(15)) - outStr += " %s" % (str(info['LastUpdate']).ljust(20)) - outStr += " %s" % (info['LFN'].ljust(30)) - outStr += " %s" % (info['SE'].ljust(15)) - outStr += " %s" % (str(info['Reason']).ljust(10)) - - # Task info - if 'showJobs' in switchDict: - resTasks = client.getTasks({'ReplicaID': crid}) - if resTasks['OK']: - if resTasks['Value']: - tasks = resTasks['Value'] - jobs = [] - for tid in tasks: - jobs.append(tasks[tid]['SourceTaskID']) - outStr += ' %s ' % (str(jobs).ljust(10)) - else: - outStr += ' %s ' % (" --- ".ljust(10)) - # Stage request info - # what if there's no request to the site yet? - resStageRequests = client.getStageRequests({'ReplicaID': crid}) - if not resStageRequests['OK']: - gLogger.error(resStageRequests['Message']) - if resStageRequests['Records']: - stageRequests = resStageRequests['Value'] - for info in stageRequests.values(): - outStr += " %s" % (str(info['PinExpiryTime']).ljust(20)) - outStr += " %s" % (str(info['PinLength']).ljust(10)) - outStr += "\n" - - gLogger.notice(outStr) - else: - gLogger.notice("No entries") - - -if __name__ == "__main__": - - subLogger = gLogger.getSubLogger(__file__) - - registerSwitches() - - switchDict = parseSwitches() - run() - - # Bye - DIRACExit(0) -''' Example: -dirac-stager-show-requests.py --status=Staged --se=GRIDKA-RDST --limit=10 --showJobs=YES -Query limited to 10 entries - - Status LastUpdate LFN SE Reason Jobs PinExpiryTime PinLength - Staged 2013-06-05 20:10:50 /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00054816_1.full.dst GRIDKA-RDST None ['48498752'] 2013-06-05 22:10:50 86400 - Staged 2013-06-06 15:54:29 /lhcb/LHCb/Collision12/FULL.DST/00020846/0001/00020846_00013202_1.full.dst GRIDKA-RDST None ['48516851'] 2013-06-06 16:54:29 43200 - Staged 2013-06-07 02:35:41 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032726_1.full.dst GRIDKA-RDST None ['48520736'] 2013-06-07 03:35:41 43200 - Staged 2013-06-06 04:16:50 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00030567_1.full.dst GRIDKA-RDST None ['48510852'] 2013-06-06 06:16:50 86400 - Staged 2013-06-07 03:44:04 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032699_1.full.dst GRIDKA-RDST None ['48520737'] 2013-06-07 04:44:04 43200 - Staged 2013-06-05 23:37:46 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032576_1.full.dst GRIDKA-RDST None ['48508687'] 2013-06-06 01:37:46 86400 - Staged 2013-06-10 08:50:09 /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056424_1.full.dst GRIDKA-RDST None ['48518896'] 2013-06-10 09:50:09 43200 - Staged 2013-06-06 11:03:25 /lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00022161_1.full.dst GRIDKA-RDST None ['48515583'] 2013-06-06 12:03:25 43200 - Staged 2013-06-06 11:11:50 /lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00029215_1.full.dst GRIDKA-RDST None ['48515072'] 2013-06-06 12:11:50 43200 - Staged 2013-06-07 03:19:26 /lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00022323_1.full.dst GRIDKA-RDST None ['48515600'] 2013-06-07 04:19:26 43200 - ''' -################################################################################ diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-show-stats.py b/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-show-stats.py deleted file mode 100755 index fdaf34bbddc..00000000000 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-show-stats.py +++ /dev/null @@ -1,75 +0,0 @@ -#! /usr/bin/env python -######################################################################## -# $HeadURL$ -# File : dirac-stager-show-stats -# Author : Daniela Remenska -######################################################################## -""" -Reports breakdown of file(s) number/size in different staging states across Storage Elements. -Currently used Cache per SE is also reported. (active pins) - -Example: - - dirac-stager-show-stats - - Status SE NumberOfFiles Size(GB) - -------------------------------------------------------------------------- - Staged GRIDKA-RDST 1 4.5535 - StageSubmitted GRIDKA-RDST 5 22.586 - Waiting PIC-RDST 3 13.6478 - - WARNING: the Size for files with Status=New is not yet determined at the point of selection! - - --------------------- current status of the SE Caches from the DB----------- - GRIDKA-RDST : 6 replicas with a size of 29.141 GB. - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" -from DIRAC.Core.Base import Script -from DIRAC import gConfig, gLogger, exit as DIRACExit, S_OK, version - -Script.setUsageMessage(__doc__) -Script.parseCommandLine(ignoreErrors=False) -from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient -client = StorageManagerClient() - -res = client.getCacheReplicasSummary() -if not res['OK']: - gLogger.fatal(res['Message']) - DIRACExit(2) -stagerInfo = res['Value'] -outStr = "\n" -outStr += " %s" % ("Status".ljust(20)) -outStr += " %s" % ("SE".ljust(20)) -outStr += " %s" % ("NumberOfFiles".ljust(20)) -outStr += " %s" % ("Size(GB)".ljust(20)) -outStr += " \n--------------------------------------------------------------------------\n" -if stagerInfo: - for info in stagerInfo.values(): - outStr += " %s" % (info['Status'].ljust(20)) - outStr += " %s" % (info['SE'].ljust(20)) - outStr += " %s" % (str(info['NumFiles']).ljust(20)) - outStr += " %s\n" % (str(info['SumFiles']).ljust(20)) -else: - outStr += " %s" % ("Nothing to see here...Bye") -outStr += " %s" % ("\nWARNING: the Size for files with Status=New is not yet determined at the point of selection!\n") -outStr += "--------------------- current status of the SE Caches from the DB-----------" -res = client.getSubmittedStagePins() -if not res['OK']: - gLogger.fatal(res['Message']) - DIRACExit(2) -storageElementUsage = res['Value'] -if storageElementUsage: - for storageElement in storageElementUsage.keys(): - seDict = storageElementUsage[storageElement] - seDict['TotalSize'] = int(seDict['TotalSize'] / (1000 * 1000 * 1000.0)) - outStr += " \n %s: %s replicas with a size of %.3f GB." % ( - storageElement.ljust(15), str(seDict['Replicas']).rjust(6), seDict['TotalSize']) -else: - outStr += " %s" % "\nStageRequest.getStorageUsage: No active stage/pin requests found." -gLogger.notice(outStr) -DIRACExit(0) diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-stage-files.py b/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-stage-files.py deleted file mode 100755 index 443443d06a0..00000000000 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-stage-files.py +++ /dev/null @@ -1,81 +0,0 @@ -#! /usr/bin/env python -######################################################################## -# $HeadURL$ -# File : dirac-stager-stage-files -# Author : Daniela Remenska -######################################################################## -""" -- submit staging requests for a particular Storage Element! Default DIRAC JobID will be =0. - (not visible in the Job monitoring list though) - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" -from DIRAC.Core.Base import Script -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s ' % Script.scriptName, - 'Arguments:', - ' : LFN to Stage (or local file with list of LFNs)', - ' : Name of Storage Element'])) - -Script.parseCommandLine(ignoreErrors=True) - -args = Script.getPositionalArgs() - -if len(args) < 2: - Script.showHelp() - -seName = args[1] -fileName = args[0] - -import os -from DIRAC import exit as DIRACExit, gLogger -from DIRAC.Interfaces.API.Dirac import Dirac -from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient - -stageLfns = {} - -if os.path.exists(fileName): - try: - lfnFile = open(fileName) - lfns = [k.strip() for k in lfnFile.readlines()] - lfnFile.close() - except Exception: - gLogger.exception('Can not open file', fileName) - DIRACExit(-1) -else: - lfns = args[:len(args) - 1] - -stageLfns[seName] = lfns -stagerClient = StorageManagerClient() - -res = stagerClient.setRequest(stageLfns, 'WorkloadManagement', - 'updateJobFromStager@WorkloadManagement/JobStateUpdate', - 0) # fake JobID = 0 -if not res['OK']: - gLogger.error(res['Message']) - DIRACExit(-1) -else: - gLogger.notice("Stage request submitted for LFNs:\n %s" % lfns) - gLogger.notice("SE= %s" % seName) - gLogger.notice("You can check their status and progress with dirac-stager-monitor-file ") - -'''Example1: -dirac-stager-stage-files.py filesToStage.txt GRIDKA-RDST -Stage request submitted for LFNs: - ['/lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00023458_1.full.dst', '/lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032669_1.full.dst', '/lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032666_1.full.dst'] -SE= GRIDKA-RDST -You can check their status and progress with dirac-stager-monitor-file - -Example2: -dirac-stager-stage-files.py /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032641_1.full.dst GRIDKA-RDST -Stage request submitted for LFNs: - ['/lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032641_1.full.dst'] -SE= GRIDKA-RDST -You can check their status and progress with dirac-stager-monitor-file -''' - -DIRACExit() diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_file.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_file.py new file mode 100755 index 00000000000..92c6bcffd2f --- /dev/null +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_file.py @@ -0,0 +1,117 @@ +#! /usr/bin/env python +######################################################################## +# $HeadURL$ +# File : dirac-stager-monitor-file +# Author : Daniela Remenska +######################################################################## +""" +Give monitoring information regarding a staging file uniquely identified with (LFN,SE) + +- status +- last update +- jobs requesting this file to be staged +- SRM requestID +- pin expiry time +- pin length + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__, + 'Usage:', + ' %s LFN SE ...' % Script.scriptName, + 'Arguments:', + ' LFN: LFN of the staging file', + ' SE: Storage Element for the staging file' + ])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + if len(args) < 2: + Script.showHelp() + + from DIRAC import exit as DIRACExit, gLogger + + lfn = args[0] + se = args[1] + + from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + client = StorageManagerClient() + res = client.getCacheReplicas({'LFN': lfn, 'SE': se}) + if not res['OK']: + gLogger.error(res['Message']) + cacheReplicaInfo = res['Value'] + if cacheReplicaInfo: + replicaID = list(cacheReplicaInfo)[0] + outStr = "\n--------------------" + outStr += "\n%s: %s" % ('LFN'.ljust(8), cacheReplicaInfo[replicaID]['LFN'].ljust(100)) + outStr += "\n%s: %s" % ('SE'.ljust(8), cacheReplicaInfo[replicaID]['SE'].ljust(100)) + outStr += "\n%s: %s" % ('PFN'.ljust(8), cacheReplicaInfo[replicaID]['PFN'].ljust(100)) + outStr += "\n%s: %s" % ('Status'.ljust(8), cacheReplicaInfo[replicaID]['Status'].ljust(100)) + outStr += "\n%s: %s" % ('LastUpdate'.ljust(8), str(cacheReplicaInfo[replicaID]['LastUpdate']).ljust(100)) + outStr += "\n%s: %s" % ('Reason'.ljust(8), str(cacheReplicaInfo[replicaID]['Reason']).ljust(100)) + + resTasks = client.getTasks({'ReplicaID': replicaID}) + + if resTasks['OK']: + # print resTasks['Message'] + outStr += '\nJob IDs requesting this file to be staged:'.ljust(8) + tasks = resTasks['Value'] + for tid in tasks.keys(): + outStr += ' %s ' % (tasks[tid]['SourceTaskID']) + + resStageRequests = client.getStageRequests({'ReplicaID': replicaID}) + + if not resStageRequests['OK']: + gLogger.error(resStageRequests['Message']) + + if resStageRequests['Records']: + stageRequests = resStageRequests['Value'] + outStr += "\n------SRM staging request info--------------" + for info in stageRequests.values(): + outStr += "\n%s: %s" % ('SRM RequestID'.ljust(8), info['RequestID'].ljust(100)) + outStr += "\n%s: %s" % ('SRM StageStatus'.ljust(8), info['StageStatus'].ljust(100)) + outStr += "\n%s: %s" % ('SRM StageRequestSubmitTime'.ljust(8), str(info['StageRequestSubmitTime']).ljust(100)) + outStr += "\n%s: %s" % ('SRM StageRequestCompletedTime'.ljust(8), + str(info['StageRequestCompletedTime']).ljust(100)) + outStr += "\n%s: %s" % ('SRM PinExpiryTime'.ljust(8), str(info['PinExpiryTime']).ljust(100)) + outStr += "\n%s: %s sec" % ('SRM PinLength'.ljust(8), str(info['PinLength']).ljust(100)) + else: + outStr += '\nThere are no staging requests submitted to the site yet.'.ljust(8) + else: + outStr = "\nThere is no such file requested for staging. Check for typo's!" + # Script.showHelp() + gLogger.notice(outStr) + + DIRACExit(0) + + +if __name__ == "__main__": + main() + + +''' Example: +dirac-stager-monitor-file.py /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst GRIDKA-RDST +-------------------- +LFN : /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst +SE : GRIDKA-RDST +PFN : srm://gridka-dCache.fzk.de/pnfs/gridka.de/lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst +Status : StageSubmitted +LastUpdate: 2013-06-11 18:13:40 +Reason : None +Jobs requesting this file to be staged: 48518896 +------SRM staging request info-------------- +SRM RequestID: -1768636375 +SRM StageStatus: StageSubmitted +SRM StageRequestSubmitTime: 2013-06-11 18:13:38 +SRM StageRequestCompletedTime: None +SRM PinExpiryTime: None +SRM PinLength: 43200 +''' # NOQA diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-jobs.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_jobs.py similarity index 58% rename from src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-jobs.py rename to src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_jobs.py index 973a19dbd90..11b102977e0 100755 --- a/src/DIRAC/StorageManagementSystem/scripts/dirac-stager-monitor-jobs.py +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_jobs.py @@ -12,62 +12,71 @@ from __future__ import print_function __RCSID__ = "$Id$" from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s jobID [jobID] [jobID] ...' % Script.scriptName, - 'Arguments:', - ' JobID: DIRAC job ID \n' - ])) -Script.parseCommandLine(ignoreErrors=False) -args = Script.getPositionalArgs() +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s jobID [jobID] [jobID] ...' % Script.scriptName, + 'Arguments:', + ' JobID: DIRAC job ID \n' + ])) + Script.parseCommandLine(ignoreErrors=False) -if len(args) < 1: - Script.showHelp() + args = Script.getPositionalArgs() -from DIRAC import exit as DIRACExit, gLogger + if len(args) < 1: + Script.showHelp() -try: - jobIDs = [int(arg) for arg in args] -except BaseException: - gLogger.fatal('DIRAC Job IDs must be integers') - DIRACExit(2) + from DIRAC import exit as DIRACExit, gLogger -from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient -client = StorageManagerClient() + try: + jobIDs = [int(arg) for arg in args] + except BaseException: + gLogger.fatal('DIRAC Job IDs must be integers') + DIRACExit(2) + + from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + client = StorageManagerClient() + + outStr = "\n" + for jobID in jobIDs: + res = client.getTaskSummary(jobID) + if not res['OK']: + gLogger.error(res['Message']) + continue + if not res['Value']: + gLogger.notice('No info for job %s, probably gone from the stager...' % jobID) + continue + taskInfo = res['Value']['TaskInfo'] + replicaInfo = res['Value']['ReplicaInfo'] + outStr = "%s: %s" % ('JobID'.ljust(20), jobID) + outStr += "\n%s: %s" % ('Status'.ljust(20), taskInfo[str(jobID)]['Status']) + outStr += "\n%s: %s" % ('SubmitTime'.ljust(20), taskInfo[str(jobID)]['SubmitTime']) + outStr += "\n%s: %s" % ('CompleteTime'.ljust(20), taskInfo[str(jobID)]['CompleteTime']) + outStr += "\nStaging files for this job:" + if not res['Value']['ReplicaInfo']: + gLogger.notice('No info on files for the job = %s, that is odd' % jobID) + continue + else: + for lfn, metadata in replicaInfo.items(): + outStr += "\n\t--------------------" + outStr += "\n\t%s: %s" % ('LFN'.ljust(8), lfn.ljust(100)) + outStr += "\n\t%s: %s" % ('SE'.ljust(8), metadata['StorageElement'].ljust(100)) + outStr += "\n\t%s: %s" % ('PFN'.ljust(8), str(metadata['PFN']).ljust(100)) + outStr += "\n\t%s: %s" % ('Status'.ljust(8), metadata['Status'].ljust(100)) + outStr += "\n\t%s: %s" % ('Reason'.ljust(8), str(metadata['Reason']).ljust(100)) + outStr += "\n%s: %s" % ('LastUpdate'.ljust(8), str(metadata['LastUpdate']).ljust(100)) + outStr += "\n----------------------" + gLogger.notice(outStr) + DIRACExit(0) + + +if __name__ == "__main__": + main() -outStr = "\n" -for jobID in jobIDs: - res = client.getTaskSummary(jobID) - if not res['OK']: - gLogger.error(res['Message']) - continue - if not res['Value']: - gLogger.notice('No info for job %s, probably gone from the stager...' % jobID) - continue - taskInfo = res['Value']['TaskInfo'] - replicaInfo = res['Value']['ReplicaInfo'] - outStr = "%s: %s" % ('JobID'.ljust(20), jobID) - outStr += "\n%s: %s" % ('Status'.ljust(20), taskInfo[str(jobID)]['Status']) - outStr += "\n%s: %s" % ('SubmitTime'.ljust(20), taskInfo[str(jobID)]['SubmitTime']) - outStr += "\n%s: %s" % ('CompleteTime'.ljust(20), taskInfo[str(jobID)]['CompleteTime']) - outStr += "\nStaging files for this job:" - if not res['Value']['ReplicaInfo']: - gLogger.notice('No info on files for the job = %s, that is odd' % jobID) - continue - else: - for lfn, metadata in replicaInfo.items(): - outStr += "\n\t--------------------" - outStr += "\n\t%s: %s" % ('LFN'.ljust(8), lfn.ljust(100)) - outStr += "\n\t%s: %s" % ('SE'.ljust(8), metadata['StorageElement'].ljust(100)) - outStr += "\n\t%s: %s" % ('PFN'.ljust(8), str(metadata['PFN']).ljust(100)) - outStr += "\n\t%s: %s" % ('Status'.ljust(8), metadata['Status'].ljust(100)) - outStr += "\n\t%s: %s" % ('Reason'.ljust(8), str(metadata['Reason']).ljust(100)) - outStr += "\n%s: %s" % ('LastUpdate'.ljust(8), str(metadata['LastUpdate']).ljust(100)) - outStr += "\n----------------------" - gLogger.notice(outStr) -DIRACExit(0) ''' Example: dirac-stager-monitor-jobs.py 5688643 5688644 @@ -138,5 +147,4 @@ Status : Offline Reason : None -------------------- - -''' +''' # NOQA diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_request.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_request.py new file mode 100755 index 00000000000..d9d46033334 --- /dev/null +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_request.py @@ -0,0 +1,69 @@ +#! /usr/bin/env python +######################################################################## +# $HeadURL$ +# File : dirac-stager-monitor +# Author : Andrew C. Smith +######################################################################## +""" + Report the summary of the stage task from the DB. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... Request ...' % Script.scriptName, + 'Arguments:', + ' Request: ID of the Stage request in the StorageManager'])) + Script.parseCommandLine(ignoreErrors=False) + + args = Script.getPositionalArgs() + + if not len(args) == 1: + Script.showHelp() + + from DIRAC import exit as DIRACExit, gLogger + + try: + taskID = int(args[0]) + except BaseException: + gLogger.fatal('Stage requestID must be an integer') + DIRACExit(2) + + from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + client = StorageManagerClient() + + res = client.getTaskSummary(taskID) + if not res['OK']: + gLogger.error(res['Message']) + DIRACExit(2) + taskInfo = res['Value']['TaskInfo'] + replicaInfo = res['Value']['ReplicaInfo'] + outStr = "%s: %s" % ('TaskID'.ljust(20), taskID) + outStr += "\n%s: %s" % ('Status'.ljust(20), taskInfo[taskID]['Status']) + outStr += "\n%s: %s" % ('Source'.ljust(20), taskInfo[taskID]['Source']) + outStr += "\n%s: %s" % ('SourceTaskID'.ljust(20), taskInfo[taskID]['SourceTaskID']) + outStr += "\n%s: %s" % ('CallBackMethod'.ljust(20), taskInfo[taskID]['CallBackMethod']) + outStr += "\n%s: %s" % ('SubmitTime'.ljust(20), taskInfo[taskID]['SubmitTime']) + outStr += "\n%s: %s" % ('CompleteTime'.ljust(20), taskInfo[taskID]['CompleteTime']) + for lfn, metadata in replicaInfo.items(): + outStr += "\n" + outStr += "\n\t%s: %s" % ('LFN'.ljust(8), lfn.ljust(100)) + outStr += "\n\t%s: %s" % ('SE'.ljust(8), metadata['StorageElement'].ljust(100)) + outStr += "\n\t%s: %s" % ('PFN'.ljust(8), str(metadata['PFN']).ljust(100)) + outStr += "\n\t%s: %s" % ('Size'.ljust(8), str(metadata['FileSize']).ljust(100)) + outStr += "\n\t%s: %s" % ('Status'.ljust(8), metadata['Status'].ljust(100)) + outStr += "\n\t%s: %s" % ('Reason'.ljust(8), str(metadata['Reason']).ljust(100)) + gLogger.notice(outStr) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py new file mode 100755 index 00000000000..840109942bb --- /dev/null +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_monitor_requests.py @@ -0,0 +1,187 @@ +#! /usr/bin/env python +######################################################################## +# $HeadURL$ +# File : dirac-stager-monitor-requests +# Author : Daniela Remenska +######################################################################## +""" + Report the details of file staging requests, based on selection filters +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +_RCSID__ = "$Id$" +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC import gConfig, gLogger, exit as DIRACExit, S_OK, version + +subLogger = None + + +@DIRACScript() +def main(): + Script.setUsageMessage( + '\n'.join( + [ + __doc__.split('\n')[1], + 'Usage:', + ' %s [--status=] [--se=] [--limit=] [--showJobs=YES] ...' % + Script.scriptName, + 'Arguments:', + ' status: file status=(New, Offline, Waiting, Failed, StageSubmitted, Staged).', + ' se: storage element', + ' showJobs: whether to ALSO list the jobs asking for these files to be staged', + ' WARNING: Query may be heavy, please use --limit switch!'])) + + def registerSwitches(): + ''' + Registers all switches that can be used while calling the script from the + command line interface. + ''' + + switches = (('status=', 'Filter per file status=(New, Offline, Waiting, Failed, StageSubmitted, Staged).' + '\n If not used, all status values will be taken into account'), + ('se=', 'Filter per Storage Element. If not used, all storage elements will be taken into account.'), + ('limit=', 'Limit the number of entries returned.'), + ('showJobs=', 'Whether to ALSO list the jobs asking for these files to be staged'), + ) + + for switch in switches: + Script.registerSwitch('', switch[0], switch[1]) + + def parseSwitches(): + ''' + Parses the arguments passed by the user + ''' + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + if args: + subLogger.error("Found the following positional args '%s', but we only accept switches" % args) + subLogger.error("Please, check documentation below") + Script.showHelp(exitCode=1) + + switches = dict(Script.getUnprocessedSwitches()) + + for key in ('status', 'se', 'limit'): + if key not in switches: + subLogger.warn("You're not using switch --%s, query may take long!" % key) + + if 'status' in switches and switches['status'] not in ( + 'New', 'Offline', 'Waiting', 'Failed', 'StageSubmitted', 'Staged'): + subLogger.error("Found \"%s\" as Status value. Incorrect value used!" % switches['status']) + subLogger.error("Please, check documentation below") + Script.showHelp(exitCode=1) + + subLogger.debug("The switches used are:") + map(subLogger.debug, switches.items()) + + return switches + + # ............................................................................... + + def run(): + global subLogger + + from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + client = StorageManagerClient() + queryDict = {} + + if 'status' in switchDict: + queryDict['Status'] = str(switchDict['status']) + + if 'se' in switchDict: + queryDict['SE'] = str(switchDict['se']) + + # weird: if there are no switches (dictionary is empty), then the --limit is ignored!! + # must FIX that in StorageManagementDB.py! + # ugly fix: + newer = '1903-08-02 06:24:38' # select newer than + if 'limit' in switchDict: + gLogger.notice("Query limited to %s entries" % switchDict['limit']) + res = client.getCacheReplicas(queryDict, None, newer, None, None, int(switchDict['limit'])) + else: + res = client.getCacheReplicas(queryDict) + + if not res['OK']: + gLogger.error(res['Message']) + outStr = "\n" + if res['Records']: + replicas = res['Value'] + outStr += " %s" % ("Status".ljust(15)) + outStr += " %s" % ("LastUpdate".ljust(20)) + outStr += " %s" % ("LFN".ljust(80)) + outStr += " %s" % ("SE".ljust(10)) + outStr += " %s" % ("Reason".ljust(10)) + if 'showJobs' in switchDict: + outStr += " %s" % ("Jobs".ljust(10)) + outStr += " %s" % ("PinExpiryTime".ljust(15)) + outStr += " %s" % ("PinLength(sec)".ljust(15)) + outStr += "\n" + + for crid, info in replicas.items(): + outStr += " %s" % (info['Status'].ljust(15)) + outStr += " %s" % (str(info['LastUpdate']).ljust(20)) + outStr += " %s" % (info['LFN'].ljust(30)) + outStr += " %s" % (info['SE'].ljust(15)) + outStr += " %s" % (str(info['Reason']).ljust(10)) + + # Task info + if 'showJobs' in switchDict: + resTasks = client.getTasks({'ReplicaID': crid}) + if resTasks['OK']: + if resTasks['Value']: + tasks = resTasks['Value'] + jobs = [] + for tid in tasks: + jobs.append(tasks[tid]['SourceTaskID']) + outStr += ' %s ' % (str(jobs).ljust(10)) + else: + outStr += ' %s ' % (" --- ".ljust(10)) + # Stage request info + # what if there's no request to the site yet? + resStageRequests = client.getStageRequests({'ReplicaID': crid}) + if not resStageRequests['OK']: + gLogger.error(resStageRequests['Message']) + if resStageRequests['Records']: + stageRequests = resStageRequests['Value'] + for info in stageRequests.values(): + outStr += " %s" % (str(info['PinExpiryTime']).ljust(20)) + outStr += " %s" % (str(info['PinLength']).ljust(10)) + outStr += "\n" + + gLogger.notice(outStr) + else: + gLogger.notice("No entries") + + subLogger = gLogger.getSubLogger(__file__) + + registerSwitches() + + switchDict = parseSwitches() + run() + + # Bye + DIRACExit(0) + + +if __name__ == "__main__": + main() + + +''' Example: +dirac-stager-show-requests.py --status=Staged --se=GRIDKA-RDST --limit=10 --showJobs=YES +Query limited to 10 entries + +Status LastUpdate LFN SE Reason Jobs PinExpiryTime PinLength +Staged 2013-06-05 20:10:50 /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00054816_1.full.dst GRIDKA-RDST None ['48498752'] 2013-06-05 22:10:50 86400 +Staged 2013-06-06 15:54:29 /lhcb/LHCb/Collision12/FULL.DST/00020846/0001/00020846_00013202_1.full.dst GRIDKA-RDST None ['48516851'] 2013-06-06 16:54:29 43200 +Staged 2013-06-07 02:35:41 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032726_1.full.dst GRIDKA-RDST None ['48520736'] 2013-06-07 03:35:41 43200 +Staged 2013-06-06 04:16:50 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00030567_1.full.dst GRIDKA-RDST None ['48510852'] 2013-06-06 06:16:50 86400 +Staged 2013-06-07 03:44:04 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032699_1.full.dst GRIDKA-RDST None ['48520737'] 2013-06-07 04:44:04 43200 +Staged 2013-06-05 23:37:46 /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032576_1.full.dst GRIDKA-RDST None ['48508687'] 2013-06-06 01:37:46 86400 +Staged 2013-06-10 08:50:09 /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056424_1.full.dst GRIDKA-RDST None ['48518896'] 2013-06-10 09:50:09 43200 +Staged 2013-06-06 11:03:25 /lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00022161_1.full.dst GRIDKA-RDST None ['48515583'] 2013-06-06 12:03:25 43200 +Staged 2013-06-06 11:11:50 /lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00029215_1.full.dst GRIDKA-RDST None ['48515072'] 2013-06-06 12:11:50 43200 +Staged 2013-06-07 03:19:26 /lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00022323_1.full.dst GRIDKA-RDST None ['48515600'] 2013-06-07 04:19:26 43200 +''' # NOQA diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_show_stats.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_show_stats.py new file mode 100755 index 00000000000..008fd9eb98c --- /dev/null +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_show_stats.py @@ -0,0 +1,83 @@ +#! /usr/bin/env python +######################################################################## +# $HeadURL$ +# File : dirac-stager-show-stats +# Author : Daniela Remenska +######################################################################## +""" +Reports breakdown of file(s) number/size in different staging states across Storage Elements. +Currently used Cache per SE is also reported. (active pins) + +Example: + + dirac-stager-show-stats + + Status SE NumberOfFiles Size(GB) + -------------------------------------------------------------------------- + Staged GRIDKA-RDST 1 4.5535 + StageSubmitted GRIDKA-RDST 5 22.586 + Waiting PIC-RDST 3 13.6478 + + WARNING: the Size for files with Status=New is not yet determined at the point of selection! + + --------------------- current status of the SE Caches from the DB----------- + GRIDKA-RDST : 6 replicas with a size of 29.141 GB. + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC import gConfig, gLogger, exit as DIRACExit, S_OK, version + + +@DIRACScript() +def main(): + Script.setUsageMessage(__doc__) + Script.parseCommandLine(ignoreErrors=False) + from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + client = StorageManagerClient() + + res = client.getCacheReplicasSummary() + if not res['OK']: + gLogger.fatal(res['Message']) + DIRACExit(2) + stagerInfo = res['Value'] + outStr = "\n" + outStr += " %s" % ("Status".ljust(20)) + outStr += " %s" % ("SE".ljust(20)) + outStr += " %s" % ("NumberOfFiles".ljust(20)) + outStr += " %s" % ("Size(GB)".ljust(20)) + outStr += " \n--------------------------------------------------------------------------\n" + if stagerInfo: + for info in stagerInfo.values(): + outStr += " %s" % (info['Status'].ljust(20)) + outStr += " %s" % (info['SE'].ljust(20)) + outStr += " %s" % (str(info['NumFiles']).ljust(20)) + outStr += " %s\n" % (str(info['SumFiles']).ljust(20)) + else: + outStr += " %s" % ("Nothing to see here...Bye") + outStr += " \nWARNING: the Size for files with Status=New is not yet determined at the point of selection!\n" + outStr += "--------------------- current status of the SE Caches from the DB-----------" + res = client.getSubmittedStagePins() + if not res['OK']: + gLogger.fatal(res['Message']) + DIRACExit(2) + storageElementUsage = res['Value'] + if storageElementUsage: + for storageElement in storageElementUsage.keys(): + seDict = storageElementUsage[storageElement] + seDict['TotalSize'] = int(seDict['TotalSize'] / (1000 * 1000 * 1000.0)) + outStr += " \n %s: %s replicas with a size of %.3f GB." % ( + storageElement.ljust(15), str(seDict['Replicas']).rjust(6), seDict['TotalSize']) + else: + outStr += " %s" % "\nStageRequest.getStorageUsage: No active stage/pin requests found." + gLogger.notice(outStr) + DIRACExit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_stage_files.py b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_stage_files.py new file mode 100755 index 00000000000..77a19c0d04d --- /dev/null +++ b/src/DIRAC/StorageManagementSystem/scripts/dirac_stager_stage_files.py @@ -0,0 +1,91 @@ +#! /usr/bin/env python +######################################################################## +# $HeadURL$ +# File : dirac-stager-stage-files +# Author : Daniela Remenska +######################################################################## +""" +- submit staging requests for a particular Storage Element! Default DIRAC JobID will be =0. + (not visible in the Job monitoring list though) + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s ' % Script.scriptName, + 'Arguments:', + ' : LFN to Stage (or local file with list of LFNs)', + ' : Name of Storage Element'])) + + Script.parseCommandLine(ignoreErrors=True) + + args = Script.getPositionalArgs() + + if len(args) < 2: + Script.showHelp() + + seName = args[1] + fileName = args[0] + + import os + from DIRAC import exit as DIRACExit, gLogger + from DIRAC.Interfaces.API.Dirac import Dirac + from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient + + stageLfns = {} + + if os.path.exists(fileName): + try: + lfnFile = open(fileName) + lfns = [k.strip() for k in lfnFile.readlines()] + lfnFile.close() + except Exception: + gLogger.exception('Can not open file', fileName) + DIRACExit(-1) + else: + lfns = args[:len(args) - 1] + + stageLfns[seName] = lfns + stagerClient = StorageManagerClient() + + res = stagerClient.setRequest(stageLfns, 'WorkloadManagement', + 'updateJobFromStager@WorkloadManagement/JobStateUpdate', + 0) # fake JobID = 0 + if not res['OK']: + gLogger.error(res['Message']) + DIRACExit(-1) + else: + gLogger.notice("Stage request submitted for LFNs:\n %s" % lfns) + gLogger.notice("SE= %s" % seName) + gLogger.notice("You can check their status and progress with dirac-stager-monitor-file ") + + DIRACExit() + + +if __name__ == "__main__": + main() + + +'''Example1: +dirac-stager-stage-files.py filesToStage.txt GRIDKA-RDST +Stage request submitted for LFNs: +['/lhcb/LHCb/Collision12/FULL.DST/00020846/0002/00020846_00023458_1.full.dst', '/lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032669_1.full.dst', '/lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032666_1.full.dst'] +SE= GRIDKA-RDST +You can check their status and progress with dirac-stager-monitor-file + +Example2: +dirac-stager-stage-files.py /lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032641_1.full.dst GRIDKA-RDST +Stage request submitted for LFNs: +['/lhcb/LHCb/Collision12/FULL.DST/00020846/0003/00020846_00032641_1.full.dst'] +SE= GRIDKA-RDST +You can check their status and progress with dirac-stager-monitor-file +''' # NOQA diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-add-files.py b/src/DIRAC/TransformationSystem/scripts/dirac-transformation-add-files.py deleted file mode 100755 index 1465acf2029..00000000000 --- a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-add-files.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python - -""" - Add files to an existing transformation -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import os -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s TransID ' % Script.scriptName - ])) - -Script.parseCommandLine() - -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient - -args = Script.getPositionalArgs() -if len(args) < 2: - Script.showHelp(exitCode=1) - -# get arguments -inputFileName = args[1] - -lfns = [] -if os.path.exists(inputFileName): - inputFile = open(inputFileName, 'r') - string = inputFile.read() - inputFile.close() - lfns.extend([lfn.strip() for lfn in string.splitlines()]) -else: - lfns.append(inputFileName) - -tc = TransformationClient() -res = tc.addFilesToTransformation(args[0], lfns) # Files added here - -if not res['OK']: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(2) - -successfullyAdded = 0 -alreadyPresent = 0 -for lfn, message in res['Value']['Successful'].items(): - if message == 'Added': - successfullyAdded += 1 - elif message == 'Present': - alreadyPresent += 1 - -if successfullyAdded > 0: - DIRAC.gLogger.notice("Successfully added %d files" % successfullyAdded) -if alreadyPresent > 0: - DIRAC.gLogger.notice("Already present %d files" % alreadyPresent) -DIRAC.exit(0) diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-archive.py b/src/DIRAC/TransformationSystem/scripts/dirac-transformation-archive.py deleted file mode 100755 index efcda0a5f21..00000000000 --- a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-archive.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -""" Archive a transformation -""" - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -import sys - -from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs -parseCommandLine() - -if not getPositionalArgs(): - print('Usage: dirac-transformation-archive transID [transID] [transID]') - sys.exit() -else: - transIDs = [int(arg) for arg in getPositionalArgs()] - -from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient - -agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', - 'Transformation/TransformationCleaningAgent', - 'dirac-transformation-archive') -agent.initialize() - -client = TransformationClient() -for transID in transIDs: - agent.archiveTransformation(transID) diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-clean.py b/src/DIRAC/TransformationSystem/scripts/dirac-transformation-clean.py deleted file mode 100755 index 97f8a93ab08..00000000000 --- a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-clean.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -""" Clean a tranformation -""" - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -import sys - -from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs -parseCommandLine() - -from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient - -if not getPositionalArgs(): - print('Usage: dirac-transformation-clean transID [transID] [transID]') - sys.exit() -else: - transIDs = [int(arg) for arg in getPositionalArgs()] - -agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', - 'Transformation/TransformationCleaningAgent', - 'dirac-transformation-clean') -agent.initialize() - -client = TransformationClient() -for transID in transIDs: - agent.cleanTransformation(transID) diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-cli.py b/src/DIRAC/TransformationSystem/scripts/dirac-transformation-cli.py deleted file mode 100755 index 562dac4f19f..00000000000 --- a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-cli.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -""" Command to launch the Transformation Shell -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from DIRAC.Core.Base import Script - -Script.setUsageMessage(""" -Launch the Transformation shell - -Usage: - %s [option] -""" % Script.scriptName) - - -Script.parseCommandLine(ignoreErrors=False) - -from DIRAC.TransformationSystem.Client.TransformationCLI import TransformationCLI - -cli = TransformationCLI() -cli.cmdloop() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-get-files.py b/src/DIRAC/TransformationSystem/scripts/dirac-transformation-get-files.py deleted file mode 100755 index 6024974e45e..00000000000 --- a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-get-files.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python - -""" - Get the files attached to a transformation -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s TransID' % Script.scriptName - ])) - -Script.parseCommandLine() - -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient - -args = Script.getPositionalArgs() -if len(args) != 1: - Script.showHelp(exitCode=1) - -tc = TransformationClient() -res = tc.getTransformationFiles({'TransformationID': args[0]}) - -if not res['OK']: - DIRAC.gLogger.error(res['Message']) - DIRAC.exit(2) - -for transfile in res['Value']: - DIRAC.gLogger.notice(transfile['LFN']) diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-remove-output.py b/src/DIRAC/TransformationSystem/scripts/dirac-transformation-remove-output.py deleted file mode 100755 index 24fc4765fa9..00000000000 --- a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-remove-output.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -""" Remove the outputs produced by a transformation -""" - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -import sys - -from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs -parseCommandLine() - -if not getPositionalArgs(): - print('Usage: dirac-transformation-remove-output transID [transID] [transID]') - sys.exit() -else: - transIDs = [int(arg) for arg in getPositionalArgs()] - -from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient - -agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', - 'Transformation/TransformationCleaningAgent', - 'dirac-transformation-remove-output') -agent.initialize() - -client = TransformationClient() -for transID in transIDs: - agent.removeTransformationOutput(transID) diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-verify-outputdata.py b/src/DIRAC/TransformationSystem/scripts/dirac-transformation-verify-outputdata.py deleted file mode 100755 index 99eb84bf263..00000000000 --- a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-verify-outputdata.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -""" runs checkTransformationIntegrity from ValidateOutputDataAgent on selected Tranformation -""" - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -import sys - -from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs -parseCommandLine() - -if not getPositionalArgs(): - print('Usage: dirac-transformation-verify-outputdata transID [transID] [transID]') - sys.exit() -else: - transIDs = [int(arg) for arg in getPositionalArgs()] - -from DIRAC.TransformationSystem.Agent.ValidateOutputDataAgent import ValidateOutputDataAgent -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient - -agent = ValidateOutputDataAgent('Transformation/ValidateOutputDataAgent', - 'Transformation/ValidateOutputDataAgent', - 'dirac-transformation-verify-outputdata') -agent.initialize() - -client = TransformationClient() -for transID in transIDs: - agent.checkTransformationIntegrity(transID) diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-production-runjoblocal.py b/src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py similarity index 81% rename from src/DIRAC/TransformationSystem/scripts/dirac-production-runjoblocal.py rename to src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py index 668ce1bcae2..eb965c8c94f 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac-production-runjoblocal.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py @@ -29,32 +29,7 @@ from urllib.error import URLError as url_library_URLError # pylint: disable=no-name-in-module,import-error from DIRAC.Core.Base import Script -Script.parseCommandLine(ignoreErrors=False) - -Script.registerSwitch('D:', 'Download=', 'Defines data acquisition as DownloadInputData') -Script.registerSwitch('P:', 'Protocol=', 'Defines data acquisition as InputDataByProtocol') - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - '\nUsage:', - 'dirac-production-runjoblocal [Data imput mode] [job ID]' - '\nArguments:', - ' Download (Job ID): Defines data aquisition as DownloadInputData', - ' Protocol (Job ID): Defines data acquisition as InputDataByProtocol\n'])) - -from DIRAC import S_OK -from DIRAC.Core.Utilities.File import mkDir - - -_downloadinputdata = False -_jobID = None - -for switch in Script.getUnprocessedSwitches(): - if switch[0] in ('D', 'Download'): - _downloadinputdata = True - _jobID = switch[1] - if switch[0] in ('I', 'Protocol'): - _downloadinputdata = False - _jobID = switch[1] +from DIRAC.Core.Utilities.DIRACScript import DIRACScript def __runSystemDefaults(jobID, vo): @@ -63,9 +38,8 @@ def __runSystemDefaults(jobID, vo): the path for the other functions. """ - + from DIRAC.Core.Utilities.File import mkDir tempdir = str(vo) + "job" + str(jobID) + "temp" - mkDir(tempdir) basepath = os.getcwd() return basepath + os.path.sep + tempdir + os.path.sep @@ -88,6 +62,7 @@ def __modifyJobDescription(jobID, basepath, downloadinputdata): uses InputDataByProtocol """ + from DIRAC import S_OK if not downloadinputdata: from xml.etree import ElementTree as et archive = et.parse(basepath + "InputSandbox" + str(jobID) + os.path.sep + "jobDescription.xml") @@ -160,7 +135,30 @@ def __runJobLocally(jobID, basepath, vo): localJob.runLocal() -if __name__ == "__main__": +@DIRACScript() +def main(): + Script.parseCommandLine(ignoreErrors=False) + + Script.registerSwitch('D:', 'Download=', 'Defines data acquisition as DownloadInputData') + Script.registerSwitch('P:', 'Protocol=', 'Defines data acquisition as InputDataByProtocol') + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + '\nUsage:', + 'dirac-production-runjoblocal [Data imput mode] [job ID]' + '\nArguments:', + ' Download (Job ID): Defines data aquisition as DownloadInputData', + ' Protocol (Job ID): Defines data acquisition as InputDataByProtocol\n'])) + _downloadinputdata = False + _jobID = None + + for switch in Script.getUnprocessedSwitches(): + if switch[0] in ('D', 'Download'): + _downloadinputdata = True + _jobID = switch[1] + if switch[0] in ('I', 'Protocol'): + _downloadinputdata = False + _jobID = switch[1] + from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import Extensions ext = Extensions() _vo = ext.getCSExtensions()[0] @@ -182,3 +180,7 @@ def __runJobLocally(jobID, basepath, vo): finally: os.chdir(_dir) os.rename(_dir + '.dirac.cfg.old', _dir + '.dirac.cfg') + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_add_files.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_add_files.py new file mode 100755 index 00000000000..b3cb8e739ab --- /dev/null +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_add_files.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +""" + Add files to an existing transformation +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import os +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s TransID ' % Script.scriptName + ])) + + Script.parseCommandLine() + + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + args = Script.getPositionalArgs() + if len(args) < 2: + Script.showHelp(exitCode=1) + + # get arguments + inputFileName = args[1] + + lfns = [] + if os.path.exists(inputFileName): + inputFile = open(inputFileName, 'r') + string = inputFile.read() + inputFile.close() + lfns.extend([lfn.strip() for lfn in string.splitlines()]) + else: + lfns.append(inputFileName) + + tc = TransformationClient() + res = tc.addFilesToTransformation(args[0], lfns) # Files added here + + if not res['OK']: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(2) + + successfullyAdded = 0 + alreadyPresent = 0 + for lfn, message in res['Value']['Successful'].items(): + if message == 'Added': + successfullyAdded += 1 + elif message == 'Present': + alreadyPresent += 1 + + if successfullyAdded > 0: + DIRAC.gLogger.notice("Successfully added %d files" % successfullyAdded) + if alreadyPresent > 0: + DIRAC.gLogger.notice("Already present %d files" % alreadyPresent) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py new file mode 100755 index 00000000000..ce6af535edc --- /dev/null +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +""" Archive a transformation +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +import sys + +from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + parseCommandLine() + + if not getPositionalArgs(): + print('Usage: dirac-transformation-archive transID [transID] [transID]') + sys.exit() + else: + transIDs = [int(arg) for arg in getPositionalArgs()] + + from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', + 'Transformation/TransformationCleaningAgent', + 'dirac-transformation-archive') + agent.initialize() + + client = TransformationClient() + for transID in transIDs: + agent.archiveTransformation(transID) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py new file mode 100755 index 00000000000..acfb5e4ca3e --- /dev/null +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +""" Clean a tranformation +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +import sys + +from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + parseCommandLine() + + from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + if not getPositionalArgs(): + print('Usage: dirac-transformation-clean transID [transID] [transID]') + sys.exit() + else: + transIDs = [int(arg) for arg in getPositionalArgs()] + + agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', + 'Transformation/TransformationCleaningAgent', + 'dirac-transformation-clean') + agent.initialize() + + client = TransformationClient() + for transID in transIDs: + agent.cleanTransformation(transID) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_cli.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_cli.py new file mode 100755 index 00000000000..2484a5d7d20 --- /dev/null +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_cli.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" Command to launch the Transformation Shell +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage(""" +Launch the Transformation shell + +Usage: + %s [option] +""" % Script.scriptName) + Script.parseCommandLine(ignoreErrors=False) + + from DIRAC.TransformationSystem.Client.TransformationCLI import TransformationCLI + + cli = TransformationCLI() + cli.cmdloop() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_get_files.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_get_files.py new file mode 100755 index 00000000000..9f26d525735 --- /dev/null +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_get_files.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +""" + Get the files attached to a transformation +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([ + __doc__.split('\n')[1], + 'Usage:', + ' %s TransID' % Script.scriptName + ])) + + Script.parseCommandLine() + + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + args = Script.getPositionalArgs() + if len(args) != 1: + Script.showHelp(exitCode=1) + + tc = TransformationClient() + res = tc.getTransformationFiles({'TransformationID': args[0]}) + + if not res['OK']: + DIRAC.gLogger.error(res['Message']) + DIRAC.exit(2) + + for transfile in res['Value']: + DIRAC.gLogger.notice(transfile['LFN']) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-recover-data.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_recover_data.py similarity index 94% rename from src/DIRAC/TransformationSystem/scripts/dirac-transformation-recover-data.py rename to src/DIRAC/TransformationSystem/scripts/dirac_transformation_recover_data.py index 8e3f1cd98f0..a4f9acb9420 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-recover-data.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_recover_data.py @@ -5,7 +5,7 @@ from __future__ import print_function from DIRAC import S_OK, gLogger from DIRAC.Core.Base import Script - +from DIRAC.Core.Utilities.DIRACScript import DIRACScript __RCSID__ = '$Id$' @@ -33,7 +33,8 @@ def registerSwitches(self): ' %s [option|cfgfile] ...\n' % Script.scriptName])) -if __name__ == '__main__': +@DIRACScript() +def main(): PARAMS = Params() PARAMS.registerSwitches() Script.parseCommandLine(ignoreErrors=False) @@ -56,3 +57,7 @@ def registerSwitches(self): gLogger.notice('Found transformation: %s' % TRANS_INFO_DICT) DRA.treatTransformation(PARAMS.transID, TRANS_INFO_DICT) exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py new file mode 100755 index 00000000000..131ea60dce5 --- /dev/null +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +""" Remove the outputs produced by a transformation +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +import sys + +from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + parseCommandLine() + + if not getPositionalArgs(): + print('Usage: dirac-transformation-remove-output transID [transID] [transID]') + sys.exit() + else: + transIDs = [int(arg) for arg in getPositionalArgs()] + + from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + agent = TransformationCleaningAgent('Transformation/TransformationCleaningAgent', + 'Transformation/TransformationCleaningAgent', + 'dirac-transformation-remove-output') + agent.initialize() + + client = TransformationClient() + for transID in transIDs: + agent.removeTransformationOutput(transID) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac-transformation-replication.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_replication.py similarity index 100% rename from src/DIRAC/TransformationSystem/scripts/dirac-transformation-replication.py rename to src/DIRAC/TransformationSystem/scripts/dirac_transformation_replication.py diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_verify_outputdata.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_verify_outputdata.py new file mode 100755 index 00000000000..037e5072409 --- /dev/null +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_verify_outputdata.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +""" runs checkTransformationIntegrity from ValidateOutputDataAgent on selected Tranformation +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +import sys + +from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + parseCommandLine() + + if not getPositionalArgs(): + print('Usage: dirac-transformation-verify-outputdata transID [transID] [transID]') + sys.exit() + else: + transIDs = [int(arg) for arg in getPositionalArgs()] + + from DIRAC.TransformationSystem.Agent.ValidateOutputDataAgent import ValidateOutputDataAgent + from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient + + agent = ValidateOutputDataAgent('Transformation/ValidateOutputDataAgent', + 'Transformation/ValidateOutputDataAgent', + 'dirac-transformation-verify-outputdata') + agent.initialize() + + client = TransformationClient() + for transID in transIDs: + agent.checkTransformationIntegrity(transID) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/__init__.py b/src/DIRAC/WorkloadManagementSystem/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-kill-pilot.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-kill-pilot.py deleted file mode 100755 index b3a33782a16..00000000000 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-kill-pilot.py +++ /dev/null @@ -1,39 +0,0 @@ -#! /usr/bin/env python -######################################################################## -# File : dirac-admin-kill-pilot -# Author : A.T. -######################################################################## -""" - Kill the specified pilot -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC.Core.Base import Script - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s ' % Script.scriptName])) - -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp(exitCode=1) - -pilotRef = args[0] - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -diracAdmin = DiracAdmin() -exitCode = 0 - -result = diracAdmin.killPilot(pilotRef) -if not result['OK']: - DIRAC.gLogger.error('Failed to kill pilot', pilotRef) - DIRAC.gLogger.error(result['Message']) - exitCode = 1 - -DIRAC.exit(exitCode) diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-pilot-logging-info.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-pilot-logging-info.py deleted file mode 100755 index 7ab7dcb7cd9..00000000000 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-pilot-logging-info.py +++ /dev/null @@ -1,62 +0,0 @@ -#! /usr/bin/env python -""" - Get Pilots Logging for specific Pilot UUID or Job ID. -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -__RCSID__ = "$Id$" - -import DIRAC -from DIRAC import S_OK, gLogger -from DIRAC.Core.Base import Script - -uuid = None -jobid = None - - -def setUUID(optVal): - """ - Set UUID from arguments - """ - global uuid - uuid = optVal - return S_OK() - - -def setJobID(optVal): - """ - Set JobID from arguments - """ - global jobid - jobid = optVal - return S_OK() - - -Script.registerSwitch('u:', 'uuid=', 'get PilotsLogging for given Pilot UUID', setUUID) -Script.registerSwitch('j:', 'jobid=', 'get PilotsLogging for given Job ID', setJobID) - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s option value ' % Script.scriptName, - 'Only one option (either uuid or jobid) should be used.'])) - -Script.parseCommandLine() - -from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient - -if jobid: - result = PilotManagerClient().getPilots(jobid) - if not result['OK']: - gLogger.error(result['Message']) - DIRAC.exit(1) - gLogger.debug(result['Value']) - uuid = list(result['Value'])[0] - -result = PilotManagerClient().getPilotLoggingInfo(uuid) -if not result['OK']: - gLogger.error(result['Message']) - DIRAC.exit(1) -gLogger.notice(result['Value']) - -DIRAC.exit(0) diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-show-task-queues.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-show-task-queues.py deleted file mode 100755 index 7fb404b0680..00000000000 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-show-task-queues.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-admin-show-task-queues -# Author : Ricardo Graciani -######################################################################## -""" - Show details of currently active Task Queues -""" -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -import sys - -from DIRAC import S_OK, gLogger -from DIRAC.Core.Base import Script -from DIRAC.Core.Utilities.PrettyPrint import printTable - -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getUsernameForDN -from DIRAC.WorkloadManagementSystem.Client.MatcherClient import MatcherClient - -verbose = False - - -def setVerbose(optVal): - global verbose - verbose = True - return S_OK() - - -taskQueueID = 0 - - -def setTaskQueueID(optVal): - global taskQueueID - taskQueueID = int(optVal) - return S_OK() - - -Script.registerSwitch("v", "verbose", "give max details about task queues", setVerbose) -Script.registerSwitch("t:", "taskQueue=", "show this task queue only", setTaskQueueID) - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ' % Script.scriptName])) - -Script.parseCommandLine(initializeMonitor=False) - -result = MatcherClient().getActiveTaskQueues() -if not result['OK']: - gLogger.error(result['Message']) - sys.exit(1) - -tqDict = result['Value'] - -if not verbose: - fields = ['TaskQueue', 'Jobs', 'CPUTime', 'Owner', 'OwnerGroup', 'Sites', - 'Platforms', 'SubmitPools', 'Setup', 'Priority'] - records = [] - - for tqId in sorted(tqDict): - if taskQueueID and tqId != taskQueueID: - continue - record = [str(tqId)] - tqData = tqDict[tqId] - for key in fields[1:]: - if key == 'Owner': - value = tqData.get('OwnerDN', '-') - if value != '-': - result = getUsernameForDN(value) - if not result['OK']: - value = 'Unknown' - else: - value = result['Value'] - else: - value = tqData.get(key, '-') - if isinstance(value, list): - if len(value) > 1: - record.append(str(value[0]) + '...') - else: - record.append(str(value[0])) - else: - record.append(str(value)) - records.append(record) - - printTable(fields, records) -else: - fields = ['Key', 'Value'] - for tqId in sorted(tqDict): - if taskQueueID and tqId != taskQueueID: - continue - gLogger.notice("\n==> TQ %s" % tqId) - records = [] - tqData = tqDict[tqId] - for key in sorted(tqData): - value = tqData[key] - if isinstance(value, list): - records.append([key, {"Value": value, 'Just': 'L'}]) - else: - value = str(value) - records.append([key, {"Value": value, 'Just': 'L'}]) - - printTable(fields, records, numbering=False) diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-sync-pilot.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-sync-pilot.py deleted file mode 100644 index 91cdd2f258b..00000000000 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-admin-sync-pilot.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -""" -This is a simple script that can be used for synchronizing pilot files -to the current directory. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -import os -import json -import hashlib - - -from DIRAC.Core.Base import Script -Script.parseCommandLine() - -from DIRAC import gLogger, exit as DIRACExit -from DIRAC.WorkloadManagementSystem.Utilities.PilotCStoJSONSynchronizer import PilotCStoJSONSynchronizer - -ps = PilotCStoJSONSynchronizer() - -gLogger.verbose("Parameters for this sync:") -gLogger.verbose("repo=" + ps.pilotRepo) -gLogger.verbose("VO repo=" + ps.pilotVORepo) -gLogger.verbose("projectDir=" + ps.projectDir) -gLogger.verbose("pilotScriptsPath=" + ps.pilotScriptPath) -gLogger.verbose("pilotVOScriptsPath=" + ps.pilotVOScriptPath) -gLogger.verbose("pilotRepoBranch=" + ps.pilotRepoBranch) -gLogger.verbose("pilotVORepoBranch=" + ps.pilotVORepoBranch) - -# pilot.json -res = ps.getCSDict() -if not res['OK']: - DIRACExit(1) -pilotDict = res['Value'] -print(json.dumps(pilotDict, indent=4, sort_keys=True)) # just print here as formatting is important -with open('pilot.json', 'w') as jf: - json.dump(pilotDict, jf) - -# pilot files -res = ps.syncScripts() -if not res['OK']: - DIRACExit(1) -gLogger.always(res['Value']) -tarPath, tarFiles = res['Value'] - -allFiles = [tarPath] + tarFiles + ['pilot.json'] - -# checksums -checksumDict = {} -for pFile in allFiles: - filename = os.path.basename(pFile) - with open(pFile, 'rb') as fp: - checksumDict[filename] = hashlib.sha512(fp.read()).hexdigest() - cksPath = 'checksums.sha512' -with open(cksPath, 'wt') as chksums: - for filename, chksum in sorted(checksumDict.items()): - # same as the output from sha512sum commands - chksums.write('%s %s\n' % (chksum, filename)) - -allFiles = allFiles + [cksPath] - -print(allFiles) diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-jobexec.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac-jobexec.py deleted file mode 100755 index 6c6d22ce472..00000000000 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-jobexec.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-jobexec -# Author : Stuart Paterson -######################################################################## -""" The dirac-jobexec script is equipped to execute workflows that - are specified via their XML description. The main client of - this script is the Job Wrapper. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - - -import os -import os.path -import sys - -import DIRAC -from DIRAC.Core.Base import Script - -# Register workflow parameter switch -Script.registerSwitch('p:', 'parameter=', 'Parameters that are passed directly to the workflow') -Script.parseCommandLine() - -# from DIRAC.Core.Workflow.Parameter import * -from DIRAC import gLogger -from DIRAC.Core.Workflow.Workflow import fromXMLFile -from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport -from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient -from DIRAC.RequestManagementSystem.Client.Request import Request - -# Forcing the current directory to be the first in the PYTHONPATH -sys.path.insert(0, os.path.realpath('.')) -gLogger.showHeaders(True) - - -def jobexec(jobxml, wfParameters): - jobfile = os.path.abspath(jobxml) - if not os.path.exists(jobfile): - gLogger.warn('Path to specified workflow %s does not exist' % (jobfile)) - sys.exit(1) - workflow = fromXMLFile(jobfile) - gLogger.debug(workflow) - code = workflow.createCode() - gLogger.debug(code) - jobID = 0 - if 'JOBID' in os.environ: - jobID = os.environ['JOBID'] - gLogger.info('DIRAC JobID %s is running at site %s' % (jobID, DIRAC.siteName())) - - workflow.addTool('JobReport', JobReport(jobID)) - workflow.addTool('AccountingReport', DataStoreClient()) - workflow.addTool('Request', Request()) - - # Propagate the command line parameters to the workflow if any - for pName, pValue in wfParameters.items(): - workflow.setValue(pName, pValue) - - # Propagate the command line parameters to the workflow module instances of each step - for stepdefinition in workflow.step_definitions.values(): - for moduleInstance in stepdefinition.module_instances: - for pName, pValue in wfParameters.items(): - if moduleInstance.parameters.find(pName): - moduleInstance.parameters.setValue(pName, pValue) - - return workflow.execute() - - -positionalArgs = Script.getPositionalArgs() -if len(positionalArgs) != 1: - gLogger.debug('Positional arguments were %s' % (positionalArgs)) - DIRAC.abort(1, "Must specify the Job XML file description") - -if 'JOBID' in os.environ: - gLogger.info('JobID: %s' % (os.environ['JOBID'])) - -jobXMLfile = positionalArgs[0] -parList = Script.getUnprocessedSwitches() -parDict = {} -for switch, parameter in parList: - if switch == "p": - name, value = parameter.split('=') - value = value.strip() - - # The comma separated list in curly brackets is interpreted as a list - if value.startswith("{"): - value = value[1:-1].replace('"', '').replace(" ", '').split(',') - value = ';'.join(value) - - parDict[name] = value - -gLogger.debug('PYTHONPATH:\n%s' % ('\n'.join(sys.path))) -jobExec = jobexec(jobXMLfile, parDict) -if not jobExec['OK']: - gLogger.debug('Workflow execution finished with errors, exiting') - if jobExec['Errno']: - sys.exit(jobExec['Errno']) - else: - sys.exit(1) -else: - gLogger.debug('Workflow execution successful, exiting') - sys.exit(0) diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-cpu-normalization.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-cpu-normalization.py deleted file mode 100755 index 0b86b86084c..00000000000 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-cpu-normalization.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-cpu-normalization -# Author : Andrew McNab -######################################################################## -""" - Determine Normalization for current CPU. - - The main users of this script are the pilot jobs. - - Pilots invoke dirac-wms-cpu-normalization which - - tries to find MACHINEFEATURES/ + JOBFEATURES, - and if found populates the local cfg file with e.g. '/LocalSite/JOBFEATURES/' - - runs 1 iteration of singleDiracBenchmark(1) (for single processors only) - - stores in local cfg the following: (the example below is from the case of when MJF is not available):: - - LocalSite - { - CPUScalingFactor = 23.7 # corrected value (by JobScheduling/CPUNormalizationCorrection) - CPUNormalizationFactor = 23.7 # corrected value (by JobScheduling/CPUNormalizationCorrection) - DB12measured = 15.4 - DB12 = 15.4 - } - - The last 2 (DB12 and DB12measured) are up to now wrote down but never used. - -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -__RCSID__ = "$Id$" -import DIRAC -from DIRAC.Core.Base import Script - -Script.registerSwitch("U", "Update", "Update dirac.cfg with the resulting value") -Script.registerSwitch("R:", "Reconfig=", "Update given configuration file with the resulting value") - -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ' % Script.scriptName])) - -Script.parseCommandLine(ignoreErrors=True) - -update = False -configFile = None - -for unprocSw in Script.getUnprocessedSwitches(): - if unprocSw[0] in ("U", "Update"): - update = True - elif unprocSw[0] in ("R", "Reconfig"): - configFile = unprocSw[1] - - -if __name__ == "__main__": - - from DIRAC import gLogger, gConfig - from DIRAC.WorkloadManagementSystem.Client.DIRACbenchmark import singleDiracBenchmark - from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations - from DIRAC.Core.Utilities import MJF - - mjf = MJF.MJF() - mjf.updateConfig() - - db12JobFeature = mjf.getJobFeature('db12') - hs06JobFeature = mjf.getJobFeature('hs06') - - result = singleDiracBenchmark(1) - - if result is None: - gLogger.error('Cannot make benchmark measurements') - DIRAC.exit(1) - - db12Measured = round(result['NORM'], 1) - corr = Operations().getValue('JobScheduling/CPUNormalizationCorrection', 1.) - norm = round(result['NORM'] / corr, 1) - - gLogger.notice('Estimated CPU power is %.1f HS06' % norm) - - if update: - gConfig.setOptionValue('/LocalSite/CPUScalingFactor', hs06JobFeature if hs06JobFeature else norm) # deprecate? - gConfig.setOptionValue('/LocalSite/CPUNormalizationFactor', norm) # deprecate? - gConfig.setOptionValue('/LocalSite/DB12measured', db12Measured) - - # Set DB12 to use by default. Remember db12JobFeature is still in /LocalSite/JOBFEATURES/db12 - if db12JobFeature is not None: - gConfig.setOptionValue('/LocalSite/DB12', db12JobFeature) - else: - gConfig.setOptionValue('/LocalSite/DB12', db12Measured) - - if configFile: - gConfig.dumpLocalCFGToFile(configFile) - else: - gConfig.dumpLocalCFGToFile(gConfig.diracConfigFilePath) - - DIRAC.exit() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-wn-parameters.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-wn-parameters.py deleted file mode 100755 index 67917ca76ce..00000000000 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-wn-parameters.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python -""" - Determine number of processors and memory for the worker node -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script -from DIRAC import gLogger -from DIRAC.WorkloadManagementSystem.Utilities import JobParameters - -Script.setUsageMessage('\n'.join(['Get the parameters (Memory and Number of processors) of a worker node', - 'Usage:', - ' %s [option]... [cfgfile]' % Script.scriptName, - 'Arguments:', - ' cfgfile: DIRAC Cfg with description of the configuration (optional)'])) - -ceName = '' -ceType = '' - - -def setCEName(args): - global ceName - ceName = args - - -def setSite(args): - global Site - Site = args - - -def setQueue(args): - global Queue - Queue = args - - -Script.registerSwitch("N:", "Name=", "Computing Element Name (Mandatory)", setCEName) -Script.registerSwitch("S:", "Site=", "Site Name (Mandatory)", setSite) -Script.registerSwitch("Q:", "Queue=", "Queue Name (Mandatory)", setQueue) -Script.parseCommandLine(ignoreErrors=True) - - -gLogger.info("Getting number of processors") -numberOfProcessor = JobParameters.getNumberOfProcessors(Site, ceName, Queue) - - -gLogger.info("Getting memory (RAM) from MJF") -maxRAM = JobParameters.getMemoryFromMJF() -if not maxRAM: - gLogger.info("maxRAM could not be found in MJF, using JobParameters.getMemoryFromProc()") - maxRAM = JobParameters.getMemoryFromProc() - -# just communicating it back -gLogger.notice(numberOfProcessor, maxRAM) diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-pilot-job-info.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-pilot-job-info.py deleted file mode 100755 index 5220ac331fe..00000000000 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-pilot-job-info.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -######################################################################## -# File : dirac-wms-pilot-job-info -# Author : Philippe Charpentier -######################################################################## -""" - Retrieve info about jobs run by the given pilot -""" - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script - - -def _stringInList(subStr, sList): - resList = [] - for s in sList: - if subStr.lower() in s.lower(): - resList.append(s) - return resList - - -parameters = ['OwnerDN', 'StartExecTime', 'EndExecTime'] -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, - 'Arguments:', - ' PilotID: Grid ID of the pilot'])) -Script.registerSwitch('', 'Parameters=', ' List of strings to be matched by job parameters or attributes') -Script.parseCommandLine(ignoreErrors=True) -for switch in Script.getUnprocessedSwitches(): - if switch[0] == 'Parameters': - parameters += [par for par in switch[1].split(',')] -parameters = [(i, par.lower()) for i, par in enumerate(parameters) if par] -args = Script.getPositionalArgs() - -if len(args) < 1: - Script.showHelp() - -from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin -from DIRAC.Interfaces.API.Dirac import Dirac -diracAdmin = DiracAdmin() -dirac = Dirac() -errorList = [] - -for gridID in args: - - result = {} - res = diracAdmin.getPilotInfo(gridID) - if not res['OK']: - errorList.append((gridID, res['Message'])) - else: - jobIDs = set([int(jobID) for jobID in res['Value'][gridID]['Jobs']]) - totCPU = 0 - totWall = 0 - effRequested = False - for jobID in sorted(jobIDs): - result.setdefault(jobID, {}) - for func in (dirac.getJobParameters, dirac.getJobAttributes): - res = func(jobID) - if not res['OK']: - errorList.append(('Job %d' % jobID, res['Message'])) - else: - params = res['Value'] - if 'TotalCPUTime(s)' in params: - totCPU += float(params['TotalCPUTime(s)']) - totWall += float(params['WallClockTime(s)']) - params['CPUEfficiency'] = '%s %%' % ( - 100. * params['TotalCPUTime(s)'] / params['WallClockTime(s)']) - for i, par in parameters: - for param in [p for p in _stringInList(str(par), str(params)) if not _stringInList(str(p), - str(result[jobID]))]: - if param == 'CPUEfficiency': - effRequested = True - result[jobID]['%d.%s' % (i, param)] = params[param] - if effRequested: - result['CPUEfficiency'] = '%s %%' % (100. * totCPU / totWall) - print(diracAdmin.pPrint.pformat({gridID: result})) - -for error in errorList: - print("ERROR %s: %s" % error) diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_kill_pilot.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_kill_pilot.py new file mode 100755 index 00000000000..5b49af94d6e --- /dev/null +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_kill_pilot.py @@ -0,0 +1,47 @@ +#! /usr/bin/env python +######################################################################## +# File : dirac-admin-kill-pilot +# Author : A.T. +######################################################################## +""" + Kill the specified pilot +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s ' % Script.scriptName])) + + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp(exitCode=1) + + pilotRef = args[0] + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + diracAdmin = DiracAdmin() + exitCode = 0 + + result = diracAdmin.killPilot(pilotRef) + if not result['OK']: + DIRAC.gLogger.error('Failed to kill pilot', pilotRef) + DIRAC.gLogger.error(result['Message']) + exitCode = 1 + + DIRAC.exit(exitCode) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_pilot_logging_info.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_pilot_logging_info.py new file mode 100755 index 00000000000..882e1a854e4 --- /dev/null +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_pilot_logging_info.py @@ -0,0 +1,71 @@ +#! /usr/bin/env python +""" + Get Pilots Logging for specific Pilot UUID or Job ID. +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +__RCSID__ = "$Id$" + +import DIRAC +from DIRAC import S_OK, gLogger +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + +uuid = None +jobid = None + + +def setUUID(optVal): + """ + Set UUID from arguments + """ + global uuid + uuid = optVal + return S_OK() + + +def setJobID(optVal): + """ + Set JobID from arguments + """ + global jobid + jobid = optVal + return S_OK() + + +@DIRACScript() +def main(): + global uuid + global jobid + Script.registerSwitch('u:', 'uuid=', 'get PilotsLogging for given Pilot UUID', setUUID) + Script.registerSwitch('j:', 'jobid=', 'get PilotsLogging for given Job ID', setJobID) + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s option value ' % Script.scriptName, + 'Only one option (either uuid or jobid) should be used.'])) + + Script.parseCommandLine() + + from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient + + if jobid: + result = PilotManagerClient().getPilots(jobid) + if not result['OK']: + gLogger.error(result['Message']) + DIRAC.exit(1) + gLogger.debug(result['Value']) + uuid = list(result['Value'])[0] + + result = PilotManagerClient().getPilotLoggingInfo(uuid) + if not result['OK']: + gLogger.error(result['Message']) + DIRAC.exit(1) + gLogger.notice(result['Value']) + + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py new file mode 100755 index 00000000000..f8d161fc4a1 --- /dev/null +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-admin-show-task-queues +# Author : Ricardo Graciani +######################################################################## +""" + Show details of currently active Task Queues +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +import sys + +from DIRAC import S_OK, gLogger +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.PrettyPrint import printTable + +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getUsernameForDN +from DIRAC.WorkloadManagementSystem.Client.MatcherClient import MatcherClient +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + +verbose = False + + +def setVerbose(optVal): + global verbose + verbose = True + return S_OK() + + +taskQueueID = 0 + + +def setTaskQueueID(optVal): + global taskQueueID + taskQueueID = int(optVal) + return S_OK() + + +@DIRACScript() +def main(): + global verbose + global taskQueueID + Script.registerSwitch("v", "verbose", "give max details about task queues", setVerbose) + Script.registerSwitch("t:", "taskQueue=", "show this task queue only", setTaskQueueID) + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ' % Script.scriptName])) + + Script.parseCommandLine(initializeMonitor=False) + + result = MatcherClient().getActiveTaskQueues() + if not result['OK']: + gLogger.error(result['Message']) + sys.exit(1) + + tqDict = result['Value'] + + if not verbose: + fields = ['TaskQueue', 'Jobs', 'CPUTime', 'Owner', 'OwnerGroup', 'Sites', + 'Platforms', 'SubmitPools', 'Setup', 'Priority'] + records = [] + + for tqId in sorted(tqDict): + if taskQueueID and tqId != taskQueueID: + continue + record = [str(tqId)] + tqData = tqDict[tqId] + for key in fields[1:]: + if key == 'Owner': + value = tqData.get('OwnerDN', '-') + if value != '-': + result = getUsernameForDN(value) + if not result['OK']: + value = 'Unknown' + else: + value = result['Value'] + else: + value = tqData.get(key, '-') + if isinstance(value, list): + if len(value) > 1: + record.append(str(value[0]) + '...') + else: + record.append(str(value[0])) + else: + record.append(str(value)) + records.append(record) + + printTable(fields, records) + else: + fields = ['Key', 'Value'] + for tqId in sorted(tqDict): + if taskQueueID and tqId != taskQueueID: + continue + gLogger.notice("\n==> TQ %s" % tqId) + records = [] + tqData = tqDict[tqId] + for key in sorted(tqData): + value = tqData[key] + if isinstance(value, list): + records.append([key, {"Value": value, 'Just': 'L'}]) + else: + value = str(value) + records.append([key, {"Value": value, 'Just': 'L'}]) + + printTable(fields, records, numbering=False) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_sync_pilot.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_sync_pilot.py new file mode 100644 index 00000000000..f87cdb4c3c5 --- /dev/null +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_sync_pilot.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +""" +This is a simple script that can be used for synchronizing pilot files +to the current directory. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import os +import json +import hashlib + + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.parseCommandLine() + + from DIRAC import gLogger, exit as DIRACExit + from DIRAC.WorkloadManagementSystem.Utilities.PilotCStoJSONSynchronizer import PilotCStoJSONSynchronizer + + ps = PilotCStoJSONSynchronizer() + + gLogger.verbose("Parameters for this sync:") + gLogger.verbose("repo=" + ps.pilotRepo) + gLogger.verbose("VO repo=" + ps.pilotVORepo) + gLogger.verbose("projectDir=" + ps.projectDir) + gLogger.verbose("pilotScriptsPath=" + ps.pilotScriptPath) + gLogger.verbose("pilotVOScriptsPath=" + ps.pilotVOScriptPath) + gLogger.verbose("pilotRepoBranch=" + ps.pilotRepoBranch) + gLogger.verbose("pilotVORepoBranch=" + ps.pilotVORepoBranch) + + # pilot.json + res = ps.getCSDict() + if not res['OK']: + DIRACExit(1) + pilotDict = res['Value'] + print(json.dumps(pilotDict, indent=4, sort_keys=True)) # just print here as formatting is important + with open('pilot.json', 'w') as jf: + json.dump(pilotDict, jf) + + # pilot files + res = ps.syncScripts() + if not res['OK']: + DIRACExit(1) + gLogger.always(res['Value']) + tarPath, tarFiles = res['Value'] + + allFiles = [tarPath] + tarFiles + ['pilot.json'] + + # checksums + checksumDict = {} + for pFile in allFiles: + filename = os.path.basename(pFile) + with open(pFile, 'rb') as fp: + checksumDict[filename] = hashlib.sha512(fp.read()).hexdigest() + cksPath = 'checksums.sha512' + with open(cksPath, 'wt') as chksums: + for filename, chksum in sorted(checksumDict.items()): + # same as the output from sha512sum commands + chksums.write('%s %s\n' % (chksum, filename)) + + allFiles = allFiles + [cksPath] + + print(allFiles) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py new file mode 100755 index 00000000000..7ae1c197130 --- /dev/null +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_jobexec.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-jobexec +# Author : Stuart Paterson +######################################################################## +""" The dirac-jobexec script is equipped to execute workflows that + are specified via their XML description. The main client of + this script is the Job Wrapper. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + + +import os +import os.path +import sys + +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + # Register workflow parameter switch + Script.registerSwitch('p:', 'parameter=', 'Parameters that are passed directly to the workflow') + Script.parseCommandLine() + + # from DIRAC.Core.Workflow.Parameter import * + from DIRAC import gLogger + from DIRAC.Core.Workflow.Workflow import fromXMLFile + from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport + from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient + from DIRAC.RequestManagementSystem.Client.Request import Request + + # Forcing the current directory to be the first in the PYTHONPATH + sys.path.insert(0, os.path.realpath('.')) + gLogger.showHeaders(True) + + def jobexec(jobxml, wfParameters): + jobfile = os.path.abspath(jobxml) + if not os.path.exists(jobfile): + gLogger.warn('Path to specified workflow %s does not exist' % (jobfile)) + sys.exit(1) + workflow = fromXMLFile(jobfile) + gLogger.debug(workflow) + code = workflow.createCode() + gLogger.debug(code) + jobID = 0 + if 'JOBID' in os.environ: + jobID = os.environ['JOBID'] + gLogger.info('DIRAC JobID %s is running at site %s' % (jobID, DIRAC.siteName())) + + workflow.addTool('JobReport', JobReport(jobID)) + workflow.addTool('AccountingReport', DataStoreClient()) + workflow.addTool('Request', Request()) + + # Propagate the command line parameters to the workflow if any + for pName, pValue in wfParameters.items(): + workflow.setValue(pName, pValue) + + # Propagate the command line parameters to the workflow module instances of each step + for stepdefinition in workflow.step_definitions.values(): + for moduleInstance in stepdefinition.module_instances: + for pName, pValue in wfParameters.items(): + if moduleInstance.parameters.find(pName): + moduleInstance.parameters.setValue(pName, pValue) + + return workflow.execute() + + positionalArgs = Script.getPositionalArgs() + if len(positionalArgs) != 1: + gLogger.debug('Positional arguments were %s' % (positionalArgs)) + DIRAC.abort(1, "Must specify the Job XML file description") + + if 'JOBID' in os.environ: + gLogger.info('JobID: %s' % (os.environ['JOBID'])) + + jobXMLfile = positionalArgs[0] + parList = Script.getUnprocessedSwitches() + parDict = {} + for switch, parameter in parList: + if switch == "p": + name, value = parameter.split('=') + value = value.strip() + + # The comma separated list in curly brackets is interpreted as a list + if value.startswith("{"): + value = value[1:-1].replace('"', '').replace(" ", '').split(',') + value = ';'.join(value) + + parDict[name] = value + + gLogger.debug('PYTHONPATH:\n%s' % ('\n'.join(sys.path))) + jobExec = jobexec(jobXMLfile, parDict) + if not jobExec['OK']: + gLogger.debug('Workflow execution finished with errors, exiting') + if jobExec['Errno']: + sys.exit(jobExec['Errno']) + else: + sys.exit(1) + else: + gLogger.debug('Workflow execution successful, exiting') + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_cpu_normalization.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_cpu_normalization.py new file mode 100755 index 00000000000..bcc1131a099 --- /dev/null +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_cpu_normalization.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-cpu-normalization +# Author : Andrew McNab +######################################################################## +""" + Determine Normalization for current CPU. + + The main users of this script are the pilot jobs. + + Pilots invoke dirac-wms-cpu-normalization which + - tries to find MACHINEFEATURES/ + JOBFEATURES, + and if found populates the local cfg file with e.g. '/LocalSite/JOBFEATURES/' + - runs 1 iteration of singleDiracBenchmark(1) (for single processors only) + - stores in local cfg the following: (the example below is from the case of when MJF is not available):: + + LocalSite + { + CPUScalingFactor = 23.7 # corrected value (by JobScheduling/CPUNormalizationCorrection) + CPUNormalizationFactor = 23.7 # corrected value (by JobScheduling/CPUNormalizationCorrection) + DB12measured = 15.4 + DB12 = 15.4 + } + + The last 2 (DB12 and DB12measured) are up to now wrote down but never used. + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +__RCSID__ = "$Id$" +import DIRAC +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +@DIRACScript() +def main(): + Script.registerSwitch("U", "Update", "Update dirac.cfg with the resulting value") + Script.registerSwitch("R:", "Reconfig=", "Update given configuration file with the resulting value") + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ' % Script.scriptName])) + + Script.parseCommandLine(ignoreErrors=True) + + update = False + configFile = None + + for unprocSw in Script.getUnprocessedSwitches(): + if unprocSw[0] in ("U", "Update"): + update = True + elif unprocSw[0] in ("R", "Reconfig"): + configFile = unprocSw[1] + + if __name__ == "__main__": + + from DIRAC import gLogger, gConfig + from DIRAC.WorkloadManagementSystem.Client.DIRACbenchmark import singleDiracBenchmark + from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations + from DIRAC.Core.Utilities import MJF + + mjf = MJF.MJF() + mjf.updateConfig() + + db12JobFeature = mjf.getJobFeature('db12') + hs06JobFeature = mjf.getJobFeature('hs06') + + result = singleDiracBenchmark(1) + + if result is None: + gLogger.error('Cannot make benchmark measurements') + DIRAC.exit(1) + + db12Measured = round(result['NORM'], 1) + corr = Operations().getValue('JobScheduling/CPUNormalizationCorrection', 1.) + norm = round(result['NORM'] / corr, 1) + + gLogger.notice('Estimated CPU power is %.1f HS06' % norm) + + if update: + gConfig.setOptionValue('/LocalSite/CPUScalingFactor', hs06JobFeature if hs06JobFeature else norm) # deprecate? + gConfig.setOptionValue('/LocalSite/CPUNormalizationFactor', norm) # deprecate? + gConfig.setOptionValue('/LocalSite/DB12measured', db12Measured) + + # Set DB12 to use by default. Remember db12JobFeature is still in /LocalSite/JOBFEATURES/db12 + if db12JobFeature is not None: + gConfig.setOptionValue('/LocalSite/DB12', db12JobFeature) + else: + gConfig.setOptionValue('/LocalSite/DB12', db12Measured) + + if configFile: + gConfig.dumpLocalCFGToFile(configFile) + else: + gConfig.dumpLocalCFGToFile(gConfig.diracConfigFilePath) + + DIRAC.exit() + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-queue-cpu-time.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_queue_cpu_time.py similarity index 55% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-queue-cpu-time.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_queue_cpu_time.py index 24de49176f2..01c59ac7120 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-queue-cpu-time.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_queue_cpu_time.py @@ -13,22 +13,29 @@ import DIRAC from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -Script.registerSwitch("C:", "CPUNormalizationFactor=", "CPUNormalizationFactor, in case it is known") -Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile]' % Script.scriptName])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() -CPUNormalizationFactor = 0.0 -for unprocSw in Script.getUnprocessedSwitches(): - if unprocSw[0] in ("C", "CPUNormalizationFactor"): - CPUNormalizationFactor = float(unprocSw[1]) +@DIRACScript() +def main(): + Script.registerSwitch("C:", "CPUNormalizationFactor=", "CPUNormalizationFactor, in case it is known") + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile]' % Script.scriptName])) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() + + CPUNormalizationFactor = 0.0 + for unprocSw in Script.getUnprocessedSwitches(): + if unprocSw[0] in ("C", "CPUNormalizationFactor"): + CPUNormalizationFactor = float(unprocSw[1]) -if __name__ == "__main__": from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import getCPUTime cpuTime = getCPUTime(CPUNormalizationFactor) # I hate this kind of output... PhC print("CPU time left determined as", cpuTime) DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-wn.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn.py similarity index 98% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-wn.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn.py index 16df0b29653..db174e18c27 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-get-wn.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn.py @@ -15,9 +15,11 @@ import DIRAC import DIRAC.Core.Base.Script as Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript -if __name__ == "__main__": +@DIRACScript() +def main(): site = 'BOINC.World.org' status = ["Running"] minorStatus = None @@ -195,3 +197,7 @@ node = result[job]['Node'] jobID = result[job].get('LocalJobID') gLogger.always('%s (%s): %s - %s' % (node, jobID, job, status)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn_parameters.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn_parameters.py new file mode 100755 index 00000000000..99c82f06239 --- /dev/null +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_get_wn_parameters.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +""" + Determine number of processors and memory for the worker node +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript +from DIRAC import gLogger +from DIRAC.WorkloadManagementSystem.Utilities import JobParameters + +ceName = '' +ceType = '' +Queue = '' + + +def setCEName(args): + global ceName + ceName = args + + +def setSite(args): + global Site + Site = args + + +def setQueue(args): + global Queue + Queue = args + + +@DIRACScript() +def main(): + global ceName + global Site + global Queue + Script.setUsageMessage('\n'.join(['Get the parameters (Memory and Number of processors) of a worker node', + 'Usage:', + ' %s [option]... [cfgfile]' % Script.scriptName, + 'Arguments:', + ' cfgfile: DIRAC Cfg with description of the configuration (optional)'])) + + Script.registerSwitch("N:", "Name=", "Computing Element Name (Mandatory)", setCEName) + Script.registerSwitch("S:", "Site=", "Site Name (Mandatory)", setSite) + Script.registerSwitch("Q:", "Queue=", "Queue Name (Mandatory)", setQueue) + Script.parseCommandLine(ignoreErrors=True) + + gLogger.info("Getting number of processors") + numberOfProcessor = JobParameters.getNumberOfProcessors(Site, ceName, Queue) + + gLogger.info("Getting memory (RAM) from MJF") + maxRAM = JobParameters.getMemoryFromMJF() + if not maxRAM: + gLogger.info("maxRAM could not be found in MJF, using JobParameters.getMemoryFromProc()") + maxRAM = JobParameters.getMemoryFromProc() + + # just communicating it back + gLogger.notice(numberOfProcessor, maxRAM) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-match.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_match.py similarity index 61% rename from src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-match.py rename to src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_match.py index faab667f379..62524f8aaf1 100644 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac-wms-match.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_match.py @@ -9,6 +9,7 @@ from DIRAC.Core.Base import Script from DIRAC import S_OK, gLogger, exit as DIRACExit +from DIRAC.Core.Utilities.DIRACScript import DIRACScript __RCSID__ = "$Id$" @@ -28,36 +29,37 @@ def setSites(optVal_): return S_OK() -description = """Get computing resources capable to execute a job with the given description. - -Note that only statically defined computing resource parameters are considered although sites -can fail matching due to their dynamic state, e.g. occupancy by other jobs. Also input data -proximity is not taken into account. -""" - -Script.setUsageMessage('\n'.join([description, - 'Usage:', - ' %s [option]... ' % Script.scriptName])) +@DIRACScript() +def main(): + global fullMatch + global sites + description = """Get computing resources capable to execute a job with the given description. -Script.registerSwitch("F", "full-match", "Check all the matching criteria", setFullMatch) -Script.registerSwitch("S:", "site=", "Check matching for these sites (comma separated list)", setSites) + Note that only statically defined computing resource parameters are considered although sites + can fail matching due to their dynamic state, e.g. occupancy by other jobs. Also input data + proximity is not taken into account. + """ + Script.setUsageMessage('\n'.join([description, + 'Usage:', + ' %s [option]... ' % Script.scriptName])) -Script.parseCommandLine(ignoreErrors=True) -args = Script.getPositionalArgs() + Script.registerSwitch("F", "full-match", "Check all the matching criteria", setFullMatch) + Script.registerSwitch("S:", "site=", "Check matching for these sites (comma separated list)", setSites) -if len(args) == 0: - gLogger.error("Error: No job description provided") - Script.showHelp(exitCode=1) + Script.parseCommandLine(ignoreErrors=True) + args = Script.getPositionalArgs() -from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup -from DIRAC.ConfigurationSystem.Client.Helpers import Resources -from DIRAC.Core.Utilities.PrettyPrint import printTable -from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus -from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus -from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved, matchQueue + if len(args) == 0: + gLogger.error("Error: No job description provided") + Script.showHelp(exitCode=1) + from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup + from DIRAC.ConfigurationSystem.Client.Helpers import Resources + from DIRAC.Core.Utilities.PrettyPrint import printTable + from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus + from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus + from DIRAC.WorkloadManagementSystem.Utilities.QueueUtilities import getQueuesResolved, matchQueue -if __name__ == '__main__': with open(args[0]) as f: jdl = f.read() @@ -112,3 +114,7 @@ def setSites(optVal_): records.append((site, ce, queueInfo['Queue'], status, 'No', result['Value']['Reason'])) gLogger.notice(printTable(fields, records, sortField='Site', columnSeparator=' ', printOut=False)) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py new file mode 100755 index 00000000000..fadf153bb63 --- /dev/null +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +######################################################################## +# File : dirac-wms-pilot-job-info +# Author : Philippe Charpentier +######################################################################## +""" + Retrieve info about jobs run by the given pilot +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +from DIRAC.Core.Utilities.DIRACScript import DIRACScript + + +def _stringInList(subStr, sList): + resList = [] + for s in sList: + if subStr.lower() in s.lower(): + resList.append(s) + return resList + + +@DIRACScript() +def main(): + parameters = ['OwnerDN', 'StartExecTime', 'EndExecTime'] + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, + 'Arguments:', + ' PilotID: Grid ID of the pilot'])) + Script.registerSwitch('', 'Parameters=', ' List of strings to be matched by job parameters or attributes') + Script.parseCommandLine(ignoreErrors=True) + for switch in Script.getUnprocessedSwitches(): + if switch[0] == 'Parameters': + parameters += [par for par in switch[1].split(',')] + parameters = [(i, par.lower()) for i, par in enumerate(parameters) if par] + args = Script.getPositionalArgs() + + if len(args) < 1: + Script.showHelp() + + from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin + from DIRAC.Interfaces.API.Dirac import Dirac + diracAdmin = DiracAdmin() + dirac = Dirac() + errorList = [] + + for gridID in args: + + result = {} + res = diracAdmin.getPilotInfo(gridID) + if not res['OK']: + errorList.append((gridID, res['Message'])) + else: + jobIDs = set([int(jobID) for jobID in res['Value'][gridID]['Jobs']]) + totCPU = 0 + totWall = 0 + effRequested = False + for jobID in sorted(jobIDs): + result.setdefault(jobID, {}) + for func in (dirac.getJobParameters, dirac.getJobAttributes): + res = func(jobID) + if not res['OK']: + errorList.append(('Job %d' % jobID, res['Message'])) + else: + params = res['Value'] + if 'TotalCPUTime(s)' in params: + totCPU += float(params['TotalCPUTime(s)']) + totWall += float(params['WallClockTime(s)']) + params['CPUEfficiency'] = '%s %%' % ( + 100. * params['TotalCPUTime(s)'] / params['WallClockTime(s)']) + for i, par in parameters: + for param in [p for p in _stringInList(str(par), str(params)) + if not _stringInList(str(p), str(result[jobID]))]: + if param == 'CPUEfficiency': + effRequested = True + result[jobID]['%d.%s' % (i, param)] = params[param] + if effRequested: + result['CPUEfficiency'] = '%s %%' % (100. * totCPU / totWall) + print(diracAdmin.pPrint.pformat({gridID: result})) + + for error in errorList: + print("ERROR %s: %s" % error) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/__init__.py b/src/DIRAC/__init__.py index 134eafb5dd0..c685ba1e25f 100755 --- a/src/DIRAC/__init__.py +++ b/src/DIRAC/__init__.py @@ -70,6 +70,9 @@ import os import platform as pyPlatform from pkgutil import extend_path + +import six + __path__ = extend_path(__path__, __name__) # Set the environment variable such that openssl accepts proxy cert @@ -88,40 +91,31 @@ # Importing _strptime before instantiating the threads seem to be a working workaround import _strptime +# Define Version +if six.PY3: + from pkg_resources import get_distribution, DistributionNotFound -# Define Version, use an unusual structure to minimise conflicts with rel-v7r2 -pythonVersion = pyPlatform.python_version_tuple() -if pythonVersion[0] == "3": - version = "7.2" + try: + __version__ = get_distribution(__name__).version + version = __version__ + except DistributionNotFound: + # package is not installed + version = "Unknown" else: majorVersion = 7 minorVersion = 2 patchLevel = 0 preVersion = 28 - # Define version only for python 2, When using python 3 the version definition will change - # as needed by setup tools version = "v%sr%s" % (majorVersion, minorVersion) - buildVersion = "v%dr%d" % (majorVersion, minorVersion) + # Make it so that __version__ is always PEP-440 style + __version__ = "%s.%s" % (majorVersion, minorVersion) if patchLevel: version = "%sp%s" % (version, patchLevel) - buildVersion = "%s build %s" % (buildVersion, patchLevel) + __version__ += ".%s" % patchLevel if preVersion: version = "%s-pre%s" % (version, preVersion) - buildVersion = "%s pre %s" % (buildVersion, preVersion) - -# Check of python version - -__pythonMajorVersion = ("2", "3") -__pythonMinorVersion = ("7", "8") -if str(pythonVersion[0]) not in __pythonMajorVersion or str(pythonVersion[1]) not in __pythonMinorVersion: - print("Python Version %s not supported by DIRAC" % pyPlatform.python_version()) - print("Supported versions are: ") - for major in __pythonMajorVersion: - for minor in __pythonMinorVersion: - print("%s.%s.x" % (major, minor)) - - sys.exit(1) + __version__ += "a%s" % preVersion errorMail = "dirac.alarms@gmail.com" alarmMail = "dirac.alarms@gmail.com" @@ -219,3 +213,9 @@ def abort(exitCode, *args, **kwargs): except OSError: gLogger.exception('Error while executing DIRAC.abort') os._exit(exitCode) + + +def extension_metadata(): + return { + "priority": 0, + } diff --git a/tests/py3CheckDirs.txt b/tests/py3CheckDirs.txt index 2c872f3af71..1a3ad63d306 100644 --- a/tests/py3CheckDirs.txt +++ b/tests/py3CheckDirs.txt @@ -15,6 +15,6 @@ src/DIRAC/DataManagementSystem/Service/TornadoFileCatalogHandler.py src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py src/DIRAC/FrameworkSystem/Client/MonitoringClientIOLoop.py src/DIRAC/FrameworkSystem/Client/MonitoringClient.py -src/DIRAC/FrameworkSystem/scripts/dirac-install-tornado-service.py +src/DIRAC/FrameworkSystem/scripts/dirac_install_tornado_service.py src/DIRAC/Resources/Catalog/FileCatalogClient.py src/DIRAC/Resources/Storage/GFAL2_StorageBase.py