diff --git a/docs/diracdoctools/Utilities.py b/docs/diracdoctools/Utilities.py index 541a4a5769e..792da23f324 100644 --- a/docs/diracdoctools/Utilities.py +++ b/docs/diracdoctools/Utilities.py @@ -113,7 +113,7 @@ def runCommand(command): universal_newlines=True) if 'NOTICE:' in result: lines = [] - LOG.warn('NOTICE in output for: %s; cleaning output from datestamp..', command) + LOG.warning('NOTICE in output for: %s; cleaning output from datestamp..', command) for line in result.split('\n'): lines.append(re.sub(r"^.*NOTICE: ", "", line)) # if the output is less than 3 lines something went wrong diff --git a/docs/diracdoctools/cmd/codeReference.py b/docs/diracdoctools/cmd/codeReference.py index 6311ca76861..535b8a73857 100644 --- a/docs/diracdoctools/cmd/codeReference.py +++ b/docs/diracdoctools/cmd/codeReference.py @@ -1,8 +1,10 @@ #!/usr/bin/env python """ create rst files for documentation of DIRAC """ + from __future__ import absolute_import from __future__ import division from __future__ import print_function + import os import shutil import socket @@ -194,7 +196,7 @@ def getmodules(self, abspath, _direc, files): LOG.debug('Skipping file: %s/%s', abspath, filename) continue if 'test' in filename.lower(): - LOG.warn("File contains 'test', but is kept: %s/%s", abspath, filename) + LOG.warning("File contains 'test', but is kept: %s/%s", abspath, filename) if filename != '__init__.py': packages.append(filename.split('.py')[0]) @@ -386,5 +388,6 @@ def run(configFile='docs.conf', logLevel=logging.INFO, debug=False, buildType='f except ImportError: return 1 + if __name__ == '__main__': sys.exit(run(**(CLParser().optionDict()))) diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py index c014e5f0615..6fc9bcae334 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py @@ -20,6 +20,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function + __RCSID__ = "$Id$" from DIRAC import gLogger, exit as DIRACExit diff --git a/src/DIRAC/Core/scripts/dirac_executor.py b/src/DIRAC/Core/scripts/dirac_executor.py index bcfc9d00b10..91a60f6f7df 100755 --- a/src/DIRAC/Core/scripts/dirac_executor.py +++ b/src/DIRAC/Core/scripts/dirac_executor.py @@ -20,6 +20,7 @@ from DIRAC.Core.Utilities.DIRACScript import DIRACScript from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration + @DIRACScript() def main(): localCfg = LocalConfiguration() diff --git a/src/DIRAC/Core/scripts/dirac_service.py b/src/DIRAC/Core/scripts/dirac_service.py index 56ed44730f9..4bdded1ff52 100755 --- a/src/DIRAC/Core/scripts/dirac_service.py +++ b/src/DIRAC/Core/scripts/dirac_service.py @@ -12,6 +12,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function + import sys from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration diff --git a/src/DIRAC/Core/scripts/install_full.cfg b/src/DIRAC/Core/scripts/install_full.cfg index 50df8e0007c..29a0c4a50c6 100755 --- a/src/DIRAC/Core/scripts/install_full.cfg +++ b/src/DIRAC/Core/scripts/install_full.cfg @@ -163,9 +163,9 @@ LocalInstallation # Flag determining whether the Web Portal will be installed WebPortal = Yes WebApp = yes - # - # The following options defined the MySQL DB connectivity - # + + # The following options define the DB's connectivity + # For MySQL Database { User = Dirac @@ -175,6 +175,7 @@ LocalInstallation Host = Port = } + # For ElasticSearch NoSQLDatabases { User = @@ -183,4 +184,3 @@ LocalInstallation Port = } } - diff --git a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py index f21e897bebc..b6de2f66864 100644 --- a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py +++ b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py @@ -561,9 +561,10 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): if mySetup is None: mySetup = self.setup - result = self.monitoringClient.getInstallations({'UnInstallationTime': None, 'Instance': component}, - {'System': system}, - {}, True) + result = self.monitoringClient.getInstallations( + {'UnInstallationTime': None, 'Instance': component}, + {'DIRACSystem': system}, + {}, True) if not result['OK']: return result installations = result['Value'] @@ -586,10 +587,11 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): else: isRenamed = True - result = self.monitoringClient.getInstallations({'UnInstallationTime': None}, - {'System': system, - 'Module': installation['Component']['Module']}, - {}, True) + result = self.monitoringClient.getInstallations( + {'UnInstallationTime': None}, + {'System': system, 'DIRACModule': installation['Component']['Module']}, + {}, + True) if not result['OK']: return result installations = result['Value'] @@ -871,9 +873,10 @@ def removeDatabaseOptionsFromCS(self, gConfig_o, system, dbName, mySetup=None): if mySetup is None: mySetup = self.setup - result = self.monitoringClient.installationExists({'UnInstallationTime': None}, - {'System': system, 'Type': 'DB', 'Module': dbName}, - {}) + result = self.monitoringClient.installationExists( + {'UnInstallationTime': None}, + {'DIRACSystem': system, 'Type': 'DB', 'DIRACModule': dbName}, + {}) if not result['OK']: return result exists = result['Value'] diff --git a/src/DIRAC/FrameworkSystem/Client/ComponentMonitoringClient.py b/src/DIRAC/FrameworkSystem/Client/ComponentMonitoringClient.py index 9bf0bd24533..063584b16a7 100644 --- a/src/DIRAC/FrameworkSystem/Client/ComponentMonitoringClient.py +++ b/src/DIRAC/FrameworkSystem/Client/ComponentMonitoringClient.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function + __RCSID__ = "$Id$" from DIRAC.Core.Base.Client import Client, createClient diff --git a/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py b/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py index 7e7161b97aa..d1fa02f5040 100644 --- a/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py +++ b/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py @@ -37,8 +37,8 @@ class Component(componentsBase): 'mysql_charset': 'utf8'} componentID = Column('ComponentID', Integer, primary_key=True) - system = Column('System', String(32), nullable=False) - module = Column('Module', String(32), nullable=False) + system = Column('DIRACSystem', String(32), nullable=False) + module = Column('DIRACModule', String(32), nullable=False) cType = Column('Type', String(32), nullable=False) def __init__(self, system=null(), module=null(), cType=null()): @@ -56,8 +56,8 @@ def fromDict(self, dictionary): """ self.componentID = dictionary.get('ComponentID', self.componentID) - self.system = dictionary.get('System', self.system) - self.module = dictionary.get('Module', self.module) + self.system = dictionary.get('DIRACSystem', self.system) + self.module = dictionary.get('DIRACModule', self.module) self.cType = dictionary.get('Type', self.cType) return S_OK('Successfully read from dictionary') @@ -72,8 +72,8 @@ def toDict(self, includeInstallations=False, includeHosts=False): """ dictionary = {'ComponentID': self.componentID, - 'System': self.system, - 'Module': self.module, + 'DIRACSystem': self.system, + 'DIRACModule': self.module, 'Type': self.cType} if includeInstallations: diff --git a/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py b/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py index c0bdbbfbc3f..49969fa9424 100644 --- a/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py +++ b/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py @@ -312,7 +312,7 @@ def export_updateSoftware(self, version, rootPath="", gridVersion=""): if "WebApp" in extensionList: extensionList.remove("WebApp") - webPortal = gConfig.getValue('/LocalInstallation/WebApp', False) # this is the new portal + webPortal = gConfig.getValue('/LocalInstallation/WebApp', False) if webPortal: if "WebAppDIRAC" not in extensionList: extensionList.append("WebAppDIRAC") diff --git a/src/DIRAC/FrameworkSystem/Utilities/MonitoringUtilities.py b/src/DIRAC/FrameworkSystem/Utilities/MonitoringUtilities.py index 11f0933b429..e7b4c60944c 100644 --- a/src/DIRAC/FrameworkSystem/Utilities/MonitoringUtilities.py +++ b/src/DIRAC/FrameworkSystem/Utilities/MonitoringUtilities.py @@ -48,8 +48,8 @@ def monitorInstallation(componentType, system, component, module=None, cpu=None, result = monitoringClient.installationExists({'Instance': instance, 'UnInstallationTime': None}, {'Type': componentType, - 'System': system, - 'Module': module}, + 'DIRACSystem': system, + 'DIRACModule': module}, {'HostName': hostname, 'CPU': cpu}) @@ -62,8 +62,8 @@ def monitorInstallation(componentType, system, component, module=None, cpu=None, 'InstalledBy': user, 'Instance': instance}, {'Type': componentType, - 'System': system, - 'Module': module}, + 'DIRACSystem': system, + 'DIRACModule': module}, {'HostName': hostname, 'CPU': cpu}, True) @@ -99,9 +99,9 @@ def monitorUninstallation(system, component, cpu=None, hostname=None): hostname = socket.getfqdn() instance = component[0: 32] - result = monitoringClient.updateInstallations({'Instance': instance, 'UnInstallationTime': None}, - {'System': system}, - {'HostName': hostname, 'CPU': cpu}, - {'UnInstallationTime': datetime.datetime.utcnow(), - 'UnInstalledBy': user}) + result = monitoringClient.updateInstallations( + {'Instance': instance, 'UnInstallationTime': None}, + {'DIRACSystem': system}, + {'HostName': hostname, 'CPU': cpu}, + {'UnInstallationTime': datetime.datetime.utcnow(), 'UnInstalledBy': user}) return result diff --git a/src/DIRAC/FrameworkSystem/private/standardLogging/LogLevels.py b/src/DIRAC/FrameworkSystem/private/standardLogging/LogLevels.py index 81dec18e191..4d0ca3b3cde 100644 --- a/src/DIRAC/FrameworkSystem/private/standardLogging/LogLevels.py +++ b/src/DIRAC/FrameworkSystem/private/standardLogging/LogLevels.py @@ -29,7 +29,7 @@ class LogLevels(object): DEBUG = logging.DEBUG VERBOSE = 15 INFO = logging.INFO - WARN = logging.WARN + WARN = logging.WARNING NOTICE = 35 ERROR = logging.ERROR ALWAYS = 45 diff --git a/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg b/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg index 36a892df022..2d67ee4996a 100644 --- a/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg +++ b/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg @@ -114,7 +114,7 @@ Agents PilotSyncAgent { PollingTime = 600 - # Directory where the files can be moved. If running on the WebPortal, use /opt/dirac/webRoot/www/pilot + # Directory where the files can be moved. If running on the WebApp, use /opt/dirac/webRoot/www/pilot SaveDirectory = # List of locations where to upload the pilot files. Can be https://some.where, or DIRAC SE names. UploadLocations = diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py b/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py index e63346839ba..b456ad97923 100644 --- a/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py @@ -204,7 +204,7 @@ def pilotWrapperScript(pilotFilesCompressedEncodedDict=None, except Exception as x: print("tarfile failed with message %%s" %% repr(x), file=sys.stderr) logger.error("tarfile failed with message %%s" %% repr(x)) - logger.warn("Trying tar command (tar -xvf pilot.tar)") + logger.warning("Trying tar command (tar -xvf pilot.tar)") res = os.system("tar -xvf pilot.tar") if res: logger.error("tar failed with exit code %%d, giving up" %% int(res)) diff --git a/tests/Integration/Framework/Test_InstalledComponentsDB.py b/tests/Integration/Framework/Test_InstalledComponentsDB.py index ff48504beb4..ef2eceaeea9 100644 --- a/tests/Integration/Framework/Test_InstalledComponentsDB.py +++ b/tests/Integration/Framework/Test_InstalledComponentsDB.py @@ -48,15 +48,15 @@ def testComponents(self): """ # Create a sample component - result = self.client.addComponent({'System': 'Test', - 'Module': 'TestModule', + result = self.client.addComponent({'DIRACSystem': 'Test', + 'DIRACModule': 'TestModule', 'Type': 'TestingFeature'}) self.assertTrue(result['OK']) # Check if the component exists - result = self.client.getComponents({'System': 'Test', - 'Module': 'TestModule', + result = self.client.getComponents({'DIRACSystem': 'Test', + 'DIRACModule': 'TestModule', 'Type': 'TestingFeature'}, False, False) @@ -64,16 +64,16 @@ def testComponents(self): self.assertTrue(result['OK'] and len(result['Value']) > 0) # Update the fields of the created component - result = self.client.updateComponents({'System': 'Test', - 'Module': 'TestModule', + result = self.client.updateComponents({'DIRACSystem': 'Test', + 'DIRACModule': 'TestModule', 'Type': 'TestingFeature'}, - {'Module': 'NewTestModule'}) + {'DIRACModule': 'NewTestModule'}) self.assertTrue(result['OK']) # Check if the component with the modified fields exists - result = self.client.getComponents({'System': 'Test', - 'Module': 'NewTestModule', + result = self.client.getComponents({'DIRACSystem': 'Test', + 'DIRACModule': 'NewTestModule', 'Type': 'TestingFeature'}, False, False) @@ -81,15 +81,15 @@ def testComponents(self): self.assertTrue(result['OK'] and len(result['Value']) > 0) # Remove the Component - result = self.client.removeComponents({'System': 'Test', - 'Module': 'NewTestModule', + result = self.client.removeComponents({'DIRACSystem': 'Test', + 'DIRACModule': 'NewTestModule', 'Type': 'TestingFeature'}) self.assertTrue(result['OK']) # Check if the component was actually removed - result = self.client.getComponents({'System': 'Test', - 'Module': 'NewTestModule', + result = self.client.getComponents({'DIRACSystem': 'Test', + 'DIRACModule': 'NewTestModule', 'Type': 'TestingFeature'}, False, False) @@ -97,40 +97,40 @@ def testComponents(self): self.assertTrue(result['OK'] and len(result['Value']) <= 0) # Try to create an incomplete component - result = self.client.addComponent({'System': 'Test'}) + result = self.client.addComponent({'DIRACSystem': 'Test'}) self.assertFalse(result['OK']) # Multiple removal - self.client.addComponent({'System': 'Test', - 'Module': 'TestModule1', + self.client.addComponent({'DIRACSystem': 'Test', + 'DIRACModule': 'TestModule1', 'Type': 'TestingFeature1'}) - self.client.addComponent({'System': 'Test', - 'Module': 'TestModule2', + self.client.addComponent({'DIRACSystem': 'Test', + 'DIRACModule': 'TestModule2', 'Type': 'TestingFeature1'}) - self.client.addComponent({'System': 'Test', - 'Module': 'TestModule1', + self.client.addComponent({'DIRACSystem': 'Test', + 'DIRACModule': 'TestModule1', 'Type': 'TestingFeature2'}) - self.client.removeComponents({'System': 'Test', 'Module': 'TestModule1'}) + self.client.removeComponents({'DIRACSystem': 'Test', 'DIRACModule': 'TestModule1'}) - result = self.client.getComponents({'System': 'Test', - 'Module': 'TestModule2', + result = self.client.getComponents({'DIRACSystem': 'Test', + 'DIRACModule': 'TestModule2', 'Type': 'TestingFeature1'}, False, False) self.assertTrue(result['OK'] and len(result['Value']) >= 1) - result = self.client.getComponents({'System': 'Test', - 'Module': 'TestModule1'}, + result = self.client.getComponents({'DIRACSystem': 'Test', + 'DIRACModule': 'TestModule1'}, False, False) self.assertTrue(result['OK'] and len(result['Value']) <= 0) - self.client.removeComponents({'System': 'Test', - 'Module': 'TestModule2', + self.client.removeComponents({'DIRACSystem': 'Test', + 'DIRACModule': 'TestModule2', 'Type': 'TestingFeature1'}) self.assertTrue(result['OK']) @@ -221,8 +221,8 @@ def testInstallations(self): result = self.client.addInstallation({'InstallationTime': datetime.datetime.now(), 'UnInstallationTime': datetime.datetime.now(), 'Instance': 'TestInstallA111'}, - {'System': 'UnexistentSystem', - 'Module': 'UnexistentModule', + {'DIRACSystem': 'UnexistentSystem', + 'DIRACModule': 'UnexistentModule', 'Type': 'UnexistentType'}, {'HostName': 'fictional', 'CPU': 'TestCPU'}, @@ -232,8 +232,8 @@ def testInstallations(self): # Check if the installation exists result = self.client.getInstallations({'Instance': 'TestInstallA111'}, - {'System': 'UnexistentSystem', - 'Module': 'UnexistentModule', + {'DIRACSystem': 'UnexistentSystem', + 'DIRACModule': 'UnexistentModule', 'Type': 'UnexistentType'}, {'HostName': 'fictional', 'CPU': 'TestCPU'}, @@ -243,8 +243,8 @@ def testInstallations(self): # Update the fields of the created installation result = self.client.updateInstallations({'Instance': 'TestInstallA111'}, - {'System': 'UnexistentSystem', - 'Module': 'UnexistentModule', + {'DIRACSystem': 'UnexistentSystem', + 'DIRACModule': 'UnexistentModule', 'Type': 'UnexistentType'}, {'HostName': 'fictional', 'CPU': 'TestCPU'}, @@ -255,8 +255,8 @@ def testInstallations(self): # Check if the installation with the modified fields exists result = self.client.getInstallations({'Instance': 'TestInstallA222'}, - {'System': 'UnexistentSystem', - 'Module': 'UnexistentModule', + {'DIRACSystem': 'UnexistentSystem', + 'DIRACModule': 'UnexistentModule', 'Type': 'UnexistentType'}, {'HostName': 'fictional', 'CPU': 'TestCPU'}, @@ -266,8 +266,8 @@ def testInstallations(self): # Remove the Installation result = self.client.removeInstallations({'Instance': 'TestInstallA222'}, - {'System': 'UnexistentSystem', - 'Module': 'UnexistentModule', + {'DIRACSystem': 'UnexistentSystem', + 'DIRACModule': 'UnexistentModule', 'Type': 'UnexistentType'}, {'HostName': 'fictional', 'CPU': 'TestCPU'}) @@ -276,8 +276,8 @@ def testInstallations(self): # Check if the installation was actually removed result = self.client.getInstallations({'Instance': 'TestInstallA222'}, - {'System': 'UnexistentSystem', - 'Module': 'UnexistentModule', + {'DIRACSystem': 'UnexistentSystem', + 'DIRACModule': 'UnexistentModule', 'Type': 'UnexistentType'}, {'HostName': 'fictional', 'CPU': 'TestCPU'}, @@ -290,8 +290,8 @@ def testInstallations(self): {'InstallationTime': datetime.datetime.now(), 'UnInstallationTime': datetime.datetime.now(), 'Instance': 'TestInstallA333'}, - {'System': 'UnexistentSystem', - 'Module': 'UnexistentModule22A', + {'DIRACSystem': 'UnexistentSystem', + 'DIRACModule': 'UnexistentModule22A', 'Type': 'UnexistentType'}, {'HostName': 'fictional', 'CPU': 'TestCPU'}, @@ -304,8 +304,8 @@ def testInstallations(self): {'InstallationTime': datetime.datetime.now(), 'UnInstallationTime': datetime.datetime.now(), 'Instance': 'MultipleRemovalInstall1'}, - {'System': 'UnexistentSystem', - 'Module': 'UnexistentModule', + {'DIRACSystem': 'UnexistentSystem', + 'DIRACModule': 'UnexistentModule', 'Type': 'UnexistentType'}, {'HostName': 'fictional', 'CPU': 'TestCPU'}, @@ -314,8 +314,8 @@ def testInstallations(self): {'InstallationTime': datetime.datetime.now(), 'UnInstallationTime': datetime.datetime.now(), 'Instance': 'MultipleRemovalInstall2'}, - {'System': 'UnexistentSystem', - 'Module': 'UnexistentModule', + {'DIRACSystem': 'UnexistentSystem', + 'DIRACModule': 'UnexistentModule', 'Type': 'UnexistentType'}, {'HostName': 'fictional', 'CPU': 'TestCPU'}, @@ -324,8 +324,8 @@ def testInstallations(self): {'InstallationTime': datetime.datetime.now(), 'UnInstallationTime': datetime.datetime.now(), 'Instance': 'MultipleRemovalInstall3'}, - {'System': 'UnexistentSystem', - 'Module': 'UnexistentModule2', + {'DIRACSystem': 'UnexistentSystem', + 'DIRACModule': 'UnexistentModule2', 'Type': 'UnexistentType'}, {'HostName': 'fictional', 'CPU': 'TestCPU'}, @@ -341,35 +341,35 @@ def testInstallations(self): self.assertTrue(result['OK'] and len(result['Value']) == 2) self.client.removeInstallations({}, - {'Module': 'UnexistentModule'}, + {'DIRACModule': 'UnexistentModule'}, {}) result = self.client.getInstallations({}, - {'Module': 'UnexistentModule2'}, + {'DIRACModule': 'UnexistentModule2'}, {}, False) self.assertTrue(result['OK'] and len(result['Value']) >= 1) result = self.client.getInstallations({}, - {'Module': 'UnexistentModule'}, + {'DIRACModule': 'UnexistentModule'}, {}, False) self.assertTrue(result['OK'] and len(result['Value']) <= 0) self.client.removeInstallations({}, - {'Module': 'UnexistentModule2'}, + {'DIRACModule': 'UnexistentModule2'}, {}) self.assertTrue(result['OK']) # Clean up what we created self.client.removeHosts({'HostName': 'fictional', 'CPU': 'TestCPU'}) - self.client.removeComponents({'System': 'UnexistentSystem', - 'Module': 'UnexistentModule', + self.client.removeComponents({'DIRACSystem': 'UnexistentSystem', + 'DIRACModule': 'UnexistentModule', 'Type': 'UnexistentType'}) - self.client.removeComponents({'System': 'UnexistentSystem', - 'Module': 'UnexistentModule2', + self.client.removeComponents({'DIRACSystem': 'UnexistentSystem', + 'DIRACModule': 'UnexistentModule2', 'Type': 'UnexistentType'}) def testHostLogging(self):