diff --git a/release.notes b/release.notes index 3c10ba1f5ea..56ef333bf7d 100644 --- a/release.notes +++ b/release.notes @@ -2,8 +2,7 @@ *TransformationSystem -NEW: (#7170) -Implement a finer grained permissions model for Transformations. +NEW: (#7170) Implement a finer grained permissions model for Transformations. *DataManagementSystem diff --git a/src/DIRAC/Interfaces/API/DiracAdmin.py b/src/DIRAC/Interfaces/API/DiracAdmin.py index e3c14466310..54205ac7641 100755 --- a/src/DIRAC/Interfaces/API/DiracAdmin.py +++ b/src/DIRAC/Interfaces/API/DiracAdmin.py @@ -385,22 +385,6 @@ def getVOMSProxy(self, userDN, userGroup, vomsAttr=False, validity=43200, limite userDN, userGroup, limited=limited, requiredVOMSAttribute=vomsAttr, requiredTimeLeft=validity ) - ############################################################################# - def getPilotProxy(self, userDN, userGroup, validity=43200): - """Retrieves a pilot proxy with default 12hr validity and stores - this in a file in the local directory by default. - - Example usage: - - >>> gLogger.notice(diracAdmin.getVOMSProxy()) - {'OK': True, 'Value': } - - :return: S_OK,S_ERROR - - """ - - return gProxyManager.getPilotProxyFromDIRACGroup(userDN, userGroup, requiredTimeLeft=validity) - ############################################################################# def resetJob(self, jobID): """Reset a job or list of jobs in the WMS. This operation resets the reschedule diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py index 815347932a1..cc6701d2af4 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py @@ -1,8 +1,4 @@ #!/usr/bin/env python -######################################################################## -# File : dirac-admin-get-job-pilots -# Author : Stuart Paterson -######################################################################## """ Retrieve info about pilots that have matched a given Job @@ -10,32 +6,25 @@ $ dirac-admin-get-job-pilots 1848 {'https://marlb.in2p3.fr:9000/bqYViq6KrVgGfr6wwgT45Q': {'AccountingSent': 'False', 'BenchMark': 8.1799999999999997, - 'Broker': 'marwms.in2p3.fr', 'DestinationSite': 'lpsc-ce.in2p3.fr', 'GridSite': 'LCG.LPSC.fr', 'GridType': 'gLite', 'Jobs': [1848L], 'LastUpdateTime': datetime.datetime(2011, 2, 21, 12, 39, 10), 'OutputReady': 'True', - 'OwnerDN': '/O=GRID/C=FR/O=CNRS/OU=LPC/CN=Sebastien Guizard', 'OwnerGroup': '/biomed', - 'ParentID': 0L, - 'PilotID': 2247L, 'PilotJobReference': 'https://marlb.in2p3.fr:9000/biq6KT45Q', 'PilotStamp': '', 'Status': 'Done', 'SubmissionTime': datetime.datetime(2011, 2, 21, 12, 27, 52), 'TaskQueueID': 399L}} """ -# pylint: disable=wrong-import-position from DIRAC.Core.Base.Script import Script @Script() def main(): - # Registering arguments will automatically add their description to the help menu Script.registerArgument(["JobID: DIRAC ID of the Job"]) - # parseCommandLine show help when mandatory arguments are not specified or incorrect argument _, args = Script.parseCommandLine(ignoreErrors=True) from DIRAC import exit as DIRACExit @@ -58,8 +47,8 @@ def main(): errorList.append((job, result["Message"])) exitCode = 2 - for error in errorList: - print("ERROR %s: %s" % error) + for job, error in errorList: + print(f"ERROR for {job}: {error}") DIRACExit(exitCode) diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py index d5aefea5b68..b3cbaa7cd5c 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_info.py @@ -1,8 +1,4 @@ #!/usr/bin/env python -######################################################################## -# File : dirac-admin-get-pilot-info -# Author : Ricardo Graciani -######################################################################## """ Retrieve available info about the given pilot @@ -10,23 +6,18 @@ $ dirac-admin-get-pilot-info https://marlb.in2p3.fr:9000/26KCLKBFtxXKHF4_ZrQjkw {'https://marlb.in2p3.fr:9000/26KCLKBFtxXKHF4_ZrQjkw': {'AccountingSent': 'False', 'BenchMark': 0.0, - 'Broker': 'marwms.in2p3.fr', 'DestinationSite': 'cclcgceli01.in2p3.fr', 'GridSite': 'LCG.IN2P3.fr', 'GridType': 'gLite', 'LastUpdateTime': datetime.datetime(2011, 2, 21, 12, 49, 14), 'OutputReady': 'False', - 'OwnerDN': '/O=GRID/C=FR/O=CNRS/OU=LPC/CN=Sebastien Guizard', 'OwnerGroup': '/biomed', - 'ParentID': 0L, - 'PilotID': 2241L, 'PilotJobReference': 'https://marlb.in2p3.fr:9000/2KHFrQjkw', 'PilotStamp': '', 'Status': 'Done', 'SubmissionTime': datetime.datetime(2011, 2, 21, 12, 27, 52), 'TaskQueueID': 399L}} """ -# pylint: disable=wrong-import-position from DIRAC.Core.Base.Script import Script extendedPrint = False @@ -63,7 +54,6 @@ def main(): for key in [ "PilotJobReference", "Status", - "OwnerDN", "OwnerGroup", "SubmissionTime", "DestinationSite", @@ -85,10 +75,9 @@ def main(): diracAdmin.log.notice(f"{tab}Job ID: {jobID}") tab += " " for key in [ - "OwnerDN", - "OwnerGroup", "JobName", "Status", + "OwnerGroup", "StartExecTime", "LastUpdateTime", "EndExecTime", @@ -99,8 +88,8 @@ def main(): else: print(diracAdmin.pPrint.pformat({gridID: res})) - for error in errorList: - print("ERROR %s: %s" % error) + for job, error in errorList: + print(f"ERROR for {job}: {error}") DIRACExit(exitCode) diff --git a/src/DIRAC/ResourceStatusSystem/Command/PilotCommand.py b/src/DIRAC/ResourceStatusSystem/Command/PilotCommand.py index 62a98eb1286..e022898a53a 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/PilotCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/PilotCommand.py @@ -93,9 +93,9 @@ def doNew(self, masterParams=None): return S_ERROR(f'"{element}" is not Site nor Resource') if element == "Resource": - pilotsResultPivot = self.pilots.getGroupedPilotSummary({}, ["GridSite", "DestinationSite", "OwnerGroup"]) + pilotsResultPivot = self.pilots.getGroupedPilotSummary(["GridSite", "DestinationSite", "OwnerGroup"]) elif element == "Site": - pilotsResultPivot = self.pilots.getGroupedPilotSummary({}, ["GridSite", "OwnerGroup"]) + pilotsResultPivot = self.pilots.getGroupedPilotSummary(["GridSite", "OwnerGroup"]) else: # You should never see this error return S_ERROR(f'"{element}" is not Site nor Resource') diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/PilotStatusAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/PilotStatusAgent.py index 9743457febd..b77e29ed703 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/PilotStatusAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/PilotStatusAgent.py @@ -208,7 +208,7 @@ def _addPilotsAccountingReport(self, pilotsData): pA.setValueByKey("Site", "Unknown") pA.setValueByKey("GridCE", pData["DestinationSite"]) pA.setValueByKey("GridMiddleware", pData["GridType"]) - pA.setValueByKey("GridResourceBroker", pData["Broker"]) + pA.setValueByKey("GridResourceBroker", "DIRAC") pA.setValueByKey("GridStatus", pData["Status"]) if "Jobs" not in pData: pA.setValueByKey("Jobs", 0) diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py index 279410393bc..0490abb9319 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py @@ -841,7 +841,6 @@ def _addPilotTQReference(self, queue, taskQueueDict, pilotList, stampDict): result = self.pilotAgentsDB.addPilotTQReference( pilotsList, tqID, - self.pilotDN, self.pilotGroup, self.localhost, self.queueDict[queue]["CEType"], @@ -1201,7 +1200,6 @@ def _updatePilotStatusPerQueue(self, queue, proxy): "GridType": ceType, "GridSite": siteName, "Status": PilotStatus.PILOT_TRANSIENT_STATES, - "OwnerDN": self.pilotDN, "OwnerGroup": self.pilotGroup, } ) @@ -1356,7 +1354,7 @@ def sendPilotAccounting(self, pilotDict): pA.setValueByKey("Site", "Unknown") pA.setValueByKey("GridCE", pilotDict[pRef]["DestinationSite"]) pA.setValueByKey("GridMiddleware", pilotDict[pRef]["GridType"]) - pA.setValueByKey("GridResourceBroker", pilotDict[pRef]["Broker"]) + pA.setValueByKey("GridResourceBroker", "DIRAC") pA.setValueByKey("GridStatus", pilotDict[pRef]["Status"]) if "Jobs" not in pilotDict[pRef]: pA.setValueByKey("Jobs", 0) diff --git a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py index ddadfa0cecf..657114f6f19 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py @@ -23,7 +23,7 @@ import DIRAC.Core.Utilities.TimeUtilities as TimeUtilities from DIRAC import S_ERROR, S_OK -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername, getUsernameForDN, getVOForGroup +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping from DIRAC.Core.Base.DB import DB from DIRAC.Core.Utilities import DErrno @@ -38,9 +38,7 @@ def __init__(self, parentLogger=None): self.lock = threading.Lock() ########################################################################################## - def addPilotTQReference( - self, pilotRef, taskQueueID, ownerDN, ownerGroup, broker="Unknown", gridType="DIRAC", pilotStampDict={} - ): + def addPilotTQReference(self, pilotRef, taskQueueID, ownerGroup, gridType="DIRAC", pilotStampDict={}): """Add a new pilot job reference""" err = "PilotAgentsDB.addPilotTQReference: Failed to retrieve a new Id." @@ -50,16 +48,11 @@ def addPilotTQReference( if ref in pilotStampDict: stamp = pilotStampDict[ref] - res = self._escapeString(ownerDN) - if not res["OK"]: - return res - escapedOwnerDN = res["Value"] - req = ( - "INSERT INTO PilotAgents( PilotJobReference, TaskQueueID, OwnerDN, " - + "OwnerGroup, Broker, GridType, SubmissionTime, LastUpdateTime, Status, PilotStamp ) " - + "VALUES ('%s',%d,%s,'%s','%s','%s',UTC_TIMESTAMP(),UTC_TIMESTAMP(),'Submitted','%s')" - % (ref, int(taskQueueID), escapedOwnerDN, ownerGroup, broker, gridType, stamp) + "INSERT INTO PilotAgents " + + "(PilotJobReference, TaskQueueID, OwnerGroup, GridType, SubmissionTime, LastUpdateTime, Status, PilotStamp) " + + "VALUES ('%s',%d,'%s','%s',UTC_TIMESTAMP(),UTC_TIMESTAMP(),'Submitted','%s')" + % (ref, int(taskQueueID), ownerGroup, gridType, stamp) ) result = self._update(req) @@ -186,7 +179,7 @@ def countPilots(self, condDict, older=None, newer=None, timeStamp="SubmissionTim return S_OK(result["Value"][0][0]) ######################################################################################### - def getPilotGroups(self, groupList=["Status", "OwnerDN", "OwnerGroup", "GridType"], condDict={}): + def getPilotGroups(self, groupList=["Status", "OwnerGroup", "GridType"], condDict={}): """ Get all exisiting combinations of groupList Values """ @@ -269,20 +262,17 @@ def clearPilots(self, interval=30, aborted_interval=7): return S_OK(idList) ########################################################################################## - def getPilotInfo(self, pilotRef=False, parentId=False, conn=False, paramNames=[], pilotID=False): + def getPilotInfo(self, pilotRef=False, conn=False, paramNames=[], pilotID=False): """Get all the information for the pilot job reference or reference list""" parameters = ( [ "PilotJobReference", - "OwnerDN", "OwnerGroup", "GridType", - "Broker", "Status", "DestinationSite", "BenchMark", - "ParentID", "OutputReady", "AccountingSent", "SubmissionTime", @@ -299,7 +289,10 @@ def getPilotInfo(self, pilotRef=False, parentId=False, conn=False, paramNames=[] cmd = f"SELECT {', '.join(parameters)} FROM PilotAgents" condSQL = [] - for key, value in [("PilotJobReference", pilotRef), ("PilotID", pilotID), ("ParentID", parentId)]: + for key, value in [ + ("PilotJobReference", pilotRef), + ("PilotID", pilotID), + ]: resList = [] for v in value if isinstance(value, list) else [value] if value else []: result = self._escapeString(v) @@ -318,8 +311,6 @@ def getPilotInfo(self, pilotRef=False, parentId=False, conn=False, paramNames=[] msg = "No pilots found" if pilotRef: msg += f" for PilotJobReference(s): {pilotRef}" - if parentId: - msg += f" with parent id: {parentId}" return S_ERROR(DErrno.EWMSNOPILOT, msg) resDict = {} @@ -583,61 +574,16 @@ def getPilotSummary(self, startdate="", enddate=""): return S_OK(summary_dict) - # def getPilotSummaryShort( self, startTimeWindow = None, endTimeWindow = None, ce = '' ): - # """ - # Spin off the method getPilotSummary. It is doing things in such a way that - # do not make much sense. This method returns the pilots that were updated in the - # time window [ startTimeWindow, endTimeWindow ), if they are present. - # """ - # - # sqlSelect = 'SELECT DestinationSite,Status,count(Status) FROM PilotAgents' - # - # whereSelect = [] - # - # if startTimeWindow is not None: - # whereSelect.append( ' LastUpdateTime >= "%s"' % startTimeWindow ) - # if endTimeWindow is not None: - # whereSelect.append( ' LastUpdateTime < "%s"' % endTimeWindow ) - # if ce: - # whereSelect.append( ' DestinationSite = "%s"' % ce ) - # - # if whereSelect: - # sqlSelect += ' WHERE' - # sqlSelect += ' AND'.join( whereSelect ) - # - # sqlSelect += ' GROUP BY DestinationSite,Status' - # - # resSelect = self._query( sqlSelect ) - # if not resSelect[ 'OK' ]: - # return resSelect - # - # result = { 'Total' : collections.defaultdict( int ) } - # - # for row in resSelect[ 'Value' ]: - # - # ceName, statusName, statusCount = row - # - # if not ceName in result: - # result[ ceName ] = {} - # result[ ceName ][ statusName ] = int( statusCount ) - # - # result[ 'Total' ][ statusName ] += int( statusCount ) - # - # return S_OK( result ) - ########################################################################################## - def getGroupedPilotSummary(self, selectDict, columnList): + def getGroupedPilotSummary(self, columnList): """ The simplified pilot summary based on getPilotSummaryWeb method. It calculates pilot efficiency based on the same algorithm as in the Web version, basically takes into account Done and Aborted pilots only from the last day. The selection is done entirely in SQL. - :param dict selectDict: A dictionary to pass additional conditions to select statements, i.e. - it allows to define start time for Done and Aborted Pilots. Unused. :param list columnList: A list of column to consider when grouping to calculate efficiencies. e.g. ['GridSite', 'DestinationSite'] is used to calculate efficiencies - for sites and CEs. If we want to add an OwnerGroup it would be: - ['GridSite', 'DestinationSite', 'OwnerGroup']. + for sites and CEs. :return: S_OK/S_ERROR with a dict containing the ParameterNames and Records lists. """ @@ -1030,7 +976,7 @@ def getPilotSummaryWeb(self, selectDict, sortList, startItem, maxItems): def getPilotMonitorSelectors(self): """Get distinct values for the Pilot Monitor page selectors""" - paramNames = ["OwnerDN", "OwnerGroup", "GridType", "Broker", "Status", "DestinationSite", "GridSite"] + paramNames = ["OwnerGroup", "GridType", "Status", "DestinationSite", "GridSite"] resultDict = {} for param in paramNames: @@ -1040,14 +986,6 @@ def getPilotMonitorSelectors(self): else: resultDict = [] - if param == "OwnerDN": - userList = [] - for dn in result["Value"]: - resultUser = getUsernameForDN(dn) - if resultUser["OK"]: - userList.append(resultUser["Value"]) - resultDict["Owner"] = userList - return S_OK(resultDict) ########################################################################################## @@ -1057,16 +995,7 @@ def getPilotMonitorWeb(self, selectDict, sortList, startItem, maxItems): resultDict = {} if "LastUpdateTime" in selectDict: del selectDict["LastUpdateTime"] - if "Owner" in selectDict: - userList = selectDict["Owner"] - if not isinstance(userList, list): - userList = [userList] - dnList = [] - for uName in userList: - uList = getDNForUsername(uName)["Value"] - dnList += uList - selectDict["OwnerDN"] = dnList - del selectDict["Owner"] + startDate = selectDict.get("FromDate", None) if startDate: del selectDict["FromDate"] @@ -1108,14 +1037,11 @@ def getPilotMonitorWeb(self, selectDict, sortList, startItem, maxItems): paramNames = [ "PilotJobReference", - "OwnerDN", "OwnerGroup", "GridType", - "Broker", "Status", "DestinationSite", "BenchMark", - "ParentID", "SubmissionTime", "PilotID", "LastUpdateTime", diff --git a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql index afddcdd7b11..418ad4877d2 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql +++ b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql @@ -36,16 +36,13 @@ CREATE TABLE `PilotAgents` ( `Queue` VARCHAR(128) NOT NULL DEFAULT 'Unknown', `GridSite` VARCHAR(128) NOT NULL DEFAULT 'Unknown', `Broker` VARCHAR(128) NOT NULL DEFAULT 'Unknown', - `OwnerDN` VARCHAR(255) NOT NULL, `OwnerGroup` VARCHAR(128) NOT NULL, `GridType` VARCHAR(32) NOT NULL DEFAULT 'LCG', - `GridRequirements` TEXT, `BenchMark` DOUBLE NOT NULL DEFAULT 0.0, `SubmissionTime` DATETIME DEFAULT NULL, `LastUpdateTime` DATETIME DEFAULT NULL, `Status` VARCHAR(32) NOT NULL DEFAULT 'Unknown', `StatusReason` VARCHAR(255) NOT NULL DEFAULT 'Unknown', - `ParentID` INT(11) UNSIGNED NOT NULL DEFAULT 0, `OutputReady` ENUM('True','False') NOT NULL DEFAULT 'False', `AccountingSent` ENUM('True','False') NOT NULL DEFAULT 'False', PRIMARY KEY (`PilotID`), diff --git a/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py index bcf6fe5199d..4a5f891da79 100644 --- a/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py @@ -7,6 +7,8 @@ from DIRAC import S_OK, S_ERROR import DIRAC.Core.Utilities.TimeUtilities as TimeUtilities +from DIRAC.ConfigurationSystem.Client.Helpers import Registry +from DIRAC.Core.Utilities.Decorators import deprecated from DIRAC.Core.DISET.RequestHandler import RequestHandler from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getUsernameForDN, getDNForUsername @@ -70,14 +72,21 @@ def export_getCurrentPilotCounters(cls, attrDict={}): ########################################################################################## types_addPilotTQReference = [list, int, str, str] + @deprecated("Use addPilotTQRef") @classmethod def export_addPilotTQReference( cls, pilotRef, taskQueueID, ownerDN, ownerGroup, broker="Unknown", gridType="DIRAC", pilotStampDict={} ): """Add a new pilot job reference""" - return cls.pilotAgentsDB.addPilotTQReference( - pilotRef, taskQueueID, ownerDN, ownerGroup, broker, gridType, pilotStampDict - ) + + return cls.pilotAgentsDB.addPilotTQReference(pilotRef, taskQueueID, gridType, pilotStampDict) + + types_addPilotTQRef = [list, int, str] + + @classmethod + def export_addPilotTQRef(cls, pilotRef, taskQueueID, ownerGroup, gridType="DIRAC", pilotStampDict={}): + """Add a new pilot job reference""" + return cls.pilotAgentsDB.addPilotTQReference(pilotRef, taskQueueID, ownerGroup, gridType, pilotStampDict) ############################################################################## types_getPilotOutput = [str] @@ -96,7 +105,6 @@ def export_getPilotOutput(self, pilotReference): pilotDict = result["Value"][pilotReference] - ownerDN = pilotDict["OwnerDN"] group = pilotDict["OwnerGroup"] # FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files? @@ -108,7 +116,6 @@ def export_getPilotOutput(self, pilotReference): resultDict = {} resultDict["StdOut"] = stdout resultDict["StdErr"] = error - resultDict["OwnerDN"] = ownerDN resultDict["OwnerGroup"] = group resultDict["FileList"] = [] return S_OK(resultDict) @@ -146,7 +153,6 @@ def export_getPilotOutput(self, pilotReference): resultDict = {} resultDict["StdOut"] = stdout resultDict["StdErr"] = error - resultDict["OwnerDN"] = ownerDN resultDict["OwnerGroup"] = group resultDict["FileList"] = [] shutil.rmtree(ce.ceParameters["WorkingDirectory"]) @@ -259,19 +265,18 @@ def export_getPilotSummaryWeb(cls, selectDict, sortList, startItem, maxItems): return cls.pilotAgentsDB.getPilotSummaryWeb(selectDict, sortList, startItem, maxItems) ############################################################################## - types_getGroupedPilotSummary = [dict, list] + types_getGroupedPilotSummary = [list] @classmethod - def export_getGroupedPilotSummary(cls, selectDict, columnList): + def export_getGroupedPilotSummary(cls, columnList): """ Get pilot summary showing grouped by columns in columnList, all pilot states and pilot efficiencies in a single row. - :param selectDict: additional arguments to SELECT clause :param columnList: a list of columns to GROUP BY (less status column) :return: a dictionary containing column names and data records """ - return cls.pilotAgentsDB.getGroupedPilotSummary(selectDict, columnList) + return cls.pilotAgentsDB.getGroupedPilotSummary(columnList) ############################################################################## types_getPilots = [[str, int]] @@ -325,7 +330,7 @@ def export_killPilot(cls, pilotRefList): if isinstance(pilotRefList, str): pilotRefs = [pilotRefList] - # Regroup pilots per site and per ownerDN + # Regroup pilots per site pilotRefDict = {} for pilotReference in pilotRefs: result = cls.pilotAgentsDB.getPilotInfo(pilotReference) @@ -333,10 +338,8 @@ def export_killPilot(cls, pilotRefList): return S_ERROR("Failed to get info for pilot " + pilotReference) pilotDict = result["Value"][pilotReference] - ownerDN = pilotDict["OwnerDN"] - group = pilotDict["OwnerGroup"] queue = "@@@".join( - [ownerDN, group, pilotDict["GridSite"], pilotDict["DestinationSite"], pilotDict["Queue"]] + [pilotDict["OwnerGroup"], pilotDict["GridSite"], pilotDict["DestinationSite"], pilotDict["Queue"]] ) gridType = pilotDict["GridType"] pilotRefDict.setdefault(queue, {}) @@ -428,33 +431,13 @@ def export_getPilotStatistics(cls, attribute, selectDict): if endDate: del selectDict["ToDate"] - # Owner attribute is not part of PilotAgentsDB - # It has to be converted into a OwnerDN - owners = selectDict.get("Owner") - if owners: - ownerDNs = [] - for owner in owners: - result = getDNForUsername(owner) - if not result["OK"]: - return result - ownerDNs.append(result["Value"]) - - selectDict["OwnerDN"] = ownerDNs - del selectDict["Owner"] - result = cls.pilotAgentsDB.getCounters( "PilotAgents", [attribute], selectDict, newer=startDate, older=endDate, timeStamp="LastUpdateTime" ) statistics = {} if result["OK"]: for status, count in result["Value"]: - if "OwnerDN" in status: - userName = getUsernameForDN(status["OwnerDN"]) - if userName["OK"]: - status["OwnerDN"] = userName["Value"] - statistics[status["OwnerDN"]] = count - else: - statistics[status[attribute]] = count + statistics[status[attribute]] = count return S_OK(statistics) diff --git a/src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py b/src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py index e18ec02cbbe..ccbe6713346 100644 --- a/src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py @@ -6,7 +6,8 @@ from DIRAC import S_OK, S_ERROR, gLogger, gConfig from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getQueue -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getGroupOption, getUsernameForDN +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getGroupOption, getUsernameForDN, getVOForGroup +from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory @@ -111,7 +112,7 @@ def killPilotsInQueues(pilotRefDict): ceFactory = ComputingElementFactory() for key, pilotDict in pilotRefDict.items(): - owner, group, site, ce, queue = key.split("@@@") + pilotGroup, site, ce, queue = key.split("@@@") result = getQueue(site, ce, queue) if not result["OK"]: return result @@ -122,13 +123,20 @@ def killPilotsInQueues(pilotRefDict): return result ce = result["Value"] - group = getGroupOption(group, "VOMSRole", group) - ret = gProxyManager.getPilotProxyFromVOMSGroup(owner, group) - if not ret["OK"]: - gLogger.error("Could not get proxy:", f"User '{owner}' Group '{group}' : {ret['Message']}") - return S_ERROR("Failed to get the pilot's owner proxy") - proxy = ret["Value"] - ce.setProxy(proxy) + pilotDN = Operations(vo=getVOForGroup(pilotGroup)).getValue("Pilot/GenericPilotDN") + + if pilotGroup and pilotDN: + res = getUsernameForDN(pilotDN) + if not res["OK"]: + return res + owner = res["Value"] + group = getGroupOption(pilotGroup, "VOMSRole", pilotGroup) + ret = gProxyManager.getPilotProxyFromVOMSGroup(owner, group) + if not ret["OK"]: + gLogger.error("Could not get proxy:", f"User '{owner}' Group '{group}' : {ret['Message']}") + return S_ERROR("Failed to get the pilot's owner proxy") + proxy = ret["Value"] + ce.setProxy(proxy) pilotList = pilotDict["PilotList"] result = ce.killJob(pilotList) diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_add_pilot.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_add_pilot.py index 7b2b55d3b7e..72ba14a3c17 100644 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_add_pilot.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_add_pilot.py @@ -8,7 +8,7 @@ This script is here to solve specifically this issue, even though it can be used for other things too. Example: - $ dirac-admin-add-pilot htcondor:123456 user_DN user_group DIRAC A11D8D2E-60F8-17A6-5520-E2276F41 --Status=Running + $ dirac-admin-add-pilot htcondor:123456 group DIRAC A11D8D2E-60F8-17A6-5520-E2276F41 --Status=Running """ @@ -67,7 +67,6 @@ def main(): Script.registerSwitches(params.switches) Script.registerArgument("pilotRef: pilot reference") - Script.registerArgument("ownerDN: pilot owner DN") Script.registerArgument("ownerGroup: pilot owner group") Script.registerArgument("gridType: grid type") Script.registerArgument("pilotStamp: DIRAC pilot stamp") @@ -75,7 +74,7 @@ def main(): Script.parseCommandLine(ignoreErrors=False) # Get grouped positional arguments - pilotRef, ownerDN, ownerGroup, gridType, pilotStamp = Script.getPositionalArgs(group=True) + pilotRef, ownerGroup, gridType, pilotStamp = Script.getPositionalArgs(group=True) # Import the required DIRAC modules from DIRAC.Core.Utilities import DErrno @@ -89,9 +88,7 @@ def main(): if not DErrno.cmpError(res, DErrno.EWMSNOPILOT): gLogger.error(res["Message"]) DIRACExit(1) - res = pmc.addPilotTQReference( - [pilotRef], params.taskQueueID, ownerDN, ownerGroup, "Unknown", gridType, {pilotRef: pilotStamp} - ) + res = pmc.addPilotTQRef([pilotRef], params.taskQueueID, ownerGroup, gridType, {pilotRef: pilotStamp}) if not res["OK"]: gLogger.error(res["Message"]) DIRACExit(1) diff --git a/tests/Integration/WorkloadManagementSystem/Test_PilotAgentsDB.py b/tests/Integration/WorkloadManagementSystem/Test_PilotAgentsDB.py index 4d14222c424..91a21ac548a 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_PilotAgentsDB.py +++ b/tests/Integration/WorkloadManagementSystem/Test_PilotAgentsDB.py @@ -39,7 +39,6 @@ def preparePilots(stateCount, testSite, testCE, testGroup): res = paDB.addPilotTQReference( pilotRef, 123, - "ownerDN", testGroup, ) assert res["OK"] is True, res["Message"] @@ -81,12 +80,7 @@ def cleanUpPilots(pilotRef): def test_basic(): """usual insert/verify""" - res = paDB.addPilotTQReference( - ["pilotRef"], - 123, - "ownerDN", - "ownerGroup", - ) + res = paDB.addPilotTQReference(["pilotRef"], 123, "ownerGroup") assert res["OK"] is True res = paDB.deletePilot("pilotRef") @@ -103,16 +97,14 @@ def test_getGroupedPilotSummary(mocked_fcn): """ stateCount = [10, 50, 7, 3, 12, 8, 6, 4] testGroup = "ownerGroup" - testGroupVO = "ownerGroupVO" testCE = "TestCE" testSite = "TestSite" mocked_fcn.return_value = "ownerGroupVO" pilotRef = preparePilots(stateCount, testSite, testCE, testGroup) - selectDict = {} columnList = ["GridSite", "DestinationSite", "OwnerGroup"] - res = paDB.getGroupedPilotSummary(selectDict, columnList) + res = paDB.getGroupedPilotSummary(columnList) cleanUpPilots(pilotRef) expectedParameterList = [ @@ -146,7 +138,6 @@ def test_getGroupedPilotSummary(mocked_fcn): assert len(record) == len(expectedParameterList) assert record[0] == testSite assert record[1] == testCE - assert record[2] == testGroupVO # pilot state counts: for i, entry in enumerate(record[3:11]): diff --git a/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py b/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py index 686342047e2..f97bf7be961 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py +++ b/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py @@ -27,7 +27,7 @@ def test_PilotsDB(): for jobID in ["aPilot", "anotherPilot"]: pilots.deletePilots(jobID) - res = pilots.addPilotTQReference(["aPilot"], 1, "/a/ownerDN", "a/owner/Group") + res = pilots.addPilotTQRef(["aPilot"], 1, "a/owner/Group") assert res["OK"], res["Message"] res = pilots.getCurrentPilotCounters({}) assert res["OK"], res["Message"] @@ -38,14 +38,13 @@ def test_PilotsDB(): assert res["OK"], res["Message"] assert res["Value"] == {} - res = pilots.addPilotTQReference(["anotherPilot"], 1, "/a/ownerDN", "a/owner/Group") + res = pilots.addPilotTQRef(["anotherPilot"], 1, "a/owner/Group") assert res["OK"], res["Message"] res = pilots.storePilotOutput("anotherPilot", "This is an output", "this is an error") assert res["OK"], res["Message"] res = pilots.getPilotOutput("anotherPilot") assert res["OK"], res["Message"] assert res["Value"] == { - "OwnerDN": "/a/ownerDN", "OwnerGroup": "a/owner/Group", "StdErr": "this is an error", "FileList": [], @@ -70,11 +69,8 @@ def test_PilotsDB(): "GridType": ["DIRAC"], "OwnerGroup": ["a/owner/Group"], "DestinationSite": ["NotAssigned"], - "Broker": ["Unknown"], "Status": ["Submitted"], - "OwnerDN": ["/a/ownerDN"], "GridSite": ["Unknown"], - "Owner": [], } res = pilots.getPilotSummaryWeb({}, [], 0, 100) assert res["OK"], res["Message"]