From f5c9c73e85412dd7afb2f5c094949d768b5eb159 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Thu, 11 Jun 2020 18:11:45 +0200 Subject: [PATCH 01/44] M2: close the socket after dereferencing the Connection instance --- .../private/Transports/M2SSLTransport.py | 84 +++++++++++++++++-- 1 file changed, 75 insertions(+), 9 deletions(-) diff --git a/Core/DISET/private/Transports/M2SSLTransport.py b/Core/DISET/private/Transports/M2SSLTransport.py index 9def1d7fd63..fde34b9fbb3 100755 --- a/Core/DISET/private/Transports/M2SSLTransport.py +++ b/Core/DISET/private/Transports/M2SSLTransport.py @@ -45,6 +45,9 @@ def __init__(self, *args, **kwargs): as for other transports. If ctx is specified (as an instance of SSL.Context) then use that rather than creating a new context. """ + # The thread init of M2Crypto is not really thread safe. + # So we put it a second time + M2Threading.init() self.remoteAddress = None self.peerCredentials = {} self.__timeout = 1 @@ -98,9 +101,7 @@ def initAsClient(self): error = "%s:%s" % (e, repr(e)) if self.oSocket is not None: - self.oSocket.close() - self.oSocket.socket.close() - self.oSocket = None + self.close() return S_ERROR(error) @@ -125,20 +126,85 @@ def initAsServer(self): return S_OK() def close(self): + # pylint: disable=line-too-long """ Close this socket. """ + if self.oSocket: + # TL;DR: + # Do NOT touch that method + # # Surprisingly (to me at least), M2Crypto does not close - # the socket when calling SSL.Connection.close - # It only does it when the garbage collector kicks in - # We have to manually close it here, otherwise the connections - # will hang forever + # the underlying socket when calling SSL.Connection.close + # It only does it when the garbage collector kicks in (see ~M2Crypto.SSL.Connection.Connection.__del__) + # If the socket is not closed, the connection may hang forever. + # + # Thus, we are setting self.oSocket to None to allow the GC to do the work, but since we are not sure + # that it will run, we anyway force the connection to be closed + # + # However, we should close the underlying socket only after SSL was shutdown properly. + # This is because OpenSSL `ssl3_shutdown` (see callstack below) may still read some data + # (see https://github.com/openssl/openssl/blob/master/ssl/s3_lib.c#L4509):: + # + # + # 1 0x00007fffe9d48fc0 in sock_read () from /lib/libcrypto.so.1.0.0 + # 2 0x00007fffe9d46e83 in BIO_read () from /lib/libcrypto.so.1.0.0 + # 3 0x00007fffe9eab9dd in ssl3_read_n () from /lib/libssl.so.1.0.0 + # 4 0x00007fffe9ead216 in ssl3_read_bytes () from /lib/libssl.so.1.0.0 + # 5 0x00007fffe9ea999c in ssl3_shutdown () from /lib/libssl.so.1.0.0 + # 6 0x00007fffe9ed4f93 in ssl_free () from /lib/libssl.so.1.0.0 + # 7 0x00007fffe9d46d5b in BIO_free () from /lib/libcrypto.so.1.0.0 + # 8 0x00007fffe9f30a96 in bio_free (bio=0x5555556f3200) at SWIG/_m2crypto_wrap.c:5008 + # 9 0x00007fffe9f30b1e in _wrap_bio_free (self=, args=) at SWIG/_m2crypto_wrap.c + # + # We unfortunately have no way to force that order, and there is a risk of deadlock + # when running in a multi threaded environment like the agents:: + # + # Thread A opens socket, gets FD = 111 + # Thread A works on it + # Thread A closes FD 111 (underlying socket.close()) + # Thread B opens socket, gets FD = 111 + # Thread A calls read on FD=111 from ssl3_shutdown + # + # This is illustrated on the strace below:: + # + # 26461 14:25:15.266692 write(111]:42688->[]:9140]>, + # "blabla", 37 + # 26464 14:25:15.266857 <... connect resumed>) = 0 <0.000195> + # 26464 14:25:15.267023 getsockname(120:44252->188.185.84.86:9140]>, + # 26461 14:25:15.267176 <... write resumed>) = 37 <0.000453> + # 26464 14:25:15.267425 <... getsockname resumed>{sa_family=AF_INET, sin_port=htons(44252), + # sin_addr=inet_addr("")}, [28->16]) = 0 <0.000292> + # 26461 14:25:15.267466 close(111]:42688->[]:9140]> + # 26464 14:25:15.267637 close(120:44252->188.185.84.86:9140]> + # 26464 14:25:15.267738 <... close resumed>) = 0 <0.000086> + # 26461 14:25:15.267768 <... close resumed>) = 0 <0.000285> + # 26464 14:25:15.267827 socket(AF_INET6, SOCK_DGRAM|SOCK_CLOEXEC, IPPROTO_IP + # 26461 14:25:15.267888 futex(0x21f8620, FUTEX_WAKE_PRIVATE, 1 + # 26464 14:25:15.267976 <... socket resumed>) = 111 <0.000138> + # 26461 14:25:15.268092 <... futex resumed>) = 1 <0.000196> + # 26464 14:25:15.268195 connect(111, + # {sa_family=AF_INET6, sin6_port=htons(9140), + # inet_pton(AF_INET6, "", &sin6_addr), + # sin6_flowinfo=htonl(0), sin6_scope_id=0 + # }, 28 + # 26461 14:25:15.268294 read(111]:42480->[]:9140]>, + # 26464 14:25:15.268503 <... connect resumed>) = 0 <0.000217> + # 26464 14:25:15.268673 getsockname(111]:42480->[]:9140]>, + # 26464 14:25:15.268862 <... getsockname resumed>{sa_family=AF_INET6, sin6_port=htons(42480), + # inet_pton(AF_INET6, "", &sin6_addr), sin6_flowinfo=htonl(0), sin6_scope_id= + # 0}, [28]) = 0 <0.000168> + # 26464 14:25:15.269048 + # close(111]:42480->[]:9140]> + # + self.oSocket.close() - self.oSocket.socket.close() - # del self.oSocket + underlyingSocket = self.oSocket.socket self.oSocket = None + underlyingSocket.close() return S_OK() def renewServerContext(self): + # pylint: disable=line-too-long """ Renews the server context. This reloads the certificates and re-initialises the SSL context. From d72ae88a5b3911833d492870e9f3227e3559fb61 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 12 Jun 2020 10:59:46 +0200 Subject: [PATCH 02/44] CI: disable coverage in pytest (because Test_PoolCE is too sensitive to timing issues) --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 74fb4a3f68c..81dd39cb17f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,4 +5,4 @@ python_files=Test_*.py assert*.py # The reason here is that we do nasty things with the pythonpath # in order to make sure that M2Crypto and pyGSI do not step # on each other's feet -addopts = -rx -v --color=yes --showlocals --tb=long --ignore=tests --ignore=Core/Security/test --cov=. --cov-report term-missing +addopts = -rx -v --color=yes --showlocals --tb=long --ignore=tests --ignore=Core/Security/test From e32252be82cbe84b1ab813909a31f9d9f44f05d3 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 15 Jun 2020 10:32:42 +0200 Subject: [PATCH 03/44] RMS/DMS: cancel FTS operations if the matching RMS request does not exist anymore --- DataManagementSystem/Agent/FTS3Agent.py | 38 +- RequestManagementSystem/DB/RequestDB.py | 735 +++++++++++------------- 2 files changed, 375 insertions(+), 398 deletions(-) diff --git a/DataManagementSystem/Agent/FTS3Agent.py b/DataManagementSystem/Agent/FTS3Agent.py index 47fef84be99..1adeaca1eb6 100644 --- a/DataManagementSystem/Agent/FTS3Agent.py +++ b/DataManagementSystem/Agent/FTS3Agent.py @@ -354,24 +354,36 @@ def _treatOperation(self, operation): continueOperationProcessing = True # Check the status of the associated RMS Request. - # If it is canceled then we will not create new FTS3Jobs, and mark + # If it is canceled or does not exist anymore then we will not create new FTS3Jobs, and mark # this as FTS3Operation canceled. if operation.rmsReqID: res = ReqClient().getRequestStatus(operation.rmsReqID) if not res['OK']: - log.error("Could not get request status", res) - return operation, res - rmsReqStatus = res['Value'] - - if rmsReqStatus == 'Canceled': - log.info( - "The RMS Request is canceled, canceling the FTS3Operation", - "rmsReqID: %s, FTS3OperationID: %s" % - (operation.rmsReqID, - operation.operationID)) - operation.status = 'Canceled' - continueOperationProcessing = False + # If the Request does not exist anymore + if cmpError(res, errno.ENOENT): + log.info( + "The RMS Request does not exist anymore, canceling the FTS3Operation", + "rmsReqID: %s, FTS3OperationID: %s" % + (operation.rmsReqID, + operation.operationID)) + operation.status = 'Canceled' + continueOperationProcessing = False + else: + log.error("Could not get request status", res) + return operation, res + + else: + rmsReqStatus = res['Value'] + + if rmsReqStatus == 'Canceled': + log.info( + "The RMS Request is canceled, canceling the FTS3Operation", + "rmsReqID: %s, FTS3OperationID: %s" % + (operation.rmsReqID, + operation.operationID)) + operation.status = 'Canceled' + continueOperationProcessing = False if continueOperationProcessing: res = operation.prepareNewJobs( diff --git a/RequestManagementSystem/DB/RequestDB.py b/RequestManagementSystem/DB/RequestDB.py index e84a1a8360c..4f9a9e1b813 100644 --- a/RequestManagementSystem/DB/RequestDB.py +++ b/RequestManagementSystem/DB/RequestDB.py @@ -16,6 +16,7 @@ db holding Request, Operation and File """ +import errno import random import datetime @@ -24,7 +25,7 @@ from sqlalchemy.orm import relationship, backref, sessionmaker, joinedload_all, mapper from sqlalchemy.sql import update from sqlalchemy import create_engine, func, Table, Column, MetaData, ForeignKey, \ - Integer, String, DateTime, Enum, BLOB, BigInteger, distinct + Integer, String, DateTime, Enum, BLOB, BigInteger, distinct # # from DIRAC from DIRAC import S_OK, S_ERROR, gLogger @@ -34,7 +35,6 @@ from DIRAC.ConfigurationSystem.Client.Utilities import getDBParameters - __RCSID__ = "$Id $" @@ -44,234 +44,225 @@ # Description of the file table -fileTable = Table( 'File', metadata, - Column( 'FileID', Integer, primary_key = True ), - Column( 'OperationID', Integer, - ForeignKey( 'Operation.OperationID', ondelete = 'CASCADE' ), - nullable = False ), - Column( 'Status', Enum( 'Waiting', 'Done', 'Failed', 'Scheduled' ), server_default = 'Waiting' ), - Column( 'LFN', String( 255 ), index = True ), - Column( 'PFN', String( 255 ) ), - Column( 'ChecksumType', Enum( 'ADLER32', 'MD5', 'SHA1', '' ), server_default = '' ), - Column( 'Checksum', String( 255 ) ), - Column( 'GUID', String( 36 ) ), - Column( 'Size', BigInteger ), - Column( 'Attempt', Integer ), - Column( 'Error', String( 255 ) ), - mysql_engine = 'InnoDB' ) +fileTable = Table('File', metadata, + Column('FileID', Integer, primary_key=True), + Column('OperationID', Integer, + ForeignKey('Operation.OperationID', ondelete='CASCADE'), + nullable=False), + Column('Status', Enum('Waiting', 'Done', 'Failed', 'Scheduled'), server_default='Waiting'), + Column('LFN', String(255), index=True), + Column('PFN', String(255)), + Column('ChecksumType', Enum('ADLER32', 'MD5', 'SHA1', ''), server_default=''), + Column('Checksum', String(255)), + Column('GUID', String(36)), + Column('Size', BigInteger), + Column('Attempt', Integer), + Column('Error', String(255)), + mysql_engine='InnoDB') # Map the File object to the fileTable, with a few special attributes -mapper( File, fileTable, properties = {'_Status': fileTable.c.Status, - '_LFN': fileTable.c.LFN, - '_ChecksumType' : fileTable.c.ChecksumType, - '_GUID' : fileTable.c.GUID} ) +mapper(File, fileTable, properties={'_Status': fileTable.c.Status, + '_LFN': fileTable.c.LFN, + '_ChecksumType': fileTable.c.ChecksumType, + '_GUID': fileTable.c.GUID}) # Description of the Operation table -operationTable = Table( 'Operation', metadata, - Column( 'TargetSE', String( 255 ) ), - Column( 'CreationTime', DateTime ), - Column( 'SourceSE', String( 255 ) ), - Column( 'Arguments', BLOB ), - Column( 'Error', String( 255 ) ), - Column( 'Type', String( 64 ), nullable = False ), - Column( 'Order', Integer, nullable = False ), - Column( 'Status', - Enum( 'Waiting', 'Assigned', 'Queued', 'Done', 'Failed', 'Canceled', 'Scheduled' ), - server_default = 'Queued' ), - Column( 'LastUpdate', DateTime ), - Column( 'SubmitTime', DateTime ), - Column( 'Catalog', String( 255 ) ), - Column( 'OperationID', Integer, primary_key = True ), - Column( 'RequestID', Integer, - ForeignKey( 'Request.RequestID', ondelete = 'CASCADE' ), - nullable = False ), - mysql_engine = 'InnoDB' ) +operationTable = Table('Operation', metadata, + Column('TargetSE', String(255)), + Column('CreationTime', DateTime), + Column('SourceSE', String(255)), + Column('Arguments', BLOB), + Column('Error', String(255)), + Column('Type', String(64), nullable=False), + Column('Order', Integer, nullable=False), + Column('Status', + Enum('Waiting', 'Assigned', 'Queued', 'Done', 'Failed', 'Canceled', 'Scheduled'), + server_default='Queued'), + Column('LastUpdate', DateTime), + Column('SubmitTime', DateTime), + Column('Catalog', String(255)), + Column('OperationID', Integer, primary_key=True), + Column('RequestID', Integer, + ForeignKey('Request.RequestID', ondelete='CASCADE'), + nullable=False), + mysql_engine='InnoDB') # Map the Operation object to the operationTable, with a few special attributes -mapper( Operation, operationTable, properties = {'_CreationTime': operationTable.c.CreationTime, - '_Order': operationTable.c.Order, - '_Status': operationTable.c.Status, - '_LastUpdate': operationTable.c.LastUpdate, - '_SubmitTime': operationTable.c.SubmitTime, - '_Catalog': operationTable.c.Catalog, - '__files__':relationship( File, - backref = backref( '_parent', lazy = 'immediate' ), - lazy = 'immediate', - passive_deletes = True, - cascade = "all, delete-orphan" )} ) +mapper(Operation, operationTable, properties={'_CreationTime': operationTable.c.CreationTime, + '_Order': operationTable.c.Order, + '_Status': operationTable.c.Status, + '_LastUpdate': operationTable.c.LastUpdate, + '_SubmitTime': operationTable.c.SubmitTime, + '_Catalog': operationTable.c.Catalog, + '__files__': relationship(File, + backref=backref('_parent', lazy='immediate'), + lazy='immediate', + passive_deletes=True, + cascade="all, delete-orphan")}) # Description of the Request Table -requestTable = Table( 'Request', metadata, - Column( 'DIRACSetup', String( 32 ) ), - Column( 'CreationTime', DateTime ), - Column( 'JobID', Integer, server_default = '0' ), - Column( 'OwnerDN', String( 255 ) ), - Column( 'RequestName', String( 255 ), nullable = False ), - Column( 'Error', String( 255 ) ), - Column( 'Status', - Enum( 'Waiting', 'Assigned', 'Done', 'Failed', 'Canceled', 'Scheduled' ), - server_default = 'Waiting' ), - Column( 'LastUpdate', DateTime ), - Column( 'OwnerGroup', String( 32 ) ), - Column( 'SubmitTime', DateTime ), - Column( 'RequestID', Integer, primary_key = True ), - Column( 'SourceComponent', BLOB ), - Column( 'NotBefore', DateTime ), - mysql_engine = 'InnoDB' ) +requestTable = Table('Request', metadata, + Column('DIRACSetup', String(32)), + Column('CreationTime', DateTime), + Column('JobID', Integer, server_default='0'), + Column('OwnerDN', String(255)), + Column('RequestName', String(255), nullable=False), + Column('Error', String(255)), + Column('Status', + Enum('Waiting', 'Assigned', 'Done', 'Failed', 'Canceled', 'Scheduled'), + server_default='Waiting'), + Column('LastUpdate', DateTime), + Column('OwnerGroup', String(32)), + Column('SubmitTime', DateTime), + Column('RequestID', Integer, primary_key=True), + Column('SourceComponent', BLOB), + Column('NotBefore', DateTime), + mysql_engine='InnoDB') # Map the Request object to the requestTable, with a few special attributes -mapper( Request, requestTable, properties = {'_CreationTime': requestTable.c.CreationTime, - '_Status': requestTable.c.Status, - '_LastUpdate': requestTable.c.LastUpdate, - '_SubmitTime': requestTable.c.SubmitTime, - '_NotBefore': requestTable.c.NotBefore, - '__operations__' : relationship( Operation, - backref = backref( '_parent', - lazy = 'immediate' ), - order_by = operationTable.c.Order, - lazy = 'immediate', - passive_deletes = True, - cascade = "all, delete-orphan" )} ) - +mapper(Request, requestTable, properties={'_CreationTime': requestTable.c.CreationTime, + '_Status': requestTable.c.Status, + '_LastUpdate': requestTable.c.LastUpdate, + '_SubmitTime': requestTable.c.SubmitTime, + '_NotBefore': requestTable.c.NotBefore, + '__operations__': relationship(Operation, + backref=backref('_parent', + lazy='immediate'), + order_by=operationTable.c.Order, + lazy='immediate', + passive_deletes=True, + cascade="all, delete-orphan")}) ######################################################################## -class RequestDB( object ): +class RequestDB(object): """ .. class:: RequestDB db holding requests """ - - def __getDBConnectionInfo( self, fullname ): + def __getDBConnectionInfo(self, fullname): """ Collect from the CS all the info needed to connect to the DB. This should be in a base class eventually """ - result = getDBParameters( fullname ) - if not result[ 'OK' ]: - raise Exception( 'Cannot get database parameters: %s' % result[ 'Message' ] ) + result = getDBParameters(fullname) + if not result['OK']: + raise Exception('Cannot get database parameters: %s' % result['Message']) - dbParameters = result[ 'Value' ] - self.dbHost = dbParameters[ 'Host' ] - self.dbPort = dbParameters[ 'Port' ] - self.dbUser = dbParameters[ 'User' ] - self.dbPass = dbParameters[ 'Password' ] - self.dbName = dbParameters[ 'DBName' ] + dbParameters = result['Value'] + self.dbHost = dbParameters['Host'] + self.dbPort = dbParameters['Port'] + self.dbUser = dbParameters['User'] + self.dbPass = dbParameters['Password'] + self.dbName = dbParameters['DBName'] - - def __init__( self ): + def __init__(self): """c'tor :param self: self reference """ - self.log = gLogger.getSubLogger( 'RequestDB' ) + self.log = gLogger.getSubLogger('RequestDB') # Initialize the connection info - self.__getDBConnectionInfo( 'RequestManagement/ReqDB' ) - - + self.__getDBConnectionInfo('RequestManagement/ReqDB') - runDebug = ( gLogger.getLevel() == 'DEBUG' ) + runDebug = (gLogger.getLevel() == 'DEBUG') self.engine = create_engine('mysql://%s:%s@%s:%s/%s' % (self.dbUser, self.dbPass, self.dbHost, self.dbPort, self.dbName), echo=runDebug, pool_recycle=3600) metadata.bind = self.engine - self.DBSession = sessionmaker( bind = self.engine ) + self.DBSession = sessionmaker(bind=self.engine) - - def createTables( self ): + def createTables(self): """ create tables """ try: - metadata.create_all( self.engine ) + metadata.create_all(self.engine) except Exception as e: - return S_ERROR( e ) + return S_ERROR(e) return S_OK() - - def cancelRequest( self, requestID ): + def cancelRequest(self, requestID): session = self.DBSession() try: - updateRet = session.execute( update( Request )\ - .where( Request.RequestID == requestID )\ - .values( {Request._Status : 'Canceled', - Request._LastUpdate : datetime.datetime.utcnow()\ - .strftime( Request._datetimeFormat )} ) ) + updateRet = session.execute(update(Request) + .where(Request.RequestID == requestID) + .values({Request._Status: 'Canceled', + Request._LastUpdate: datetime.datetime.utcnow() + .strftime(Request._datetimeFormat)})) session.commit() # No row was changed if not updateRet.rowcount: - return S_ERROR( "No such request %s" % requestID ) + return S_ERROR("No such request %s" % requestID) return S_OK() except Exception as e: session.rollback() - self.log.exception( "cancelRequest: unexpected exception", lException = e ) - return S_ERROR( "cancelRequest: unexpected exception %s" % e ) + self.log.exception("cancelRequest: unexpected exception", lException=e) + return S_ERROR("cancelRequest: unexpected exception %s" % e) finally: session.close() - - def putRequest( self, request ): + def putRequest(self, request): """ update or insert request into db :param ~Request.Request request: Request instance """ - session = self.DBSession( expire_on_commit = False ) + session = self.DBSession(expire_on_commit=False) try: try: - if hasattr( request, 'RequestID' ): + if hasattr(request, 'RequestID'): - status = session.query( Request._Status )\ - .filter( Request.RequestID == request.RequestID )\ + status = session.query(Request._Status)\ + .filter(Request.RequestID == request.RequestID)\ .one() if status[0] == 'Canceled': - self.log.info( "Request %s(%s) was canceled, don't put it back" % ( request.RequestID, request.RequestName ) ) - return S_OK( request.RequestID ) + self.log.info("Request %s(%s) was canceled, don't put it back" % (request.RequestID, request.RequestName)) + return S_OK(request.RequestID) - except NoResultFound, e: + except NoResultFound as e: pass # Since the object request is not attached to the session, we merge it to have an update # instead of an insert with duplicate primary key - request = session.merge( request ) - session.add( request ) + request = session.merge(request) + session.add(request) session.commit() session.expunge_all() - return S_OK( request.RequestID ) + return S_OK(request.RequestID) except Exception as e: session.rollback() - self.log.exception( "putRequest: unexpected exception", lException = e ) - return S_ERROR( "putRequest: unexpected exception %s" % e ) + self.log.exception("putRequest: unexpected exception", lException=e) + return S_ERROR("putRequest: unexpected exception %s" % e) finally: session.close() - - def getScheduledRequest( self, operationID ): + def getScheduledRequest(self, operationID): session = self.DBSession() try: - requestID = session.query( Request.RequestID )\ - .join( Request.__operations__ )\ - .filter( Operation.OperationID == operationID )\ - .one() - return self.getRequest( requestID[0] ) + requestID = session.query(Request.RequestID)\ + .join(Request.__operations__)\ + .filter(Operation.OperationID == operationID)\ + .one() + return self.getRequest(requestID[0]) except NoResultFound: return S_OK() finally: @@ -291,9 +282,7 @@ def getScheduledRequest( self, operationID ): # return S_ERROR( "getRequestName: no request found for RequestID=%s" % requestID ) # finally: # session.close() - - - def getRequest( self, reqID = 0, assigned = True ): + def getRequest(self, reqID=0, assigned=True): """ read request for execution :param reqID: request's ID (default 0) If 0, take a pseudo random one @@ -301,8 +290,8 @@ def getRequest( self, reqID = 0, assigned = True ): """ # expire_on_commit is set to False so that we can still use the object after we close the session - session = self.DBSession( expire_on_commit = False ) - log = self.log.getSubLogger( 'getRequest' if assigned else 'peekRequest' ) + session = self.DBSession(expire_on_commit=False) + log = self.log.getSubLogger('getRequest' if assigned else 'peekRequest') requestID = None @@ -311,83 +300,84 @@ def getRequest( self, reqID = 0, assigned = True ): if reqID: requestID = reqID - log.verbose( "selecting request '%s'%s" % ( reqID, ' (Assigned)' if assigned else '' ) ) + log.verbose("selecting request '%s'%s" % (reqID, ' (Assigned)' if assigned else '')) status = None try: - status = session.query( Request._Status )\ - .filter( Request.RequestID == reqID )\ + status = session.query(Request._Status)\ + .filter(Request.RequestID == reqID)\ .one() - except NoResultFound, e: - return S_ERROR( "getRequest: request '%s' not exists" % reqID ) + except NoResultFound as e: + return S_ERROR("getRequest: request '%s' not exists" % reqID) if status and status == "Assigned" and assigned: - return S_ERROR( "getRequest: status of request '%s' is 'Assigned', request cannot be selected" % reqID ) + return S_ERROR("getRequest: status of request '%s' is 'Assigned', request cannot be selected" % reqID) else: - now = datetime.datetime.utcnow().replace( microsecond = 0 ) + now = datetime.datetime.utcnow().replace(microsecond=0) reqIDs = set() try: - reqAscIDs = session.query( Request.RequestID )\ - .filter( Request._Status == 'Waiting' )\ - .filter( Request._NotBefore < now )\ - .order_by( Request._LastUpdate )\ - .limit( 100 )\ + reqAscIDs = session.query(Request.RequestID)\ + .filter(Request._Status == 'Waiting')\ + .filter(Request._NotBefore < now)\ + .order_by(Request._LastUpdate)\ + .limit(100)\ .all() - reqIDs = set( [reqID[0] for reqID in reqAscIDs] ) + reqIDs = set([reqID[0] for reqID in reqAscIDs]) - reqDescIDs = session.query( Request.RequestID )\ - .filter( Request._Status == 'Waiting' )\ - .filter( Request._NotBefore < now )\ - .order_by( Request._LastUpdate.desc() )\ - .limit( 50 )\ + reqDescIDs = session.query(Request.RequestID)\ + .filter(Request._Status == 'Waiting')\ + .filter(Request._NotBefore < now)\ + .order_by(Request._LastUpdate.desc())\ + .limit(50)\ .all() - reqIDs |= set( [reqID[0] for reqID in reqDescIDs] ) + reqIDs |= set([reqID[0] for reqID in reqDescIDs]) # No Waiting requests - except NoResultFound, e: + except NoResultFound as e: return S_OK() if not reqIDs: return S_OK() - reqIDs = list( reqIDs ) - random.shuffle( reqIDs ) + reqIDs = list(reqIDs) + random.shuffle(reqIDs) requestID = reqIDs[0] - # If we are here, the request MUST exist, so no try catch # the joinedload_all is to force the non-lazy loading of all the attributes, especially _parent - request = session.query( Request )\ - .options( joinedload_all( '__operations__.__files__' ) )\ - .filter( Request.RequestID == requestID )\ + request = session.query(Request)\ + .options(joinedload_all('__operations__.__files__'))\ + .filter(Request.RequestID == requestID)\ .one() if not reqID: - log.verbose( "selected request %s('%s')%s" % ( request.RequestID, request.RequestName, ' (Assigned)' if assigned else '' ) ) - + log.verbose( + "selected request %s('%s')%s" % + (request.RequestID, + request.RequestName, + ' (Assigned)' if assigned else '')) if assigned: - session.execute( update( Request )\ - .where( Request.RequestID == requestID )\ - .values( {Request._Status : 'Assigned', - Request._LastUpdate : datetime.datetime.utcnow()\ - .strftime( Request._datetimeFormat )} ) - ) + session.execute(update(Request) + .where(Request.RequestID == requestID) + .values({Request._Status: 'Assigned', + Request._LastUpdate: datetime.datetime.utcnow() + .strftime(Request._datetimeFormat)}) + ) session.commit() session.expunge_all() - return S_OK( request ) + return S_OK(request) except Exception as e: session.rollback() - log.exception( "getRequest: unexpected exception", lException = e ) - return S_ERROR( "getRequest: unexpected exception : %s" % e ) + log.exception("getRequest: unexpected exception", lException=e) + return S_ERROR("getRequest: unexpected exception : %s" % e) finally: session.close() - - def getBulkRequests( self, numberOfRequest = 10, assigned = True ): + def getBulkRequests(self, numberOfRequest=10, assigned=True): """ read as many requests as requested for execution :param int numberOfRequest: Number of Request we want (default 10) @@ -398,8 +388,8 @@ def getBulkRequests( self, numberOfRequest = 10, assigned = True ): """ # expire_on_commit is set to False so that we can still use the object after we close the session - session = self.DBSession( expire_on_commit = False ) - log = self.log.getSubLogger( 'getBulkRequest' if assigned else 'peekBulkRequest' ) + session = self.DBSession(expire_on_commit=False) + log = self.log.getSubLogger('getBulkRequest' if assigned else 'peekBulkRequest') requestDict = {} @@ -407,93 +397,87 @@ def getBulkRequests( self, numberOfRequest = 10, assigned = True ): # If we are here, the request MUST exist, so no try catch # the joinedload_all is to force the non-lazy loading of all the attributes, especially _parent try: - now = datetime.datetime.utcnow().replace( microsecond = 0 ) - requestIDs = session.query( Request.RequestID )\ - .with_for_update()\ - .filter( Request._Status == 'Waiting' )\ - .filter( Request._NotBefore < now )\ - .order_by( Request._LastUpdate )\ - .limit( numberOfRequest )\ - .all() + now = datetime.datetime.utcnow().replace(microsecond=0) + requestIDs = session.query(Request.RequestID)\ + .with_for_update()\ + .filter(Request._Status == 'Waiting')\ + .filter(Request._NotBefore < now)\ + .order_by(Request._LastUpdate)\ + .limit(numberOfRequest)\ + .all() requestIDs = [ridTuple[0] for ridTuple in requestIDs] - log.debug( "Got request ids %s" % requestIDs ) + log.debug("Got request ids %s" % requestIDs) - requests = session.query( Request )\ - .options( joinedload_all( '__operations__.__files__' ) )\ - .filter( Request.RequestID.in_( requestIDs ) )\ + requests = session.query(Request)\ + .options(joinedload_all('__operations__.__files__'))\ + .filter(Request.RequestID.in_(requestIDs))\ .all() - log.debug( "Got %s Request objects " % len( requests ) ) - requestDict = dict( ( req.RequestID, req ) for req in requests ) + log.debug("Got %s Request objects " % len(requests)) + requestDict = dict((req.RequestID, req) for req in requests) # No Waiting requests - except NoResultFound, e: + except NoResultFound as e: pass if assigned and requestDict: - session.execute( update( Request )\ - .where( Request.RequestID.in_( requestDict.keys() ) )\ - .values( {Request._Status : 'Assigned', - Request._LastUpdate : datetime.datetime.utcnow()\ - .strftime( Request._datetimeFormat )} ) - ) + session.execute(update(Request) + .where(Request.RequestID.in_(requestDict.keys())) + .values({Request._Status: 'Assigned', + Request._LastUpdate: datetime.datetime.utcnow() + .strftime(Request._datetimeFormat)}) + ) session.commit() session.expunge_all() except Exception as e: session.rollback() - log.exception( "unexpected exception", lException = e ) - return S_ERROR( "getBulkRequest: unexpected exception : %s" % e ) + log.exception("unexpected exception", lException=e) + return S_ERROR("getBulkRequest: unexpected exception : %s" % e) finally: session.close() - return S_OK( requestDict ) + return S_OK(requestDict) - - - def peekRequest( self, requestID ): + def peekRequest(self, requestID): """ get request (ro), no update on states :param requestID: Request.RequestID """ - return self.getRequest( requestID, False ) - + return self.getRequest(requestID, False) - - def getRequestIDsList( self, statusList = None, limit = None, since = None, until = None, getJobID = False ): + def getRequestIDsList(self, statusList=None, limit=None, since=None, until=None, getJobID=False): """ select requests with status in :statusList: """ - statusList = statusList if statusList else list( Request.FINAL_STATES ) + statusList = statusList if statusList else list(Request.FINAL_STATES) limit = limit if limit else 100 session = self.DBSession() requestIDs = [] try: if getJobID: - reqQuery = session.query( Request.RequestID, Request._Status, Request._LastUpdate, Request.JobID )\ - .filter( Request._Status.in_( statusList ) ) + reqQuery = session.query(Request.RequestID, Request._Status, Request._LastUpdate, Request.JobID)\ + .filter(Request._Status.in_(statusList)) else: - reqQuery = session.query( Request.RequestID, Request._Status, Request._LastUpdate )\ - .filter( Request._Status.in_( statusList ) ) + reqQuery = session.query(Request.RequestID, Request._Status, Request._LastUpdate)\ + .filter(Request._Status.in_(statusList)) if since: - reqQuery = reqQuery.filter( Request._LastUpdate > since ) + reqQuery = reqQuery.filter(Request._LastUpdate > since) if until: - reqQuery = reqQuery.filter( Request._LastUpdate < until ) + reqQuery = reqQuery.filter(Request._LastUpdate < until) - reqQuery = reqQuery.order_by( Request._LastUpdate )\ - .limit( limit ) - requestIDs = [ tuple( reqIDTuple ) for reqIDTuple in reqQuery.all() ] + reqQuery = reqQuery.order_by(Request._LastUpdate)\ + .limit(limit) + requestIDs = [tuple(reqIDTuple) for reqIDTuple in reqQuery.all()] except Exception as e: session.rollback() - self.log.exception( "getRequestIDsList: unexpected exception", lException = e ) - return S_ERROR( "getRequestIDsList: unexpected exception : %s" % e ) + self.log.exception("getRequestIDsList: unexpected exception", lException=e) + return S_ERROR("getRequestIDsList: unexpected exception : %s" % e) finally: session.close() - return S_OK( requestIDs ) - + return S_OK(requestIDs) - - def deleteRequest( self, requestID ): + def deleteRequest(self, requestID): """ delete request given its ID :param str requestID: request.RequestID @@ -503,58 +487,55 @@ def deleteRequest( self, requestID ): session = self.DBSession() try: - session.query( Request ).filter( Request.RequestID == requestID ).delete() + session.query(Request).filter(Request.RequestID == requestID).delete() session.commit() except Exception as e: session.rollback() - self.log.exception( "deleteRequest: unexpected exception", lException = e ) - return S_ERROR( "deleteRequest: unexpected exception : %s" % e ) + self.log.exception("deleteRequest: unexpected exception", lException=e) + return S_ERROR("deleteRequest: unexpected exception : %s" % e) finally: session.close() return S_OK() - - def getDBSummary( self ): + def getDBSummary(self): """ get db summary """ # # this will be returned - retDict = { "Request" : {}, "Operation" : {}, "File" : {} } + retDict = {"Request": {}, "Operation": {}, "File": {}} session = self.DBSession() try: - requestQuery = session.query( Request._Status, func.count( Request.RequestID ) )\ - .group_by( Request._Status )\ + requestQuery = session.query(Request._Status, func.count(Request.RequestID))\ + .group_by(Request._Status)\ .all() for status, count in requestQuery: retDict["Request"][status] = count - operationQuery = session.query( Operation.Type, Operation._Status, func.count( Operation.OperationID ) )\ - .group_by( Operation.Type, Operation._Status )\ + operationQuery = session.query(Operation.Type, Operation._Status, func.count(Operation.OperationID))\ + .group_by(Operation.Type, Operation._Status)\ .all() for oType, status, count in operationQuery: - retDict['Operation'].setdefault( oType, {} )[status] = count - + retDict['Operation'].setdefault(oType, {})[status] = count - fileQuery = session.query( File._Status, func.count( File.FileID ) )\ - .group_by( File._Status )\ + fileQuery = session.query(File._Status, func.count(File.FileID))\ + .group_by(File._Status)\ .all() for status, count in fileQuery: retDict["File"][status] = count except Exception as e: - self.log.exception( "getDBSummary: unexpected exception", lException = e ) - return S_ERROR( "getDBSummary: unexpected exception : %s" % e ) + self.log.exception("getDBSummary: unexpected exception", lException=e) + return S_ERROR("getDBSummary: unexpected exception : %s" % e) finally: session.close() - return S_OK( retDict ) + return S_OK(retDict) - - def getRequestSummaryWeb( self, selectDict, sortList, startItem, maxItems ): + def getRequestSummaryWeb(self, selectDict, sortList, startItem, maxItems): """ Returns a list of Request for the web portal :param dict selectDict: parameter on which to restrain the query {key : Value} @@ -565,30 +546,25 @@ def getRequestSummaryWeb( self, selectDict, sortList, startItem, maxItems ): :param int startItem: start item (for pagination) :param int maxItems: max items (for pagination) """ - - - parameterList = [ 'RequestID', 'RequestName', 'JobID', 'OwnerDN', 'OwnerGroup', - 'Status', "Error", "CreationTime", "LastUpdate"] - + parameterList = ['RequestID', 'RequestName', 'JobID', 'OwnerDN', 'OwnerGroup', + 'Status', "Error", "CreationTime", "LastUpdate"] resultDict = {} session = self.DBSession() try: - summaryQuery = session.query( Request.RequestID, Request.RequestName, - Request.JobID, Request.OwnerDN, Request.OwnerGroup, - Request._Status, Request.Error, - Request._CreationTime, Request._LastUpdate ) - - + summaryQuery = session.query(Request.RequestID, Request.RequestName, + Request.JobID, Request.OwnerDN, Request.OwnerGroup, + Request._Status, Request.Error, + Request._CreationTime, Request._LastUpdate) for key, value in selectDict.items(): if key == 'ToDate': - summaryQuery = summaryQuery.filter( Request._LastUpdate < value ) + summaryQuery = summaryQuery.filter(Request._LastUpdate < value) elif key == 'FromDate': - summaryQuery = summaryQuery.filter( Request._LastUpdate > value ) + summaryQuery = summaryQuery.filter(Request._LastUpdate > value) else: tableName = 'Request' @@ -600,56 +576,55 @@ def getRequestSummaryWeb( self, selectDict, sortList, startItem, maxItems ): elif key == 'Status': key = '_Status' - if isinstance( value, list ): - summaryQuery = summaryQuery.filter( eval( '%s.%s.in_(%s)' % ( tableName, key, value ) ) ) + if isinstance(value, list): + summaryQuery = summaryQuery.filter(eval('%s.%s.in_(%s)' % (tableName, key, value))) else: - summaryQuery = summaryQuery.filter( eval( '%s.%s' % ( tableName, key ) ) == value ) + summaryQuery = summaryQuery.filter(eval('%s.%s' % (tableName, key)) == value) if sortList: - summaryQuery = summaryQuery.order_by( eval( 'Request.%s.%s()' % ( sortList[0][0], sortList[0][1].lower() ) ) ) + summaryQuery = summaryQuery.order_by(eval('Request.%s.%s()' % (sortList[0][0], sortList[0][1].lower()))) try: requestLists = summaryQuery.all() - except NoResultFound, e: + except NoResultFound as e: resultDict['ParameterNames'] = parameterList resultDict['Records'] = [] - return S_OK( resultDict ) + return S_OK(resultDict) except Exception as e: - return S_ERROR( 'Error getting the webSummary %s' % e ) + return S_ERROR('Error getting the webSummary %s' % e) - nRequests = len( requestLists ) + nRequests = len(requestLists) - if startItem <= len( requestLists ): + if startItem <= len(requestLists): firstIndex = startItem else: - return S_ERROR( 'getRequestSummaryWeb: Requested index out of range' ) + return S_ERROR('getRequestSummaryWeb: Requested index out of range') - if ( startItem + maxItems ) <= len( requestLists ): + if (startItem + maxItems) <= len(requestLists): secondIndex = startItem + maxItems else: - secondIndex = len( requestLists ) + secondIndex = len(requestLists) records = [] - for i in range( firstIndex, secondIndex ): + for i in range(firstIndex, secondIndex): row = requestLists[i] - records.append( [ str( x ) for x in row] ) + records.append([str(x) for x in row]) resultDict['ParameterNames'] = parameterList resultDict['Records'] = records resultDict['TotalRecords'] = nRequests - return S_OK( resultDict ) + return S_OK(resultDict) # except Exception as e: - self.log.exception( "getRequestSummaryWeb: unexpected exception", lException = e ) - return S_ERROR( "getRequestSummaryWeb: unexpected exception : %s" % e ) + self.log.exception("getRequestSummaryWeb: unexpected exception", lException=e) + return S_ERROR("getRequestSummaryWeb: unexpected exception : %s" % e) finally: session.close() - - def getRequestCountersWeb( self, groupingAttribute, selectDict ): + def getRequestCountersWeb(self, groupingAttribute, selectDict): """ For the web portal. Returns a dictionary {value : counts} for a given key. The key can be any field from the RequestTable. or "Type", @@ -668,50 +643,47 @@ def getRequestCountersWeb( self, groupingAttribute, selectDict ): groupingAttribute = 'Request.%s' % groupingAttribute try: - summaryQuery = session.query( eval( groupingAttribute ), func.count( Request.RequestID ) ) + summaryQuery = session.query(eval(groupingAttribute), func.count(Request.RequestID)) for key, value in selectDict.items(): if key == 'ToDate': - summaryQuery = summaryQuery.filter( Request._LastUpdate < value ) + summaryQuery = summaryQuery.filter(Request._LastUpdate < value) elif key == 'FromDate': - summaryQuery = summaryQuery.filter( Request._LastUpdate > value ) + summaryQuery = summaryQuery.filter(Request._LastUpdate > value) else: objectType = 'Request' if key == 'Type': - summaryQuery = summaryQuery.join( Request.__operations__ ) + summaryQuery = summaryQuery.join(Request.__operations__) objectType = 'Operation' elif key == 'Status': key = '_Status' - if isinstance( value, list ): - summaryQuery = summaryQuery.filter( eval( '%s.%s.in_(%s)' % ( objectType, key, value ) ) ) + if isinstance(value, list): + summaryQuery = summaryQuery.filter(eval('%s.%s.in_(%s)' % (objectType, key, value))) else: - summaryQuery = summaryQuery.filter( eval( '%s.%s' % ( objectType, key ) ) == value ) + summaryQuery = summaryQuery.filter(eval('%s.%s' % (objectType, key)) == value) - summaryQuery = summaryQuery.group_by( groupingAttribute ) + summaryQuery = summaryQuery.group_by(groupingAttribute) try: requestLists = summaryQuery.all() - resultDict = dict( requestLists ) - except NoResultFound, e: + resultDict = dict(requestLists) + except NoResultFound as e: pass except Exception as e: - return S_ERROR( 'Error getting the webCounters %s' % e ) - + return S_ERROR('Error getting the webCounters %s' % e) - - return S_OK( resultDict ) + return S_OK(resultDict) except Exception as e: - self.log.exception( "getRequestSummaryWeb: unexpected exception", lException = e ) - return S_ERROR( "getRequestSummaryWeb: unexpected exception : %s" % e ) + self.log.exception("getRequestSummaryWeb: unexpected exception", lException=e) + return S_ERROR("getRequestSummaryWeb: unexpected exception : %s" % e) finally: session.close() - - def getDistinctValues( self, tableName, columnName ): + def getDistinctValues(self, tableName, columnName): """ For a given table and a given field, return the list of of distinct values in the DB""" session = self.DBSession() @@ -719,54 +691,52 @@ def getDistinctValues( self, tableName, columnName ): if columnName == 'Status': columnName = '_Status' try: - result = session.query( distinct( eval ( "%s.%s" % ( tableName, columnName ) ) ) ).all() + result = session.query(distinct(eval("%s.%s" % (tableName, columnName)))).all() distinctValues = [dist[0] for dist in result] - except NoResultFound, e: + except NoResultFound as e: pass except Exception as e: - self.log.exception( "getDistinctValues: unexpected exception", lException = e ) - return S_ERROR( "getDistinctValues: unexpected exception : %s" % e ) + self.log.exception("getDistinctValues: unexpected exception", lException=e) + return S_ERROR("getDistinctValues: unexpected exception : %s" % e) finally: session.close() - return S_OK( distinctValues ) - + return S_OK(distinctValues) - def getRequestIDsForJobs( self, jobIDs ): + def getRequestIDsForJobs(self, jobIDs): """ read request ids for jobs given jobIDs :param jobIDs: list of jobIDs :type jobIDs: python:list """ - self.log.debug( "getRequestIDsForJobs: got %s jobIDs to check" % str( jobIDs ) ) + self.log.debug("getRequestIDsForJobs: got %s jobIDs to check" % str(jobIDs)) if not jobIDs: - return S_ERROR( "Must provide jobID list as argument." ) - if isinstance( jobIDs, ( long, int ) ): - jobIDs = [ jobIDs ] - jobIDs = set( jobIDs ) + return S_ERROR("Must provide jobID list as argument.") + if isinstance(jobIDs, (long, int)): + jobIDs = [jobIDs] + jobIDs = set(jobIDs) - reqDict = { "Successful": {}, "Failed": {} } + reqDict = {"Successful": {}, "Failed": {}} session = self.DBSession() try: - ret = session.query( Request.JobID, Request.RequestID )\ - .filter( Request.JobID.in_( jobIDs ) )\ - .all() + ret = session.query(Request.JobID, Request.RequestID)\ + .filter(Request.JobID.in_(jobIDs))\ + .all() - reqDict['Successful'] = dict( ( jobId, reqID ) for jobId, reqID in ret ) - reqDict['Failed'] = dict( ( jobid, 'Request not found' ) for jobid in jobIDs - set( reqDict['Successful'] ) ) + reqDict['Successful'] = dict((jobId, reqID) for jobId, reqID in ret) + reqDict['Failed'] = dict((jobid, 'Request not found') for jobid in jobIDs - set(reqDict['Successful'])) except Exception as e: - self.log.exception( "getRequestIDsForJobs: unexpected exception", lException = e ) - return S_ERROR( "getRequestIDsForJobs: unexpected exception : %s" % e ) + self.log.exception("getRequestIDsForJobs: unexpected exception", lException=e) + return S_ERROR("getRequestIDsForJobs: unexpected exception : %s" % e) finally: session.close() - return S_OK( reqDict ) + return S_OK(reqDict) - - def readRequestsForJobs( self, jobIDs = None ): + def readRequestsForJobs(self, jobIDs=None): """ read request for jobs :param jobIDs: list of JobIDs @@ -774,50 +744,48 @@ def readRequestsForJobs( self, jobIDs = None ): :return: S_OK( "Successful" : { jobID1 : Request, jobID2: Request, ... } "Failed" : { jobID3: "error message", ... } ) """ - self.log.debug( "readRequestForJobs: got %s jobIDs to check" % str( jobIDs ) ) + self.log.debug("readRequestForJobs: got %s jobIDs to check" % str(jobIDs)) if not jobIDs: - return S_ERROR( "Must provide jobID list as argument." ) - if isinstance( jobIDs, ( long, int ) ): - jobIDs = [ jobIDs ] - jobIDs = set( jobIDs ) + return S_ERROR("Must provide jobID list as argument.") + if isinstance(jobIDs, (long, int)): + jobIDs = [jobIDs] + jobIDs = set(jobIDs) - reqDict = { "Successful": {}, "Failed": {} } + reqDict = {"Successful": {}, "Failed": {}} # expire_on_commit is set to False so that we can still use the object after we close the session - session = self.DBSession( expire_on_commit = False ) + session = self.DBSession(expire_on_commit=False) try: - ret = session.query( Request.JobID, Request )\ - .options( joinedload_all( '__operations__.__files__' ) )\ - .filter( Request.JobID.in_( jobIDs ) ).all() + ret = session.query(Request.JobID, Request)\ + .options(joinedload_all('__operations__.__files__'))\ + .filter(Request.JobID.in_(jobIDs)).all() - reqDict['Successful'] = dict( ( jobId, reqObj ) for jobId, reqObj in ret ) + reqDict['Successful'] = dict((jobId, reqObj) for jobId, reqObj in ret) - reqDict['Failed'] = dict( ( jobid, 'Request not found' ) for jobid in jobIDs - set( reqDict['Successful'] ) ) + reqDict['Failed'] = dict((jobid, 'Request not found') for jobid in jobIDs - set(reqDict['Successful'])) session.expunge_all() except Exception as e: - self.log.exception( "readRequestsForJobs: unexpected exception", lException = e ) - return S_ERROR( "readRequestsForJobs: unexpected exception : %s" % e ) + self.log.exception("readRequestsForJobs: unexpected exception", lException=e) + return S_ERROR("readRequestsForJobs: unexpected exception : %s" % e) finally: session.close() - return S_OK( reqDict ) - + return S_OK(reqDict) - def getRequestStatus( self, requestID ): + def getRequestStatus(self, requestID): """ get request status for a given request ID """ - self.log.debug( "getRequestStatus: checking status for '%s' request" % requestID ) + self.log.debug("getRequestStatus: checking status for '%s' request" % requestID) session = self.DBSession() try: - status = session.query( Request._Status ).filter( Request.RequestID == requestID ).one() - except NoResultFound: - return S_ERROR( "Request %s does not exist" % requestID ) + status = session.query(Request._Status).filter(Request.RequestID == requestID).one() + except NoResultFound: + return S_ERROR(errno.ENOENT, "Request %s does not exist" % requestID) finally: session.close() - return S_OK( status[0] ) - + return S_OK(status[0]) - def getRequestFileStatus( self, requestID, lfnList ): + def getRequestFileStatus(self, requestID, lfnList): """ get status for files in request given its id :param str requestID: Request.RequestID @@ -827,71 +795,68 @@ def getRequestFileStatus( self, requestID, lfnList ): session = self.DBSession() try: - res = dict.fromkeys( lfnList, "UNKNOWN" ) - requestRet = session.query( File._LFN, File._Status )\ - .join( Request.__operations__ )\ - .join( Operation.__files__ )\ - .filter( Request.RequestID == requestID )\ - .filter( File._LFN.in_( lfnList ) )\ - .all() + res = dict.fromkeys(lfnList, "UNKNOWN") + requestRet = session.query(File._LFN, File._Status)\ + .join(Request.__operations__)\ + .join(Operation.__files__)\ + .filter(Request.RequestID == requestID)\ + .filter(File._LFN.in_(lfnList))\ + .all() for lfn, status in requestRet: res[lfn] = status - return S_OK( res ) + return S_OK(res) except Exception as e: - self.log.exception( "getRequestFileStatus: unexpected exception", lException = e ) - return S_ERROR( "getRequestFileStatus: unexpected exception : %s" % e ) + self.log.exception("getRequestFileStatus: unexpected exception", lException=e) + return S_ERROR("getRequestFileStatus: unexpected exception : %s" % e) finally: session.close() - - def getRequestInfo( self, requestID ): + def getRequestInfo(self, requestID): """ get request info given Request.RequestID """ session = self.DBSession() try: - requestInfoQuery = session.query( Request.RequestID, Request._Status, Request.RequestName, - Request.JobID, Request.OwnerDN, Request.OwnerGroup, - Request.DIRACSetup, Request.SourceComponent, Request._CreationTime, - Request._SubmitTime, Request._LastUpdate )\ - .filter( Request.RequestID == requestID ) - + requestInfoQuery = session.query(Request.RequestID, Request._Status, Request.RequestName, + Request.JobID, Request.OwnerDN, Request.OwnerGroup, + Request.DIRACSetup, Request.SourceComponent, Request._CreationTime, + Request._SubmitTime, Request._LastUpdate)\ + .filter(Request.RequestID == requestID) try: requestInfo = requestInfoQuery.one() except NoResultFound: - return S_ERROR( 'No such request' ) + return S_ERROR('No such request') - return S_OK( tuple( requestInfo ) ) + return S_OK(tuple(requestInfo)) except Exception as e: - self.log.exception( "getRequestInfo: unexpected exception", lException = e ) - return S_ERROR( "getRequestInfo: unexpected exception : %s" % e ) + self.log.exception("getRequestInfo: unexpected exception", lException=e) + return S_ERROR("getRequestInfo: unexpected exception : %s" % e) finally: session.close() - def getDigest( self, requestID ): + def getDigest(self, requestID): """ get digest for request given its id :param str requestName: request id """ - self.log.debug( "getDigest: will create digest for request '%s'" % requestID ) - request = self.getRequest( requestID, False ) + self.log.debug("getDigest: will create digest for request '%s'" % requestID) + request = self.getRequest(requestID, False) if not request["OK"]: - self.log.error( "getDigest: %s" % request["Message"] ) + self.log.error("getDigest: %s" % request["Message"]) return request request = request["Value"] - if not isinstance( request, Request ): - self.log.info( "getDigest: request '%s' not found" ) + if not isinstance(request, Request): + self.log.info("getDigest: request '%s' not found") return S_OK() return request.getDigest() - - def getRequestIDForName( self, requestName ): + def getRequestIDForName(self, requestName): """ read request id for given name if the name is not unique, an error is returned @@ -901,22 +866,22 @@ def getRequestIDForName( self, requestName ): reqID = 0 try: - ret = session.query( Request.RequestID )\ - .filter( Request.RequestName == requestName )\ + ret = session.query(Request.RequestID)\ + .filter(Request.RequestName == requestName)\ .all() if not ret: - return S_ERROR( 'No such request %s' % requestName ) - elif len( ret ) > 1: - return S_ERROR( 'RequestName %s not unique (%s matches)' % ( requestName, len( ret ) ) ) + return S_ERROR('No such request %s' % requestName) + elif len(ret) > 1: + return S_ERROR('RequestName %s not unique (%s matches)' % (requestName, len(ret))) reqID = ret[0][0] - except NoResultFound, e: - return S_ERROR( 'No such request' ) + except NoResultFound as e: + return S_ERROR('No such request') except Exception as e: - self.log.exception( "getRequestIDsForName: unexpected exception", lException = e ) - return S_ERROR( "getRequestIDsForName: unexpected exception : %s" % e ) + self.log.exception("getRequestIDsForName: unexpected exception", lException=e) + return S_ERROR("getRequestIDsForName: unexpected exception : %s" % e) finally: session.close() - return S_OK( reqID ) + return S_OK(reqID) From 62d719595e3158eea8e22c2f1bae4dd1d3ea33fc Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 15 Jun 2020 10:32:54 +0200 Subject: [PATCH 04/44] gitignore: ignore env file for vscode --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0e1c97cd729..7bea1aaba40 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,7 @@ tests/CI/SERVERCONFIG # VSCode .vscode +.env # docs # this is auto generated From 21ee3995aa32168dc59e78ec946c9f71b28b1cb1 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 16 Jun 2020 14:33:29 +0200 Subject: [PATCH 05/44] REMOVE READLONLY VARIABLES --- tests/Jenkins/dirac_ci.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Jenkins/dirac_ci.sh b/tests/Jenkins/dirac_ci.sh index 387848c9ce8..9e3472802fc 100644 --- a/tests/Jenkins/dirac_ci.sh +++ b/tests/Jenkins/dirac_ci.sh @@ -66,11 +66,11 @@ fi # Creating default structure mkdir -p "$WORKSPACE/TestCode" # Where the test code resides -readonly TESTCODE=${_} +TESTCODE=${_} mkdir -p "$WORKSPACE/ServerInstallDIR" # Where servers are installed -readonly SERVERINSTALLDIR=${_} +SERVERINSTALLDIR=${_} mkdir -p "$WORKSPACE/ClientInstallDIR" # Where clients are installed -readonly CLIENTINSTALLDIR=${_} +CLIENTINSTALLDIR=${_} # Location of the CFG file to be used (this can be replaced by the extensions) INSTALL_CFG_FILE="${TESTCODE}/DIRAC/tests/Jenkins/install.cfg" From 1900ac8e248296ac7a180137e9456a13c74aefba Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 16 Jun 2020 15:47:49 +0200 Subject: [PATCH 06/44] SMS: drop tables in the correct order --- StorageManagementSystem/DB/StorageManagementDB.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/StorageManagementSystem/DB/StorageManagementDB.sql b/StorageManagementSystem/DB/StorageManagementDB.sql index e1cb451b30e..a446b922fce 100644 --- a/StorageManagementSystem/DB/StorageManagementDB.sql +++ b/StorageManagementSystem/DB/StorageManagementDB.sql @@ -9,7 +9,11 @@ use StorageManagementDB; +DROP TABLE IF EXISTS TaskReplicas; +DROP TABLE IF EXISTS StageRequests; +DROP TABLE IF EXISTS CacheReplicas; DROP TABLE IF EXISTS Tasks; + CREATE TABLE Tasks( TaskID INTEGER AUTO_INCREMENT, Status VARCHAR(32) DEFAULT 'New', @@ -23,7 +27,6 @@ CREATE TABLE Tasks( INDEX(TaskID,Status) )ENGINE=INNODB; -DROP TABLE IF EXISTS TaskReplicas; CREATE TABLE TaskReplicas( TaskID INTEGER(8) NOT NULL REFERENCES Tasks(TaskID), ReplicaID INTEGER(8) NOT NULL REFERENCES CacheReplicas(ReplicaID), @@ -33,7 +36,7 @@ CREATE TABLE TaskReplicas( CREATE TRIGGER taskreplicasAfterInsert AFTER INSERT ON TaskReplicas FOR EACH ROW UPDATE CacheReplicas SET CacheReplicas.Links=CacheReplicas.Links+1 WHERE CacheReplicas.ReplicaID=NEW.ReplicaID; CREATE TRIGGER taskreplicasAfterDelete AFTER DELETE ON TaskReplicas FOR EACH ROW UPDATE CacheReplicas SET CacheReplicas.Links=CacheReplicas.Links-1 WHERE CacheReplicas.ReplicaID=OLD.ReplicaID; -DROP TABLE IF EXISTS CacheReplicas; + CREATE TABLE CacheReplicas( ReplicaID INTEGER AUTO_INCREMENT, Type VARCHAR(32) NOT NULL, @@ -52,7 +55,7 @@ CREATE TABLE CacheReplicas( INDEX(ReplicaID,Status,SE) )ENGINE=INNODB; -DROP TABLE IF EXISTS StageRequests; + CREATE TABLE StageRequests( ReplicaID INTEGER(8) NOT NULL REFERENCES CacheReplicas(ReplicaID), StageStatus VARCHAR(32) DEFAULT 'StageSubmitted', From 8c094852784af843c7aee5f57b1562c8b8582db4 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 16 Jun 2020 17:46:19 +0200 Subject: [PATCH 07/44] tests: fix dropDBs bad escape --- tests/Jenkins/utilities.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index 5268ad15e72..ee1a1b9292d 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -892,8 +892,9 @@ diracMVDFCDB(){ dropDBs(){ echo '==> [dropDBs]' - dbs=$(cut -d ' ' -f 2 < databases | cut -d '.' -f 1 | grep -v ^RequestDB | grep -v ^FileCatalogDB) - python "${TESTCODE}/DIRAC/tests/Jenkins/dirac-drop-db.py" "$dbs" "${DEBUG}" + # make dbs a real array to avoid future mistake with escaping + mapfile -t dbs < <(cut -d ' ' -f 2 < /tmp/databases.txt | cut -d '.' -f 1 | grep -v ^RequestDB | grep -v ^FileCatalogDB) + python "${TESTCODE}/DIRAC/tests/Jenkins/dirac-drop-db.py" "${dbs[@]}" "${DEBUG}" } #------------------------------------------------------------------------------- From 48144008702d1f0659123bab432f39fa10b8bf2f Mon Sep 17 00:00:00 2001 From: Simon Fayer Date: Tue, 16 Jun 2020 20:04:24 +0100 Subject: [PATCH 08/44] Print error if LFN list is missing --- DataManagementSystem/scripts/dirac-dms-add-file.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/DataManagementSystem/scripts/dirac-dms-add-file.py b/DataManagementSystem/scripts/dirac-dms-add-file.py index a2bcb95804b..2ecc74bcd6b 100755 --- a/DataManagementSystem/scripts/dirac-dms-add-file.py +++ b/DataManagementSystem/scripts/dirac-dms-add-file.py @@ -56,6 +56,11 @@ def getDict( item_list ): lfn_dict['guid'] = guid return lfn_dict +from DIRAC.DataManagementSystem.Client.DataManager import DataManager +from DIRAC import gLogger +import DIRAC +exitCode = 0 + lfns = [] if len( args ) == 1: inputFileName = args[0] @@ -67,14 +72,12 @@ def getDict( item_list ): items[0] = items[0].replace( 'LFN:', '' ).replace( 'lfn:', '' ) lfns.append( getDict( items ) ) inputFile.close() + else: + gLogger.error("Error: LFN list '%s' missing." % inputFileName) + exitCode = 4 else: lfns.append( getDict( args ) ) -from DIRAC.DataManagementSystem.Client.DataManager import DataManager -from DIRAC import gLogger -import DIRAC -exitCode = 0 - dm = DataManager() for lfn in lfns: if not os.path.exists( lfn['localfile'] ): From f0ab1d0eef4bcbc4a33883ef3f2b3aec68ac1ecb Mon Sep 17 00:00:00 2001 From: Philippe Charpentier Date: Wed, 17 Jun 2020 09:53:19 +0200 Subject: [PATCH 09/44] fix hospital handling --- TransformationSystem/Client/TaskManager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TransformationSystem/Client/TaskManager.py b/TransformationSystem/Client/TaskManager.py index a711c0b9e63..1f6afddb677 100644 --- a/TransformationSystem/Client/TaskManager.py +++ b/TransformationSystem/Client/TaskManager.py @@ -827,8 +827,8 @@ def _checkSickTransformations(self, transID): """ Check if the transformation is in the transformations to be processed at Hospital or Clinic """ transID = int(transID) - clinicPath = "Hospital/Transformations" - if transID in set(int(x) for x in self.opsH.getValue(clinicPath, [])): + clinicPath = "Hospital" + if transID in set(int(x) for x in self.opsH.getValue(os.path.join(clinicPath, "Transformations"), [])): return clinicPath if "Clinics" in self.opsH.getSections("Hospital").get('Value', []): basePath = os.path.join("Hospital", "Clinics") From a7c1fdacd8d9556fd7d18109e9b6e124f689765f Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Mon, 8 Jun 2020 12:47:33 +0200 Subject: [PATCH 10/44] dirac-admin-add-host: adding host to ComponentMonitoring --- Interfaces/scripts/dirac-admin-add-host.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Interfaces/scripts/dirac-admin-add-host.py b/Interfaces/scripts/dirac-admin-add-host.py index 7d7ca02c25c..2ed1376f542 100755 --- a/Interfaces/scripts/dirac-admin-add-host.py +++ b/Interfaces/scripts/dirac-admin-add-host.py @@ -76,6 +76,19 @@ def addProperty( arg ): errorList.append( ( "commit", result[ 'Message' ] ) ) exitCode = 255 +if exitCode == 0: + from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient + cmc = ComponentMonitoringClient() + ret = cmc.hostExists(dict(HostName=hostName)) + if not ret['OK']: + Script.gLogger.error('Cannot check if host is registered in ComponentMoniroting', ret['Message']) + elif ret['Value']: + Script.gLogger.info('Host already registered in ComponentMonitoring') + else: + ret = cmc.addHost(dict(HostName=hostName, CPU='TO_COME')) + if not ret['OK']: + Script.gLogger.error('Failed to add Host to ComponentMonitoring', ret['Message']) + for error in errorList: Script.gLogger.error( "%s: %s" % error ) From 52f597dd930cda913f1bd6b72beba6276d720562 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Thu, 11 Jun 2020 09:53:10 +0200 Subject: [PATCH 11/44] CSAPI: fix and expand parameter docstrings indentation was off for some, causing failure to properly markup --- ConfigurationSystem/Client/CSAPI.py | 49 ++++++++++++++++------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/ConfigurationSystem/Client/CSAPI.py b/ConfigurationSystem/Client/CSAPI.py index dd052219262..f547b2b36f3 100644 --- a/ConfigurationSystem/Client/CSAPI.py +++ b/ConfigurationSystem/Client/CSAPI.py @@ -121,7 +121,7 @@ def listHosts(self): def describeUsers(self, users=None): """ describe users by nickname - :param list users: list of users' nickanames + :param list users: list of users' nicknames :return: a S_OK(description) of the users in input """ if users is None: @@ -240,14 +240,14 @@ def addUser(self, username, properties): """ Add a user to the cs - :param str username: group name - :param dict properties: dictionary describing user properties: + :param str username: username + :param dict properties: dictionary describing user properties: - - DN - - groups - - + - DN + - groups + - - :return: True/False + :return: True/False """ if not self.__initialized['OK']: return self.__initialized @@ -279,14 +279,15 @@ def modifyUser(self, username, properties, createIfNonExistant=False): """ Modify a user - :param str username: group name - :param dict properties: dictionary describing user properties: + :param str username: group name + :param dict properties: dictionary describing user properties: - DN - Groups - - :return: S_OK, Value = True/False + :param bool createIfNonExistant: if true, registers the users if it did not exist + :return: S_OK, Value = True/False """ if not self.__initialized['OK']: return self.__initialized @@ -341,14 +342,14 @@ def addGroup(self, groupname, properties): """ Add a group to the cs - :param str groupname: group name - :param dict properties: dictionary describing group properties: + :param str groupname: group name + :param dict properties: dictionary describing group properties: - Users - Properties - - :return: True/False + :return: S_OK, Value = True/False """ if not self.__initialized['OK']: return self.__initialized @@ -366,14 +367,15 @@ def modifyGroup(self, groupname, properties, createIfNonExistant=False): """ Modify a group - :param str groupname: group name - :param dict properties: dictionary describing group properties: + :param str groupname: group name + :param dict properties: dictionary describing group properties: - Users - Properties - - :return: True/False + :param bool createIfNonExistant: if true, creates the group if it did not exist + :return: S_OK, Value = True/False """ if not self.__initialized['OK']: return self.__initialized @@ -401,14 +403,15 @@ def modifyGroup(self, groupname, properties, createIfNonExistant=False): def addHost(self, hostname, properties): """ Add a host to the cs - :param str hostname: hostname name - :param dict properties: dictionary describing host properties: + + :param str hostname: host name + :param dict properties: dictionary describing host properties: - DN - Properties - - :return: True/False + :return: S_OK, Value = True/False """ if not self.__initialized['OK']: return self.__initialized @@ -528,14 +531,16 @@ def getOpsSection(): def modifyHost(self, hostname, properties, createIfNonExistant=False): """ Modify a host - :param str hostname: hostname name - :param dict properties: dictionary describing host properties: + + :param str hostname: hostname name + :param dict properties: dictionary describing host properties: - DN - Properties - - :return: True/False + :param bool createIfNonExistant: if true, creates the host if it did not exist + :return: S_OK, Value = True/False """ if not self.__initialized['OK']: return self.__initialized From 6c669c1e47b267329e78a33cee6a4261ac5eb546 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Thu, 11 Jun 2020 10:32:02 +0200 Subject: [PATCH 12/44] Config: add documentation for type of gConfig --- ConfigurationSystem/Client/Config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ConfigurationSystem/Client/Config.py b/ConfigurationSystem/Client/Config.py index 11314f2a160..6363f77d4c5 100755 --- a/ConfigurationSystem/Client/Config.py +++ b/ConfigurationSystem/Client/Config.py @@ -6,8 +6,13 @@ from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient +#: Global gConfig object of type :class:`~DIRAC.ConfigurationSystem.private.ConfigurationClient.ConfigurationClient` gConfig = ConfigurationClient() def getConfig(): + """ + :returns: gConfig + :rtype: ~DIRAC.ConfigurationSystem.private.ConfigurationClient.ConfigurationClient + """ return gConfig From 97dd214d0df23342f514035b1aeb248e6da88d43 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Mon, 22 Jun 2020 11:38:19 +0200 Subject: [PATCH 13/44] dirac-admin-add-host: Fix typo in log message Co-authored-by: fstagni --- Interfaces/scripts/dirac-admin-add-host.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Interfaces/scripts/dirac-admin-add-host.py b/Interfaces/scripts/dirac-admin-add-host.py index 2ed1376f542..04dc291493d 100755 --- a/Interfaces/scripts/dirac-admin-add-host.py +++ b/Interfaces/scripts/dirac-admin-add-host.py @@ -81,7 +81,7 @@ def addProperty( arg ): cmc = ComponentMonitoringClient() ret = cmc.hostExists(dict(HostName=hostName)) if not ret['OK']: - Script.gLogger.error('Cannot check if host is registered in ComponentMoniroting', ret['Message']) + Script.gLogger.error('Cannot check if host is registered in ComponentMonitoring', ret['Message']) elif ret['Value']: Script.gLogger.info('Host already registered in ComponentMonitoring') else: From e4207d11a1f491082e7bdd18b100a63db74dd4a4 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Wed, 24 Jun 2020 16:39:16 +0200 Subject: [PATCH 14/44] pytest without coverage --- .github/workflows/basic.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 21febce9efd..348b18141f8 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -12,10 +12,10 @@ jobs: fail-fast: False matrix: command: - - pytest - - DIRAC_USE_M2CRYPTO=Yes DIRAC_M2CRYPTO_SPLIT_HANDSHAKE=Yes pytest - - pytest Core/Security/test - - DIRAC_USE_M2CRYPTO=Yes DIRAC_M2CRYPTO_SPLIT_HANDSHAKE=Yes pytest Core/Security/test + - pytest --no-cov + - DIRAC_USE_M2CRYPTO=Yes DIRAC_M2CRYPTO_SPLIT_HANDSHAKE=Yes pytest --no-cov + - pytest --no-cov Core/Security/test + - DIRAC_USE_M2CRYPTO=Yes DIRAC_M2CRYPTO_SPLIT_HANDSHAKE=Yes pytest --no-cov Core/Security/test - tests/checkDocs.sh # TODO This should cover more than just tests/CI # Excluded codes related to sourcing files From 6b995e99dab365ac3edd1d847fd533626e8d2968 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Wed, 24 Jun 2020 21:00:37 +0200 Subject: [PATCH 15/44] no --no-cov fir general pytest --- .github/workflows/basic.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 348b18141f8..9bcc0f00cf9 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -12,8 +12,8 @@ jobs: fail-fast: False matrix: command: - - pytest --no-cov - - DIRAC_USE_M2CRYPTO=Yes DIRAC_M2CRYPTO_SPLIT_HANDSHAKE=Yes pytest --no-cov + - pytest + - DIRAC_USE_M2CRYPTO=Yes DIRAC_M2CRYPTO_SPLIT_HANDSHAKE=Yes pytest - pytest --no-cov Core/Security/test - DIRAC_USE_M2CRYPTO=Yes DIRAC_M2CRYPTO_SPLIT_HANDSHAKE=Yes pytest --no-cov Core/Security/test - tests/checkDocs.sh From c08871db85ecf7ece4069aad3100d76d5397c72f Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Thu, 25 Jun 2020 09:26:22 +0200 Subject: [PATCH 16/44] once again --- .github/workflows/basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 9bcc0f00cf9..7d70c7d623b 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -12,7 +12,7 @@ jobs: fail-fast: False matrix: command: - - pytest + - pytest --no-cov - DIRAC_USE_M2CRYPTO=Yes DIRAC_M2CRYPTO_SPLIT_HANDSHAKE=Yes pytest - pytest --no-cov Core/Security/test - DIRAC_USE_M2CRYPTO=Yes DIRAC_M2CRYPTO_SPLIT_HANDSHAKE=Yes pytest --no-cov Core/Security/test From ebc8639fa65732f1dfb355227181190ccfca9ea6 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Thu, 25 Jun 2020 10:38:50 +0200 Subject: [PATCH 17/44] test adapted for --no-cov pytest running --- .../PilotAgent/test/Test_Pilot.py | 67 ++++++------------- 1 file changed, 22 insertions(+), 45 deletions(-) diff --git a/WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py b/WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py index 8e986940958..dc8468c991c 100644 --- a/WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py +++ b/WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py @@ -2,54 +2,31 @@ """ # imports -import unittest import json import os +import sys + +if "--no-cov" in sys.argv: + del sys.argv[sys.argv.index( '--no-cov' )] from DIRAC.WorkloadManagementSystem.PilotAgent.pilotTools import PilotParams, CommandBase from DIRAC.WorkloadManagementSystem.PilotAgent.pilotCommands import GetPilotVersion -class PilotTestCase( unittest.TestCase ): - """ Base class for the Agents test cases - """ - def setUp( self ): - self.pp = PilotParams() - - def tearDown( self ): - try: - os.remove('pilot.out') - os.remove( 'pilot.json' ) - os.remove( 'pilot.json-local' ) - except OSError: - pass - - -class CommandsTestCase( PilotTestCase ): - - def test_commandBase(self): - cb = CommandBase(self.pp) - returnCode, _outputData = cb.executeAndGetOutput("ls") - self.assertEqual(returnCode, 0) - - def test_GetPilotVersion( self ): - - # Now defining a local file for test, and all the necessary parameters - fp = open( 'pilot.json', 'w' ) - json.dump( {'TestSetup':{'Version':['v1r1', 'v2r2']}}, fp ) - fp.close() - self.pp.setup = 'TestSetup' - self.pp.pilotCFGFileLocation = 'file://%s' % os.getcwd() - gpv = GetPilotVersion( self.pp ) - self.assertIsNone( gpv.execute() ) - self.assertEqual( gpv.pp.releaseVersion, 'v1r1' ) - -############################################################################# -# Test Suite run -############################################################################# - -if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase( PilotTestCase ) - suite.addTest( unittest.defaultTestLoader.loadTestsFromTestCase( CommandsTestCase ) ) - testResult = unittest.TextTestRunner( verbosity = 2 ).run( suite ) - -# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF# +def test_GetPilotVersion(): + pp = PilotParams() + # Now defining a local file for test, and all the necessary parameters + fp = open( 'pilot.json', 'w' ) + json.dump( { 'TestSetup': { 'Version': ['v1r1', 'v2r2'] } }, fp ) + fp.close() + pp.setup = 'TestSetup' + pp.pilotCFGFileLocation = 'file://%s' % os.getcwd() + gpv = GetPilotVersion( pp ) + result = gpv.execute() + assert result is None + assert gpv.pp.releaseVersion == 'v1r1' + +def test_commandBase(): + pp = PilotParams() + cb = CommandBase( pp ) + returnCode, _outputData = cb.executeAndGetOutput( "ls" ) + assert returnCode== 0 From b996650e9fd4967250ed1072010dc8ced78a8dc6 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Thu, 25 Jun 2020 10:44:22 +0200 Subject: [PATCH 18/44] hopefully the final --no-cov --- .github/workflows/basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 7d70c7d623b..348b18141f8 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -13,7 +13,7 @@ jobs: matrix: command: - pytest --no-cov - - DIRAC_USE_M2CRYPTO=Yes DIRAC_M2CRYPTO_SPLIT_HANDSHAKE=Yes pytest + - DIRAC_USE_M2CRYPTO=Yes DIRAC_M2CRYPTO_SPLIT_HANDSHAKE=Yes pytest --no-cov - pytest --no-cov Core/Security/test - DIRAC_USE_M2CRYPTO=Yes DIRAC_M2CRYPTO_SPLIT_HANDSHAKE=Yes pytest --no-cov Core/Security/test - tests/checkDocs.sh From e8d903fe192130b808944c762f8c85f490b67529 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Thu, 25 Jun 2020 10:51:04 +0200 Subject: [PATCH 19/44] typo fixed --- WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py b/WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py index dc8468c991c..aafbc2c273a 100644 --- a/WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py +++ b/WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py @@ -29,4 +29,4 @@ def test_commandBase(): pp = PilotParams() cb = CommandBase( pp ) returnCode, _outputData = cb.executeAndGetOutput( "ls" ) - assert returnCode== 0 + assert returnCode == 0 From dc3f58165e79bb844c7c83fe04f0d4d68a1fecfb Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Thu, 25 Jun 2020 11:14:17 +0200 Subject: [PATCH 20/44] autopep8 --- .../PilotAgent/test/Test_Pilot.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py b/WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py index aafbc2c273a..a183bf9f959 100644 --- a/WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py +++ b/WorkloadManagementSystem/PilotAgent/test/Test_Pilot.py @@ -7,26 +7,28 @@ import sys if "--no-cov" in sys.argv: - del sys.argv[sys.argv.index( '--no-cov' )] + del sys.argv[sys.argv.index('--no-cov')] from DIRAC.WorkloadManagementSystem.PilotAgent.pilotTools import PilotParams, CommandBase from DIRAC.WorkloadManagementSystem.PilotAgent.pilotCommands import GetPilotVersion + def test_GetPilotVersion(): pp = PilotParams() # Now defining a local file for test, and all the necessary parameters - fp = open( 'pilot.json', 'w' ) - json.dump( { 'TestSetup': { 'Version': ['v1r1', 'v2r2'] } }, fp ) + fp = open('pilot.json', 'w') + json.dump({'TestSetup': {'Version': ['v1r1', 'v2r2']}}, fp) fp.close() pp.setup = 'TestSetup' pp.pilotCFGFileLocation = 'file://%s' % os.getcwd() - gpv = GetPilotVersion( pp ) + gpv = GetPilotVersion(pp) result = gpv.execute() assert result is None assert gpv.pp.releaseVersion == 'v1r1' + def test_commandBase(): pp = PilotParams() - cb = CommandBase( pp ) - returnCode, _outputData = cb.executeAndGetOutput( "ls" ) + cb = CommandBase(pp) + returnCode, _outputData = cb.executeAndGetOutput("ls") assert returnCode == 0 From 105bcfbdb9edfd762887aafa0f9540c6e84d0231 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Thu, 25 Jun 2020 10:41:27 +0200 Subject: [PATCH 21/44] ComponentInstaller: allow underscores in installed components The previous split would drop things after an underscore in the installed components name (e.g., Bdii2CSAgent_GLUE2) --- FrameworkSystem/Client/ComponentInstaller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FrameworkSystem/Client/ComponentInstaller.py b/FrameworkSystem/Client/ComponentInstaller.py index d71901639d4..63baf94e43c 100644 --- a/FrameworkSystem/Client/ComponentInstaller.py +++ b/FrameworkSystem/Client/ComponentInstaller.py @@ -1130,7 +1130,7 @@ def getSetupComponents(self): for cType in self.componentTypes: if body.find('dirac-%s' % (cType)) != -1: - system, compT = component.split('_')[0:2] + system, compT = component.split('_', 1) if system not in resultDict[resultIndexes[cType]]: resultDict[resultIndexes[cType]][system] = [] resultDict[resultIndexes[cType]][system].append(compT) From 2661d6b625ec0a8b0c41251f53c7d9014c2654ec Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Thu, 25 Jun 2020 12:54:59 +0200 Subject: [PATCH 22/44] dirac-transformation-clean: fix use of position arguments allows calling the script with e.g., -ddd --- TransformationSystem/scripts/dirac-transformation-clean.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/TransformationSystem/scripts/dirac-transformation-clean.py b/TransformationSystem/scripts/dirac-transformation-clean.py index ddc63836010..452afc62078 100755 --- a/TransformationSystem/scripts/dirac-transformation-clean.py +++ b/TransformationSystem/scripts/dirac-transformation-clean.py @@ -5,7 +5,7 @@ from __future__ import print_function import sys -from DIRAC.Core.Base.Script import parseCommandLine +from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs parseCommandLine() from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent @@ -15,8 +15,7 @@ print('Usage: dirac-transformation-clean transID [transID] [transID]') sys.exit() else: - transIDs = [int( arg ) for arg in sys.argv[1:]] - + transIDs = [int(arg) for arg in getPositionalArgs()] agent = TransformationCleaningAgent( 'Transformation/TransformationCleaningAgent', 'Transformation/TransformationCleaningAgent', From b3bd16cfaa9dda967378e75518fa75e2bf843f98 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Thu, 25 Jun 2020 13:13:37 +0200 Subject: [PATCH 23/44] Transformation scripts: allow command line flags (e.g., -ddd) --- .../scripts/dirac-transformation-archive.py | 6 +++--- TransformationSystem/scripts/dirac-transformation-clean.py | 2 +- .../scripts/dirac-transformation-remove-output.py | 6 +++--- .../scripts/dirac-transformation-verify-outputdata.py | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/TransformationSystem/scripts/dirac-transformation-archive.py b/TransformationSystem/scripts/dirac-transformation-archive.py index 08afe90b0f3..9cd3018bff7 100755 --- a/TransformationSystem/scripts/dirac-transformation-archive.py +++ b/TransformationSystem/scripts/dirac-transformation-archive.py @@ -5,14 +5,14 @@ from __future__ import print_function import sys -from DIRAC.Core.Base.Script import parseCommandLine +from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs parseCommandLine() -if len( sys.argv ) < 2: +if not getPositionalArgs(): print('Usage: dirac-transformation-archive transID [transID] [transID]') sys.exit() else: - transIDs = [int( arg ) for arg in sys.argv[1:]] + transIDs = [int(arg) for arg in getPositionalArgs()] from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient diff --git a/TransformationSystem/scripts/dirac-transformation-clean.py b/TransformationSystem/scripts/dirac-transformation-clean.py index 452afc62078..d9c5f5a48aa 100755 --- a/TransformationSystem/scripts/dirac-transformation-clean.py +++ b/TransformationSystem/scripts/dirac-transformation-clean.py @@ -11,7 +11,7 @@ from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient -if len( sys.argv ) < 2: +if not getPositionalArgs(): print('Usage: dirac-transformation-clean transID [transID] [transID]') sys.exit() else: diff --git a/TransformationSystem/scripts/dirac-transformation-remove-output.py b/TransformationSystem/scripts/dirac-transformation-remove-output.py index 332e91f7509..4f3d6353680 100755 --- a/TransformationSystem/scripts/dirac-transformation-remove-output.py +++ b/TransformationSystem/scripts/dirac-transformation-remove-output.py @@ -5,14 +5,14 @@ from __future__ import print_function import sys -from DIRAC.Core.Base.Script import parseCommandLine +from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs parseCommandLine() -if len( sys.argv ) < 2: +if not getPositionalArgs(): print('Usage: dirac-transformation-remove-output transID [transID] [transID]') sys.exit() else: - transIDs = [int( arg ) for arg in sys.argv[1:]] + transIDs = [int(arg) for arg in getPositionalArgs()] from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient diff --git a/TransformationSystem/scripts/dirac-transformation-verify-outputdata.py b/TransformationSystem/scripts/dirac-transformation-verify-outputdata.py index c4a59a114c7..eaf6a20682e 100755 --- a/TransformationSystem/scripts/dirac-transformation-verify-outputdata.py +++ b/TransformationSystem/scripts/dirac-transformation-verify-outputdata.py @@ -5,14 +5,14 @@ from __future__ import print_function import sys -from DIRAC.Core.Base.Script import parseCommandLine +from DIRAC.Core.Base.Script import parseCommandLine, getPositionalArgs parseCommandLine() -if len( sys.argv ) < 2: +if not getPositionalArgs(): print('Usage: dirac-transformation-verify-outputdata transID [transID] [transID]') sys.exit() else: - transIDs = [int( arg ) for arg in sys.argv[1:]] + transIDs = [int(arg) for arg in getPositionalArgs()] from DIRAC.TransformationSystem.Agent.ValidateOutputDataAgent import ValidateOutputDataAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient From 6385e74de8c450f0c3fa46c88ffd2f78db7960b6 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Sun, 28 Jun 2020 22:08:55 +0200 Subject: [PATCH 24/44] Resolve error under DIRACOS environment --- RequestManagementSystem/DB/RequestDB.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RequestManagementSystem/DB/RequestDB.py b/RequestManagementSystem/DB/RequestDB.py index affd832a27a..d6641fb03e7 100644 --- a/RequestManagementSystem/DB/RequestDB.py +++ b/RequestManagementSystem/DB/RequestDB.py @@ -688,7 +688,7 @@ def getRequestCountersWeb( self, groupingAttribute, selectDict ): else: summaryQuery = summaryQuery.filter( eval( '%s.%s' % ( objectType, key ) ) == value ) - summaryQuery = summaryQuery.group_by( groupingAttribute ) + summaryQuery = summaryQuery.group_by(eval(groupingAttribute)) try: requestLists = summaryQuery.all() From 4a28881d914944903e1ec67221b759c85946fac7 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Sun, 28 Jun 2020 22:11:44 +0200 Subject: [PATCH 25/44] v6r22p31 notes and tags --- __init__.py | 2 +- release.notes | 8 ++++++++ setup.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/__init__.py b/__init__.py index ee2397a8a4c..7a165262c9f 100755 --- a/__init__.py +++ b/__init__.py @@ -78,7 +78,7 @@ majorVersion = 6 minorVersion = 22 -patchLevel = 30 +patchLevel = 31 preVersion = 0 version = "v%sr%s" % (majorVersion, minorVersion) diff --git a/release.notes b/release.notes index 55a61a1e5af..309d6c35c47 100644 --- a/release.notes +++ b/release.notes @@ -1,3 +1,11 @@ +[v6r22p31] + +*DMS +FIX: (#4646) Print error from dirac-dms-add-file if input LFN list file is missing. + +*RMS +FIX: (#4657) RequestDB - fix getRequestCountersWeb error in DIRACOS + [v6r22p30] *DataManagementSystem diff --git a/setup.py b/setup.py index 7fd948bdbe4..b0ba3ed35a3 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name="DIRAC", - version="6.22.30", + version="6.22.31", url="https://github.com/DIRACGRID/DIRAC", license="GPLv3", package_dir=package_dir, From 412f0e96f4c0e84edb56be18b11675d25ac12ca5 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Sun, 28 Jun 2020 22:23:26 +0200 Subject: [PATCH 26/44] v7r0p28 notes and tags --- __init__.py | 2 +- release.notes | 20 ++++++++++++++++++++ setup.py | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/__init__.py b/__init__.py index bb9f1e11b1d..a8688e90e6c 100755 --- a/__init__.py +++ b/__init__.py @@ -92,7 +92,7 @@ majorVersion = 7 minorVersion = 0 -patchLevel = 27 +patchLevel = 28 preVersion = 0 version = "v%sr%s" % (majorVersion, minorVersion) diff --git a/release.notes b/release.notes index f562299c1da..22e98094a25 100644 --- a/release.notes +++ b/release.notes @@ -1,3 +1,23 @@ +[v7r0p28] + +*Core +FIX: (#4642) M2Crypto closes the socket after dereferencing the Connection instance + +*DMS +FIX: (#4644) Cancel FTS3 Operation if the RMS request does not exist + +*RMS +NEW: (#4644) getRequestStatus returns ENOENT if the request does not exist + +*TS +FIX: (#4647) TaskManager - hospital sites were not looked for correctly, which + generated an exception +FIX: (#4656) fix parsing of command line flags (e.g., -ddd) for dirac-transformation-archive/clean/remove-output/verify-outputdata + +*Interfaces +CHANGE: (#4652) dirac-admin-add-host now inserts hosts into the ComponentMonitoring if the host + is not yet known + [v7r0p27] *Framework diff --git a/setup.py b/setup.py index 01520d128fa..34d98e9b67f 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name="DIRAC", - version="7.0.27", + version="7.0.28", url="https://github.com/DIRACGRID/DIRAC", license="GPLv3", package_dir=package_dir, From 1e5b3eff7867b7b4e25871de3a954e006952eba9 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Sun, 28 Jun 2020 23:01:35 +0200 Subject: [PATCH 27/44] pep8 --- DataManagementSystem/scripts/dirac-dms-add-file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/DataManagementSystem/scripts/dirac-dms-add-file.py b/DataManagementSystem/scripts/dirac-dms-add-file.py index 4c371901f49..d3ffc0eea9d 100755 --- a/DataManagementSystem/scripts/dirac-dms-add-file.py +++ b/DataManagementSystem/scripts/dirac-dms-add-file.py @@ -60,6 +60,7 @@ def getDict(item_list): lfn_dict['guid'] = guid return lfn_dict + from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC import gLogger import DIRAC From ff771eac7915231ffc9ab8186b9e99b61564565d Mon Sep 17 00:00:00 2001 From: fstagni Date: Wed, 10 Jun 2020 13:07:56 +0200 Subject: [PATCH 28/44] py2 to 3 --- .../Agent/MCExtensionAgent.py | 2 +- .../Agent/TaskManagerAgentBase.py | 10 +++---- .../Agent/TransformationCleaningAgent.py | 2 +- .../Agent/TransformationPlugin.py | 6 ++--- .../Agent/ValidateOutputDataAgent.py | 2 +- TransformationSystem/Client/TaskManager.py | 5 +++- TransformationSystem/Client/Transformation.py | 2 +- .../Client/TransformationCLI.py | 26 +++++++++---------- TransformationSystem/DB/TransformationDB.py | 8 +++--- .../Service/TransformationManagerHandler.py | 2 +- .../Utilities/TransformationInfo.py | 2 +- .../test/Test_TransformationInfo.py | 2 +- 12 files changed, 36 insertions(+), 33 deletions(-) diff --git a/TransformationSystem/Agent/MCExtensionAgent.py b/TransformationSystem/Agent/MCExtensionAgent.py index fb11d4d0ded..791e0b4d574 100755 --- a/TransformationSystem/Agent/MCExtensionAgent.py +++ b/TransformationSystem/Agent/MCExtensionAgent.py @@ -80,7 +80,7 @@ def extendTransformation(self, transID, maxTasks): else: statusDict = res['Value'] gLogger.verbose("Current task count for transformation %d" % transID) - for status in sorted(statusDict.keys()): + for status in sorted(list(statusDict)): statusCount = statusDict[status] gLogger.verbose("%s : %s" % (status.ljust(20), str(statusCount).rjust(8))) # Determine the number of tasks to be created diff --git a/TransformationSystem/Agent/TaskManagerAgentBase.py b/TransformationSystem/Agent/TaskManagerAgentBase.py index 241803885a9..036fc0d4488 100644 --- a/TransformationSystem/Agent/TaskManagerAgentBase.py +++ b/TransformationSystem/Agent/TaskManagerAgentBase.py @@ -285,7 +285,7 @@ def _execute(self, threadID): # Queue was cleared, nothing to do continue try: - transID = transIDOPBody.keys()[0] + transID = list(transIDOPBody)[0] operations = transIDOPBody[transID]['Operations'] if transID not in self.transInQueue: self._logWarn("Got a transf not in transInQueue...?", @@ -327,7 +327,7 @@ def _execute(self, threadID): def updateTaskStatus(self, transIDOPBody, clients): """ Updates the task status """ - transID = transIDOPBody.keys()[0] + transID = list(transIDOPBody)[0] method = 'updateTaskStatus' # Get the tasks which are in an UPDATE state, i.e. job statuses + request-specific statuses @@ -410,7 +410,7 @@ def updateTaskStatus(self, transIDOPBody, clients): def updateFileStatus(self, transIDOPBody, clients): """ Update the files status """ - transID = transIDOPBody.keys()[0] + transID = list(transIDOPBody)[0] method = 'updateFileStatus' timeStamp = str(datetime.datetime.utcnow() - datetime.timedelta(minutes=10)) @@ -480,7 +480,7 @@ def updateFileStatus(self, transIDOPBody, clients): def checkReservedTasks(self, transIDOPBody, clients): """ Checking Reserved tasks """ - transID = transIDOPBody.keys()[0] + transID = list(transIDOPBody)[0] method = 'checkReservedTasks' # Select the tasks which have been in Reserved status for more than 1 hour for selected transformations @@ -546,7 +546,7 @@ def submitTasks(self, transIDOPBody, clients): :return: S_OK/S_ERROR """ - transID = transIDOPBody.keys()[0] + transID = list(transIDOPBody)[0] transBody = transIDOPBody[transID]['Body'] owner = transIDOPBody[transID]['Owner'] ownerGroup = transIDOPBody[transID]['OwnerGroup'] diff --git a/TransformationSystem/Agent/TransformationCleaningAgent.py b/TransformationSystem/Agent/TransformationCleaningAgent.py index a5f126965e5..2a346191166 100644 --- a/TransformationSystem/Agent/TransformationCleaningAgent.py +++ b/TransformationSystem/Agent/TransformationCleaningAgent.py @@ -348,7 +348,7 @@ def __getCatalogDirectoryContents(self, directories): activeDirs.extend(dirContents['SubDirs']) allFiles.update(dirContents['Files']) self.log.info("Found %d files" % len(allFiles)) - return S_OK(allFiles.keys()) + return S_OK(list(allFiles)) def cleanTransformationLogFiles(self, directory): """ clean up transformation logs from directory :directory: diff --git a/TransformationSystem/Agent/TransformationPlugin.py b/TransformationSystem/Agent/TransformationPlugin.py index f92d4d429cc..ead4c3fb49f 100644 --- a/TransformationSystem/Agent/TransformationPlugin.py +++ b/TransformationSystem/Agent/TransformationPlugin.py @@ -177,11 +177,11 @@ def _ByShare(self, shareType='CPU'): return res cpuShares = res['Value'] self.util.logInfo("Obtained the following target shares (%):") - for site in sorted(cpuShares.keys()): + for site in sorted(list(cpuShares)): self.util.logInfo("%s: %.1f" % (site.ljust(15), cpuShares[site])) # Get the existing destinations from the transformationDB - res = self.util.getExistingCounters(requestedSites=cpuShares.keys()) + res = self.util.getExistingCounters(requestedSites=list(cpuShares)) if not res['OK']: self.util.logError("Failed to get existing file share", res['Message']) return res @@ -189,7 +189,7 @@ def _ByShare(self, shareType='CPU'): if existingCount: self.util.logInfo("Existing site utilization (%):") normalisedExistingCount = self.util._normaliseShares(existingCount.copy()) # pylint: disable=protected-access - for se in sorted(normalisedExistingCount.keys()): + for se in sorted(list(normalisedExistingCount)): self.util.logInfo("%s: %.1f" % (se.ljust(15), normalisedExistingCount[se])) # Group the input files by their existing replicas diff --git a/TransformationSystem/Agent/ValidateOutputDataAgent.py b/TransformationSystem/Agent/ValidateOutputDataAgent.py index 32d2ee1aadb..049321bf51e 100755 --- a/TransformationSystem/Agent/ValidateOutputDataAgent.py +++ b/TransformationSystem/Agent/ValidateOutputDataAgent.py @@ -197,7 +197,7 @@ def checkTransformationIntegrity(self, transID): if res['Value']['Failed']: return S_ERROR("Failed to determine the existance of directories") directoryExists = res['Value']['Successful'] - for directory in sorted(directoryExists.keys()): + for directory in sorted(list(directoryExists)): if not directoryExists[directory]: continue iRes = self.consistencyInspector.catalogDirectoryToSE(directory) diff --git a/TransformationSystem/Client/TaskManager.py b/TransformationSystem/Client/TaskManager.py index 89ea9ed31f0..eee94d21323 100644 --- a/TransformationSystem/Client/TaskManager.py +++ b/TransformationSystem/Client/TaskManager.py @@ -391,7 +391,7 @@ def getSubmittedFileStatus(self, fileDicts): if transID is None: return S_OK({}) - res = self.transClient.getTransformationTasks({'TransformationID': transID, 'TaskID': taskFiles.keys()}) + res = self.transClient.getTransformationTasks({'TransformationID': transID, 'TaskID': list(taskFiles)}) if not res['OK']: return res requestFiles = {} @@ -668,10 +668,12 @@ def __prepareTasks(self, transBody, taskDict, owner, ownerGroup, ownerDN): method = '__prepareTasks' startTime = time.time() + oJobTemplate = self.jobClass(transBody) oJobTemplate.setOwner(owner) oJobTemplate.setOwnerGroup(ownerGroup) oJobTemplate.setOwnerDN(ownerDN) + try: site = oJobTemplate.workflow.findParameter('Site').getValue() except AttributeError: @@ -679,6 +681,7 @@ def __prepareTasks(self, transBody, taskDict, owner, ownerGroup, ownerDN): jobType = oJobTemplate.workflow.findParameter('JobType').getValue() templateOK = False getOutputDataTiming = 0. + for taskID, paramsDict in taskDict.iteritems(): # Create a job for each task and add it to the taskDict if not templateOK: diff --git a/TransformationSystem/Client/Transformation.py b/TransformationSystem/Client/Transformation.py index a29eb2943b5..0cff5c91eaa 100644 --- a/TransformationSystem/Client/Transformation.py +++ b/TransformationSystem/Client/Transformation.py @@ -187,7 +187,7 @@ def __getattr__(self, name): def __getParam(self): if self.item_called == 'Available': - return S_OK(self.paramTypes.keys()) + return S_OK(list(self.paramTypes)) if self.item_called == 'Parameters': return S_OK(self.paramValues) if self.item_called in self.paramValues: diff --git a/TransformationSystem/Client/TransformationCLI.py b/TransformationSystem/Client/TransformationCLI.py index cdf02a85909..37f96a1f385 100644 --- a/TransformationSystem/Client/TransformationCLI.py +++ b/TransformationSystem/Client/TransformationCLI.py @@ -541,11 +541,11 @@ def do_replicas(self, args): if not res['OK']: print("failed to get any replica information: %s" % res['Message']) return - for lfn in sorted(res['Value']['Failed'].keys()): + for lfn in sorted(list(res['Value']['Failed'])): error = res['Value']['Failed'][lfn] print("failed to get replica information for %s: %s" % (lfn, error)) - for lfn in sorted(res['Value']['Successful'].keys()): - ses = sorted(res['Value']['Successful'][lfn].keys()) + for lfn in sorted(list(res['Value']['Successful'])): + ses = sorted(list(res['Value']['Successful'][lfn])) outStr = "%s :" % lfn.ljust(100) for se in ses: outStr = "%s %s" % (outStr, se.ljust(15)) @@ -568,10 +568,10 @@ def do_addFile(self, args): if not res['OK']: print("failed to add any files: %s" % res['Message']) return - for lfn in sorted(res['Value']['Failed'].keys()): + for lfn in sorted(list(res['Value']['Failed'])): error = res['Value']['Failed'][lfn] print("failed to add %s: %s" % (lfn, error)) - for lfn in sorted(res['Value']['Successful'].keys()): + for lfn in sorted(list(res['Value']['Successful'])): print("added %s" % lfn) def do_removeFile(self, args): @@ -587,10 +587,10 @@ def do_removeFile(self, args): if not res['OK']: print("failed to remove any files: %s" % res['Message']) return - for lfn in sorted(res['Value']['Failed'].keys()): + for lfn in sorted(list(res['Value']['Failed'])): error = res['Value']['Failed'][lfn] print("failed to remove %s: %s" % (lfn, error)) - for lfn in sorted(res['Value']['Successful'].keys()): + for lfn in sorted(list(res['Value']['Successful'])): print("removed %s" % lfn) def do_addReplica(self, args): @@ -610,10 +610,10 @@ def do_addReplica(self, args): if not res['OK']: print("failed to add replica: %s" % res['Message']) return - for lfn in sorted(res['Value']['Failed'].keys()): + for lfn in sorted(list(res['Value']['Failed'])): error = res['Value']['Failed'][lfn] print("failed to add replica: %s" % (error)) - for lfn in sorted(res['Value']['Successful'].keys()): + for lfn in sorted(list(res['Value']['Successful'])): print("added %s" % lfn) def do_removeReplica(self, args): @@ -633,10 +633,10 @@ def do_removeReplica(self, args): if not res['OK']: print("failed to remove replica: %s" % res['Message']) return - for lfn in sorted(res['Value']['Failed'].keys()): + for lfn in sorted(list(res['Value']['Failed'])): error = res['Value']['Failed'][lfn] print("failed to remove replica: %s" % (error)) - for lfn in sorted(res['Value']['Successful'].keys()): + for lfn in sorted(list(res['Value']['Successful'])): print("removed %s" % lfn) def do_setReplicaStatus(self, args): @@ -663,10 +663,10 @@ def do_setReplicaStatus(self, args): if not res['OK']: print("failed to set replica status: %s" % res['Message']) return - for lfn in sorted(res['Value']['Failed'].keys()): + for lfn in sorted(list(res['Value']['Failed'])): error = res['Value']['Failed'][lfn] print("failed to set replica status: %s" % (error)) - for lfn in sorted(res['Value']['Successful'].keys()): + for lfn in sorted(list(res['Value']['Successful'])): print("updated replica status %s" % lfn) diff --git a/TransformationSystem/DB/TransformationDB.py b/TransformationSystem/DB/TransformationDB.py index e75ba36d298..3d72e58f0b6 100755 --- a/TransformationSystem/DB/TransformationDB.py +++ b/TransformationSystem/DB/TransformationDB.py @@ -524,7 +524,7 @@ def addFilesToTransformation(self, transName, lfns, connection=False): # Attach files to transformation successful = {} if fileIDs: - res = self.__addFilesToTransformation(transID, fileIDs.keys(), connection=connection) + res = self.__addFilesToTransformation(transID, list(fileIDs), connection=connection) if not res['OK']: return res for fileID in fileIDs: @@ -550,7 +550,7 @@ def getTransformationFiles(self, condDict=None, older=None, newer=None, timeStam if not res['OK']: return res originalFileIDs = res['Value'][0] - condDict['FileID'] = originalFileIDs.keys() + condDict['FileID'] = list(originalFileIDs) for val in condDict.itervalues(): if not val: @@ -1536,10 +1536,10 @@ def removeFile(self, lfns, connection=False): if lfn not in lfnFilesIDs: successful[lfn] = 'File does not exist' if fileIDs: - res = self.__setTransformationFileStatus(fileIDs.keys(), 'Deleted', connection=connection) + res = self.__setTransformationFileStatus(list(fileIDs), 'Deleted', connection=connection) if not res['OK']: return res - res = self.__setDataFileStatus(fileIDs.keys(), 'Deleted', connection=connection) + res = self.__setDataFileStatus(list(fileIDs), 'Deleted', connection=connection) if not res['OK']: return S_ERROR("TransformationDB.removeFile: Failed to remove files.") for lfn in lfnFilesIDs: diff --git a/TransformationSystem/Service/TransformationManagerHandler.py b/TransformationSystem/Service/TransformationManagerHandler.py index c31e8b24815..6cd8a1b6fcb 100644 --- a/TransformationSystem/Service/TransformationManagerHandler.py +++ b/TransformationSystem/Service/TransformationManagerHandler.py @@ -392,7 +392,7 @@ def export_removeFile(self, lfns): """ Interface provides [ LFN1, LFN2, ... ] """ if isinstance(lfns, dict): - lfns = lfns.keys() + lfns = list(lfns) res = database.removeFile(lfns) return self._parseRes(res) diff --git a/TransformationSystem/Utilities/TransformationInfo.py b/TransformationSystem/Utilities/TransformationInfo.py index ff51d8e4a7e..509151e28eb 100644 --- a/TransformationSystem/Utilities/TransformationInfo.py +++ b/TransformationSystem/Utilities/TransformationInfo.py @@ -155,7 +155,7 @@ def cleanOutputs(self, jobInfo): for lfn, err in result['Value']['Failed'].items(): reason = str(err) errorReasons[reason].append(lfn) - successfullyRemoved += len(result['Value']['Successful'].keys()) + successfullyRemoved += len(list(result['Value']['Successful'])) for reason, lfns in errorReasons.items(): self.log.error("Failed to remove %d files with error: %s" % (len(lfns), reason)) self.log.notice("Successfully removed %d files" % successfullyRemoved) diff --git a/TransformationSystem/test/Test_TransformationInfo.py b/TransformationSystem/test/Test_TransformationInfo.py index beb2a7141e1..30ab7a5dba8 100644 --- a/TransformationSystem/test/Test_TransformationInfo.py +++ b/TransformationSystem/test/Test_TransformationInfo.py @@ -352,7 +352,7 @@ def test_getJobs(tiFixture): assert ndone == 3 assert nfailed == 3 assert isinstance(jobs, OrderedDict) - assert [56, 89, 123, 456, 789, 1123] == jobs.keys() + assert [56, 89, 123, 456, 789, 1123] == list(jobs) # All ERROR tiFixture.jobMon.getJobs = Mock() From 596f1e43c2b9251a930f74c9f51e15bc8d2f725c Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 11 Jun 2020 16:57:48 +0200 Subject: [PATCH 29/44] changed dir --- virtualmachine/Vagrantfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/virtualmachine/Vagrantfile b/virtualmachine/Vagrantfile index 9337da73b4c..2a5516ae3cf 100644 --- a/virtualmachine/Vagrantfile +++ b/virtualmachine/Vagrantfile @@ -50,7 +50,8 @@ Vagrant.configure("2") do |config| # the path on the host to the actual folder. The second argument is # the path on the guest to mount the folder. And the optional third # argument is a set of non-required options. - config.vm.synced_folder "..", "/opt/dirac/versions/hostcode" + config.vm.synced_folder "..", "/home/vagrant/hostcode/DIRAC" + config.vm.synced_folder "../../WebAppDIRAC", "/home/vagrant/hostcode/WebAppDIRAC" config.vm.synced_folder "../../certs", "/home/vagrant/.globus" # Provider-specific configuration so you can fine-tune various @@ -77,16 +78,16 @@ Vagrant.configure("2") do |config| # config.vm.provision "shell", inline: <<-SHELL # #!/bin/bash - # # Create DIRAC dirs - # mkdir -p /opt/dirac/DIRAC && \ - # mkdir -p /opt/dirac/etc/grid-security/certificates && \ - # cd /opt/dirac + # sudo chown vagrant:vagrant hostcode/ + # ln -s hostcode/DIRAC/ DIRAC + # # Create etc dir and link it + # mkdir -p /home/vagrant/etc/grid-security/certificates + # ln -s /home/vagrant/etc /home/vagrant/hostcode/etc # # Installing DIRAC in /opt/dirac # curl -L -o dirac-install https://raw.githubusercontent.com/DIRACGrid/DIRAC/integration/Core/scripts/dirac-install.py && \ # chmod +x dirac-install && \ # ./dirac-install -r $DIRAC_VERSION -t client && \ - # rm -rf /opt/dirac/.installCache && \ # rm dirac-install && \ # ln -s /etc/grid-security/certificates/ /opt/dirac/etc/grid-security/certificates From 79100ebd2c6bde64a216168ffa7f85cc33dd451f Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 11 Jun 2020 17:29:58 +0200 Subject: [PATCH 30/44] added more selectable fields for methods using gJobDB.getDistinctJobAttributes --- .../Service/JobMonitoringHandler.py | 33 +++++++++---------- .../Test_Client_WMS.py | 15 ++++++--- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/WorkloadManagementSystem/Service/JobMonitoringHandler.py b/WorkloadManagementSystem/Service/JobMonitoringHandler.py index ccba404d2ca..3167a375e88 100755 --- a/WorkloadManagementSystem/Service/JobMonitoringHandler.py +++ b/WorkloadManagementSystem/Service/JobMonitoringHandler.py @@ -74,80 +74,79 @@ def initialize(self): types_getApplicationStates = [] @staticmethod - def export_getApplicationStates(): + def export_getApplicationStates(condDict=None, older=None, newer=None): """ Return Distinct Values of ApplicationStatus job Attribute in WMS """ - return gJobDB.getDistinctJobAttributes('ApplicationStatus') + return gJobDB.getDistinctJobAttributes('ApplicationStatus', condDict, older, newer) ############################################################################## types_getJobTypes = [] @staticmethod - def export_getJobTypes(): + def export_getJobTypes(condDict=None, older=None, newer=None): """ Return Distinct Values of JobType job Attribute in WMS """ - return gJobDB.getDistinctJobAttributes('JobType') + return gJobDB.getDistinctJobAttributes('JobType', condDict, older, newer) ############################################################################## types_getOwners = [] @staticmethod - def export_getOwners(): + def export_getOwners(condDict=None, older=None, newer=None): """ Return Distinct Values of Owner job Attribute in WMS """ - return gJobDB.getDistinctJobAttributes('Owner') + return gJobDB.getDistinctJobAttributes('Owner', condDict, older, newer) ############################################################################## types_getProductionIds = [] @staticmethod - def export_getProductionIds(): + def export_getProductionIds(condDict=None, older=None, newer=None): """ Return Distinct Values of ProductionId job Attribute in WMS """ - return gJobDB.getDistinctJobAttributes('JobGroup') + return gJobDB.getDistinctJobAttributes('JobGroup', condDict, older, newer) ############################################################################## types_getJobGroups = [] @staticmethod - def export_getJobGroups(condDict=None, cutDate=None): + def export_getJobGroups(condDict=None, older=None, cutDate=None): """ Return Distinct Values of ProductionId job Attribute in WMS """ - return gJobDB.getDistinctJobAttributes('JobGroup', condDict, - newer=cutDate) + return gJobDB.getDistinctJobAttributes('JobGroup', condDict, older, newer=cutDate) ############################################################################## types_getSites = [] @staticmethod - def export_getSites(): + def export_getSites(condDict=None, older=None, newer=None): """ Return Distinct Values of Site job Attribute in WMS """ - return gJobDB.getDistinctJobAttributes('Site') + return gJobDB.getDistinctJobAttributes('Site', condDict, older, newer) ############################################################################## types_getStates = [] @staticmethod - def export_getStates(): + def export_getStates(condDict=None, older=None, newer=None): """ Return Distinct Values of Status job Attribute in WMS """ - return gJobDB.getDistinctJobAttributes('Status') + return gJobDB.getDistinctJobAttributes('Status', condDict, older, newer) ############################################################################## types_getMinorStates = [] @staticmethod - def export_getMinorStates(): + def export_getMinorStates(condDict=None, older=None, newer=None): """ Return Distinct Values of Minor Status job Attribute in WMS """ - return gJobDB.getDistinctJobAttributes('MinorStatus') + return gJobDB.getDistinctJobAttributes('MinorStatus', condDict, older, newer) ############################################################################## types_getJobs = [] diff --git a/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py b/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py index 87cf125b429..30043850e8e 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py +++ b/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py @@ -26,7 +26,7 @@ # pylint: disable=protected-access,wrong-import-position,invalid-name from __future__ import print_function -from past.builtins import long + import unittest import sys import datetime @@ -365,6 +365,15 @@ def test_JobStateUpdateAndJobMonitoringMultuple(self): self.assertTrue(res['OK'], res.get('Message')) res = jobMonitor.getJobGroups() self.assertTrue(res['OK'], res.get('Message')) + resJG_empty = res['Value'] + res = jobMonitor.getJobGroups(None, datetime.datetime.utcnow()) + self.assertTrue(res['OK'], res.get('Message')) + resJG_olderThanNow = res['Value'] + self.assertEqual(resJG_empty, resJG_olderThanNow) + res = jobMonitor.getJobGroups(None, datetime.datetime.utcnow() - datetime.timedelta(days=365)) + self.assertTrue(res['OK'], res.get('Message')) + resJG_olderThanOneYear = res['Value'] + self.assertTrue(set(resJG_olderThanOneYear).issubset(set(resJG_olderThanNow))) res = jobMonitor.getStates() self.assertTrue(res['OK'], res.get('Message')) self.assertTrue(sorted(res['Value']) in [['Received'], sorted(['Received', 'Waiting'])]) @@ -382,9 +391,7 @@ def test_JobStateUpdateAndJobMonitoringMultuple(self): try: self.assertTrue( res['Value'].get('Received') + - res['Value'].get('Waiting') >= long( - len(lfnss) * - len(types))) + res['Value'].get('Waiting') >= int(len(lfnss) * len(types))) except TypeError: pass res = jobMonitor.getJobsSummary(jobIDs) From 938fa731c6c95595f9cfd843c865a23cc9a4f763 Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 11 Jun 2020 18:07:00 +0200 Subject: [PATCH 31/44] TransformationCleaningAgent will (re)clean very old transformations that are still in the system --- .../Agent/TransformationCleaningAgent.py | 58 ++++++++++++++++--- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/TransformationSystem/Agent/TransformationCleaningAgent.py b/TransformationSystem/Agent/TransformationCleaningAgent.py index 2a346191166..f8fb6ae2931 100644 --- a/TransformationSystem/Agent/TransformationCleaningAgent.py +++ b/TransformationSystem/Agent/TransformationCleaningAgent.py @@ -23,16 +23,17 @@ from DIRAC.Core.Utilities.List import breakListIntoChunks from DIRAC.Core.Utilities.Proxy import executeWithUserProxy from DIRAC.Core.Utilities.DErrno import cmpError +from DIRAC.Core.Utilities.ReturnValues import returnSingleResult +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient -from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient -from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient from DIRAC.DataManagementSystem.Client.DataManager import DataManager -from DIRAC.Resources.Storage.StorageElement import StorageElement -from DIRAC.Core.Utilities.ReturnValues import returnSingleResult +from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient +from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient +from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient +from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient # # agent's name AGENT_NAME = 'Transformation/TransformationCleaningAgent' @@ -83,7 +84,7 @@ def __init__(self, *args, **kwargs): def initialize(self): """ agent initialisation - reading and setting confing opts + reading and setting config opts :param self: self reference """ @@ -124,6 +125,49 @@ def initialize(self): # # file catalog client self.metadataClient = FileCatalogClient() + # Only at (re)start: will clean ancient transformations (remnants) + # 1) get the transformation IDs of jobs that are older than 1 year + # 2) find the status of those transformations. Those "Cleaned" and "Archived" will be + # cleaned and archived (again) + res = JobMonitoringClient().getJobGroups(None, datetime.utcnow() - timedelta(days=365)) + if not res['OK']: + self.log.error("Failed to get job groups", res['Message']) + return res + transformationIDs = res['Value'] + if transformationIDs: + res = TransformationClient().getTransformations({'TransformationID': list(transformationIDs)}) + if not res['OK']: + self.log.error("Failed to get transformations", res['Message']) + return res + transformations = res['Value'] + toClean = [] + toArchive = [] + for transDict in transformations: + if transDict['Status'] == 'Cleaned': + toClean.append(transDict) + if transDict['Status'] == 'Archived': + toArchive.append(transDict) + + for transDict in toClean: + if self.shifterProxy: + self._executeClean(transDict) + else: + self.log.info("Cleaning transformation %(TransformationID)s with %(AuthorDN)s, %(AuthorGroup)s" % + transDict) + executeWithUserProxy(self._executeClean)(transDict, + proxyUserDN=transDict['AuthorDN'], + proxyUserGroup=transDict['AuthorGroup']) + + for transDict in toArchive: + if self.shifterProxy: + self._executeArchive(transDict) + else: + self.log.info("Archiving files for transformation %(TransformationID)s with %(AuthorDN)s, %(AuthorGroup)s" % + transDict) + executeWithUserProxy(self._executeArchive)(transDict, + proxyUserDN=transDict['AuthorDN'], + proxyUserGroup=transDict['AuthorGroup']) + return S_OK() ############################################################################# From 8f2db06125f43de130e52f00dd9706feed95ab02 Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 12 Jun 2020 12:34:42 +0200 Subject: [PATCH 32/44] sanitize DNs --- ConfigurationSystem/Client/Helpers/Registry.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ConfigurationSystem/Client/Helpers/Registry.py b/ConfigurationSystem/Client/Helpers/Registry.py index eaa337add26..13eeef17da8 100644 --- a/ConfigurationSystem/Client/Helpers/Registry.py +++ b/ConfigurationSystem/Client/Helpers/Registry.py @@ -23,6 +23,7 @@ def getUsernameForDN(dn, usersList=None): :return: S_OK(str)/S_ERROR() """ + dn = dn.strip() if not usersList: result = gConfig.getSections("%s/Users" % gBaseRegistrySection) if not result['OK']: @@ -63,6 +64,7 @@ def getGroupsForDN(dn): :return: S_OK(list)/S_ERROR() -- contain list of groups """ + dn = dn.strip() result = getUsernameForDN(dn) if not result['OK']: return result @@ -128,6 +130,7 @@ def getHostnameForDN(dn): :return: S_OK()/S_ERROR() """ + dn = dn.strip() result = gConfig.getSections("%s/Hosts" % gBaseRegistrySection) if not result['OK']: return result @@ -153,6 +156,7 @@ def findDefaultGroupForDN(dn): :return: S_OK()/S_ERROR() """ + dn = dn.strip() result = getUsernameForDN(dn) if not result['OK']: return result From b371142dc4e484ce565f0fea6d60180b3eda83a5 Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 12 Jun 2020 12:57:31 +0200 Subject: [PATCH 33/44] no need for casting to list --- .../Agent/MCExtensionAgent.py | 2 +- .../Agent/TransformationPlugin.py | 4 +-- .../Agent/ValidateOutputDataAgent.py | 2 +- .../Client/TransformationCLI.py | 26 +++++++++---------- .../Utilities/TransformationInfo.py | 8 +++--- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/TransformationSystem/Agent/MCExtensionAgent.py b/TransformationSystem/Agent/MCExtensionAgent.py index 791e0b4d574..953349bdf37 100755 --- a/TransformationSystem/Agent/MCExtensionAgent.py +++ b/TransformationSystem/Agent/MCExtensionAgent.py @@ -80,7 +80,7 @@ def extendTransformation(self, transID, maxTasks): else: statusDict = res['Value'] gLogger.verbose("Current task count for transformation %d" % transID) - for status in sorted(list(statusDict)): + for status in sorted(statusDict): statusCount = statusDict[status] gLogger.verbose("%s : %s" % (status.ljust(20), str(statusCount).rjust(8))) # Determine the number of tasks to be created diff --git a/TransformationSystem/Agent/TransformationPlugin.py b/TransformationSystem/Agent/TransformationPlugin.py index ead4c3fb49f..98811de4cdb 100644 --- a/TransformationSystem/Agent/TransformationPlugin.py +++ b/TransformationSystem/Agent/TransformationPlugin.py @@ -177,7 +177,7 @@ def _ByShare(self, shareType='CPU'): return res cpuShares = res['Value'] self.util.logInfo("Obtained the following target shares (%):") - for site in sorted(list(cpuShares)): + for site in sorted(cpuShares): self.util.logInfo("%s: %.1f" % (site.ljust(15), cpuShares[site])) # Get the existing destinations from the transformationDB @@ -189,7 +189,7 @@ def _ByShare(self, shareType='CPU'): if existingCount: self.util.logInfo("Existing site utilization (%):") normalisedExistingCount = self.util._normaliseShares(existingCount.copy()) # pylint: disable=protected-access - for se in sorted(list(normalisedExistingCount)): + for se in sorted(normalisedExistingCount): self.util.logInfo("%s: %.1f" % (se.ljust(15), normalisedExistingCount[se])) # Group the input files by their existing replicas diff --git a/TransformationSystem/Agent/ValidateOutputDataAgent.py b/TransformationSystem/Agent/ValidateOutputDataAgent.py index 049321bf51e..87d194825a6 100755 --- a/TransformationSystem/Agent/ValidateOutputDataAgent.py +++ b/TransformationSystem/Agent/ValidateOutputDataAgent.py @@ -197,7 +197,7 @@ def checkTransformationIntegrity(self, transID): if res['Value']['Failed']: return S_ERROR("Failed to determine the existance of directories") directoryExists = res['Value']['Successful'] - for directory in sorted(list(directoryExists)): + for directory in sorted(directoryExists): if not directoryExists[directory]: continue iRes = self.consistencyInspector.catalogDirectoryToSE(directory) diff --git a/TransformationSystem/Client/TransformationCLI.py b/TransformationSystem/Client/TransformationCLI.py index 37f96a1f385..d209c0960bc 100644 --- a/TransformationSystem/Client/TransformationCLI.py +++ b/TransformationSystem/Client/TransformationCLI.py @@ -541,11 +541,11 @@ def do_replicas(self, args): if not res['OK']: print("failed to get any replica information: %s" % res['Message']) return - for lfn in sorted(list(res['Value']['Failed'])): + for lfn in sorted(res['Value']['Failed']): error = res['Value']['Failed'][lfn] print("failed to get replica information for %s: %s" % (lfn, error)) - for lfn in sorted(list(res['Value']['Successful'])): - ses = sorted(list(res['Value']['Successful'][lfn])) + for lfn in sorted(res['Value']['Successful']): + ses = sorted(res['Value']['Successful'][lfn]) outStr = "%s :" % lfn.ljust(100) for se in ses: outStr = "%s %s" % (outStr, se.ljust(15)) @@ -568,10 +568,10 @@ def do_addFile(self, args): if not res['OK']: print("failed to add any files: %s" % res['Message']) return - for lfn in sorted(list(res['Value']['Failed'])): + for lfn in sorted(res['Value']['Failed']): error = res['Value']['Failed'][lfn] print("failed to add %s: %s" % (lfn, error)) - for lfn in sorted(list(res['Value']['Successful'])): + for lfn in sorted(res['Value']['Successful']): print("added %s" % lfn) def do_removeFile(self, args): @@ -587,10 +587,10 @@ def do_removeFile(self, args): if not res['OK']: print("failed to remove any files: %s" % res['Message']) return - for lfn in sorted(list(res['Value']['Failed'])): + for lfn in sorted(res['Value']['Failed']): error = res['Value']['Failed'][lfn] print("failed to remove %s: %s" % (lfn, error)) - for lfn in sorted(list(res['Value']['Successful'])): + for lfn in sorted(res['Value']['Successful']): print("removed %s" % lfn) def do_addReplica(self, args): @@ -610,10 +610,10 @@ def do_addReplica(self, args): if not res['OK']: print("failed to add replica: %s" % res['Message']) return - for lfn in sorted(list(res['Value']['Failed'])): + for lfn in sorted(res['Value']['Failed']): error = res['Value']['Failed'][lfn] print("failed to add replica: %s" % (error)) - for lfn in sorted(list(res['Value']['Successful'])): + for lfn in sorted(res['Value']['Successful']): print("added %s" % lfn) def do_removeReplica(self, args): @@ -633,10 +633,10 @@ def do_removeReplica(self, args): if not res['OK']: print("failed to remove replica: %s" % res['Message']) return - for lfn in sorted(list(res['Value']['Failed'])): + for lfn in sorted(res['Value']['Failed']): error = res['Value']['Failed'][lfn] print("failed to remove replica: %s" % (error)) - for lfn in sorted(list(res['Value']['Successful'])): + for lfn in sorted(res['Value']['Successful']): print("removed %s" % lfn) def do_setReplicaStatus(self, args): @@ -663,10 +663,10 @@ def do_setReplicaStatus(self, args): if not res['OK']: print("failed to set replica status: %s" % res['Message']) return - for lfn in sorted(list(res['Value']['Failed'])): + for lfn in sorted(res['Value']['Failed']): error = res['Value']['Failed'][lfn] print("failed to set replica status: %s" % (error)) - for lfn in sorted(list(res['Value']['Successful'])): + for lfn in sorted(res['Value']['Successful']): print("updated replica status %s" % lfn) diff --git a/TransformationSystem/Utilities/TransformationInfo.py b/TransformationSystem/Utilities/TransformationInfo.py index 509151e28eb..c47da7dce90 100644 --- a/TransformationSystem/Utilities/TransformationInfo.py +++ b/TransformationSystem/Utilities/TransformationInfo.py @@ -152,10 +152,10 @@ def cleanOutputs(self, jobInfo): if not result['OK']: self.log.error("Failed to remove LFNs", result['Message']) raise RuntimeError("Failed to remove LFNs: %s" % result['Message']) - for lfn, err in result['Value']['Failed'].items(): - reason = str(err) - errorReasons[reason].append(lfn) - successfullyRemoved += len(list(result['Value']['Successful'])) + for lfn, err in result['Value']['Failed'].items(): + reason = str(err) + errorReasons[reason].append(lfn) + successfullyRemoved += len(result['Value']['Successful']) for reason, lfns in errorReasons.items(): self.log.error("Failed to remove %d files with error: %s" % (len(lfns), reason)) self.log.notice("Successfully removed %d files" % successfullyRemoved) From f9af4b86680fe5ee95e4a665c04026a7251647a8 Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 12 Jun 2020 16:24:53 +0200 Subject: [PATCH 34/44] moved clean of very old transformations to the finalize --- .../Agent/TransformationCleaningAgent.py | 102 ++++++++++-------- 1 file changed, 59 insertions(+), 43 deletions(-) diff --git a/TransformationSystem/Agent/TransformationCleaningAgent.py b/TransformationSystem/Agent/TransformationCleaningAgent.py index f8fb6ae2931..390ffadd6a7 100644 --- a/TransformationSystem/Agent/TransformationCleaningAgent.py +++ b/TransformationSystem/Agent/TransformationCleaningAgent.py @@ -124,49 +124,8 @@ def initialize(self): self.reqClient = ReqClient() # # file catalog client self.metadataClient = FileCatalogClient() - - # Only at (re)start: will clean ancient transformations (remnants) - # 1) get the transformation IDs of jobs that are older than 1 year - # 2) find the status of those transformations. Those "Cleaned" and "Archived" will be - # cleaned and archived (again) - res = JobMonitoringClient().getJobGroups(None, datetime.utcnow() - timedelta(days=365)) - if not res['OK']: - self.log.error("Failed to get job groups", res['Message']) - return res - transformationIDs = res['Value'] - if transformationIDs: - res = TransformationClient().getTransformations({'TransformationID': list(transformationIDs)}) - if not res['OK']: - self.log.error("Failed to get transformations", res['Message']) - return res - transformations = res['Value'] - toClean = [] - toArchive = [] - for transDict in transformations: - if transDict['Status'] == 'Cleaned': - toClean.append(transDict) - if transDict['Status'] == 'Archived': - toArchive.append(transDict) - - for transDict in toClean: - if self.shifterProxy: - self._executeClean(transDict) - else: - self.log.info("Cleaning transformation %(TransformationID)s with %(AuthorDN)s, %(AuthorGroup)s" % - transDict) - executeWithUserProxy(self._executeClean)(transDict, - proxyUserDN=transDict['AuthorDN'], - proxyUserGroup=transDict['AuthorGroup']) - - for transDict in toArchive: - if self.shifterProxy: - self._executeArchive(transDict) - else: - self.log.info("Archiving files for transformation %(TransformationID)s with %(AuthorDN)s, %(AuthorGroup)s" % - transDict) - executeWithUserProxy(self._executeArchive)(transDict, - proxyUserDN=transDict['AuthorDN'], - proxyUserGroup=transDict['AuthorGroup']) + # # job monitoring client + self.jobMonitoringClient = JobMonitoringClient() return S_OK() @@ -234,6 +193,63 @@ def execute(self): self.log.error("Could not get the transformations", res['Message']) return S_OK() + def finalize(self): + """ Only at finalization: will clean ancient transformations (remnants) + 1) get the transformation IDs of jobs that are older than 1 year + 2) find the status of those transformations. Those "Cleaned" and "Archived" will be + cleaned and archived (again) + """ + res = self.jobMonitoringClient.getJobGroups(None, datetime.utcnow() - timedelta(days=365)) + if not res['OK']: + self.log.error("Failed to get job groups", res['Message']) + return res + transformationIDs = res['Value'] + if transformationIDs: + res = self.transClient.getTransformations({'TransformationID': transformationIDs}) + if not res['OK']: + self.log.error("Failed to get transformations", res['Message']) + return res + transformations = res['Value'] + toClean = [] + toArchive = [] + for transDict in transformations: + if transDict['Status'] == 'Cleaned': + toClean.append(transDict) + if transDict['Status'] == 'Archived': + toArchive.append(transDict) + + for transDict in toClean: + if self.shifterProxy: + self._executeClean(transDict) + else: + self.log.info("Cleaning transformation %(TransformationID)s with %(AuthorDN)s, %(AuthorGroup)s" % + transDict) + executeWithUserProxy(self._executeClean)(transDict, + proxyUserDN=transDict['AuthorDN'], + proxyUserGroup=transDict['AuthorGroup']) + + for transDict in toArchive: + if self.shifterProxy: + self._executeArchive(transDict) + else: + self.log.info("Archiving files for transformation %(TransformationID)s with %(AuthorDN)s, %(AuthorGroup)s" % + transDict) + executeWithUserProxy(self._executeArchive)(transDict, + proxyUserDN=transDict['AuthorDN'], + proxyUserGroup=transDict['AuthorGroup']) + + # Remove JobIDs that were unknown to the TransformationSystem + jobGroupsToCheck = [str(transDict['TransformationID']).zfill(8) for transDict in toClean + toArchive] + res = self.jobMonitoringClient.getJobs({'JobGroup': jobGroupsToCheck}) + if not res['OK']: + return res + jobIDsToRemove = [int(jobID) for jobID in res['Value']] + res = self.__removeWMSTasks(jobIDsToRemove) + if not res['OK']: + return res + + return S_OK() + def _executeClean(self, transDict): """Clean transformation.""" # if transformation is of type `Replication` or `Removal`, there is nothing to clean. From 423ef412c67e8e4c68d56b5bf1fb7c0df3fc2c12 Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 25 Jun 2020 15:27:45 +0200 Subject: [PATCH 35/44] added explanation --- .../Agent/TransformationCleaningAgent.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/TransformationSystem/Agent/TransformationCleaningAgent.py b/TransformationSystem/Agent/TransformationCleaningAgent.py index 390ffadd6a7..02e1bbd84ad 100644 --- a/TransformationSystem/Agent/TransformationCleaningAgent.py +++ b/TransformationSystem/Agent/TransformationCleaningAgent.py @@ -198,6 +198,22 @@ def finalize(self): 1) get the transformation IDs of jobs that are older than 1 year 2) find the status of those transformations. Those "Cleaned" and "Archived" will be cleaned and archived (again) + + Why doing this here? Basically, it's a race: + 1) the production manager submits a transformation + 2) the TransformationAgent, and a bit later the WorkflowTaskAgent, put such transformation in their internal queue, + so eventually during their (long-ish) cycle they'll work on it. + 3) 1 minute after creating the transformation, the production manager cleans it (by hand, for whatever reason). + So, the status is changed to "Cleaning" + 4) the TransformationCleaningAgent cleans what has been created (maybe, nothing), + then sets the transformation status to "Cleaned" or "Archived" + 5) a bit later the TransformationAgent, and later the WorkflowTaskAgent, kick in, + creating tasks and jobs for a production that's effectively cleaned (but these 2 agents don't know yet). + + Of course, one could make one final check in TransformationAgent or WorkflowTaskAgent, + but these 2 agents are already doing a lot of stuff, and are pretty heavy. + So, we should just clean from time to time. + What I added here is done only when the agent finalize, and it's quite light-ish operation anyway. """ res = self.jobMonitoringClient.getJobGroups(None, datetime.utcnow() - timedelta(days=365)) if not res['OK']: From c6d239e51265445b389b2dfdc0806db6ffdd16ae Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 25 Jun 2020 15:30:16 +0200 Subject: [PATCH 36/44] implement suggestions on spacing in lists --- .../Agent/TaskManagerAgentBase.py | 2 +- .../Agent/TransformationCleaningAgent.py | 64 ++++++++++--------- TransformationSystem/DB/TransformationDB.py | 2 +- .../Utilities/TransformationInfo.py | 8 +-- .../Test_Client_WMS.py | 2 +- 5 files changed, 40 insertions(+), 38 deletions(-) diff --git a/TransformationSystem/Agent/TaskManagerAgentBase.py b/TransformationSystem/Agent/TaskManagerAgentBase.py index 036fc0d4488..879a6774ddb 100644 --- a/TransformationSystem/Agent/TaskManagerAgentBase.py +++ b/TransformationSystem/Agent/TaskManagerAgentBase.py @@ -285,7 +285,7 @@ def _execute(self, threadID): # Queue was cleared, nothing to do continue try: - transID = list(transIDOPBody)[0] + transID = list(transIDOPBody)[0] operations = transIDOPBody[transID]['Operations'] if transID not in self.transInQueue: self._logWarn("Got a transf not in transInQueue...?", diff --git a/TransformationSystem/Agent/TransformationCleaningAgent.py b/TransformationSystem/Agent/TransformationCleaningAgent.py index 02e1bbd84ad..74d49bd24ae 100644 --- a/TransformationSystem/Agent/TransformationCleaningAgent.py +++ b/TransformationSystem/Agent/TransformationCleaningAgent.py @@ -195,20 +195,22 @@ def execute(self): def finalize(self): """ Only at finalization: will clean ancient transformations (remnants) - 1) get the transformation IDs of jobs that are older than 1 year - 2) find the status of those transformations. Those "Cleaned" and "Archived" will be - cleaned and archived (again) + + 1) get the transformation IDs of jobs that are older than 1 year + 2) find the status of those transformations. Those "Cleaned" and "Archived" will be + cleaned and archived (again) Why doing this here? Basically, it's a race: + 1) the production manager submits a transformation 2) the TransformationAgent, and a bit later the WorkflowTaskAgent, put such transformation in their internal queue, - so eventually during their (long-ish) cycle they'll work on it. + so eventually during their (long-ish) cycle they'll work on it. 3) 1 minute after creating the transformation, the production manager cleans it (by hand, for whatever reason). - So, the status is changed to "Cleaning" + So, the status is changed to "Cleaning" 4) the TransformationCleaningAgent cleans what has been created (maybe, nothing), - then sets the transformation status to "Cleaned" or "Archived" + then sets the transformation status to "Cleaned" or "Archived" 5) a bit later the TransformationAgent, and later the WorkflowTaskAgent, kick in, - creating tasks and jobs for a production that's effectively cleaned (but these 2 agents don't know yet). + creating tasks and jobs for a production that's effectively cleaned (but these 2 agents don't know yet). Of course, one could make one final check in TransformationAgent or WorkflowTaskAgent, but these 2 agents are already doing a lot of stuff, and are pretty heavy. @@ -223,46 +225,46 @@ def finalize(self): if transformationIDs: res = self.transClient.getTransformations({'TransformationID': transformationIDs}) if not res['OK']: - self.log.error("Failed to get transformations", res['Message']) - return res + self.log.error("Failed to get transformations", res['Message']) + return res transformations = res['Value'] toClean = [] toArchive = [] for transDict in transformations: - if transDict['Status'] == 'Cleaned': - toClean.append(transDict) - if transDict['Status'] == 'Archived': - toArchive.append(transDict) + if transDict['Status'] == 'Cleaned': + toClean.append(transDict) + if transDict['Status'] == 'Archived': + toArchive.append(transDict) for transDict in toClean: - if self.shifterProxy: - self._executeClean(transDict) - else: - self.log.info("Cleaning transformation %(TransformationID)s with %(AuthorDN)s, %(AuthorGroup)s" % - transDict) - executeWithUserProxy(self._executeClean)(transDict, - proxyUserDN=transDict['AuthorDN'], - proxyUserGroup=transDict['AuthorGroup']) + if self.shifterProxy: + self._executeClean(transDict) + else: + self.log.info("Cleaning transformation %(TransformationID)s with %(AuthorDN)s, %(AuthorGroup)s" % + transDict) + executeWithUserProxy(self._executeClean)(transDict, + proxyUserDN=transDict['AuthorDN'], + proxyUserGroup=transDict['AuthorGroup']) for transDict in toArchive: - if self.shifterProxy: - self._executeArchive(transDict) - else: - self.log.info("Archiving files for transformation %(TransformationID)s with %(AuthorDN)s, %(AuthorGroup)s" % - transDict) - executeWithUserProxy(self._executeArchive)(transDict, - proxyUserDN=transDict['AuthorDN'], - proxyUserGroup=transDict['AuthorGroup']) + if self.shifterProxy: + self._executeArchive(transDict) + else: + self.log.info("Archiving files for transformation %(TransformationID)s with %(AuthorDN)s, %(AuthorGroup)s" % + transDict) + executeWithUserProxy(self._executeArchive)(transDict, + proxyUserDN=transDict['AuthorDN'], + proxyUserGroup=transDict['AuthorGroup']) # Remove JobIDs that were unknown to the TransformationSystem jobGroupsToCheck = [str(transDict['TransformationID']).zfill(8) for transDict in toClean + toArchive] res = self.jobMonitoringClient.getJobs({'JobGroup': jobGroupsToCheck}) if not res['OK']: - return res + return res jobIDsToRemove = [int(jobID) for jobID in res['Value']] res = self.__removeWMSTasks(jobIDsToRemove) if not res['OK']: - return res + return res return S_OK() diff --git a/TransformationSystem/DB/TransformationDB.py b/TransformationSystem/DB/TransformationDB.py index 3d72e58f0b6..1b1fd24ceb1 100755 --- a/TransformationSystem/DB/TransformationDB.py +++ b/TransformationSystem/DB/TransformationDB.py @@ -550,7 +550,7 @@ def getTransformationFiles(self, condDict=None, older=None, newer=None, timeStam if not res['OK']: return res originalFileIDs = res['Value'][0] - condDict['FileID'] = list(originalFileIDs) + condDict['FileID'] = list(originalFileIDs) for val in condDict.itervalues(): if not val: diff --git a/TransformationSystem/Utilities/TransformationInfo.py b/TransformationSystem/Utilities/TransformationInfo.py index c47da7dce90..a6a0bc97938 100644 --- a/TransformationSystem/Utilities/TransformationInfo.py +++ b/TransformationSystem/Utilities/TransformationInfo.py @@ -152,10 +152,10 @@ def cleanOutputs(self, jobInfo): if not result['OK']: self.log.error("Failed to remove LFNs", result['Message']) raise RuntimeError("Failed to remove LFNs: %s" % result['Message']) - for lfn, err in result['Value']['Failed'].items(): - reason = str(err) - errorReasons[reason].append(lfn) - successfullyRemoved += len(result['Value']['Successful']) + for lfn, err in result['Value']['Failed'].items(): + reason = str(err) + errorReasons[reason].append(lfn) + successfullyRemoved += len(result['Value']['Successful']) for reason, lfns in errorReasons.items(): self.log.error("Failed to remove %d files with error: %s" % (len(lfns), reason)) self.log.notice("Successfully removed %d files" % successfullyRemoved) diff --git a/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py b/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py index 30043850e8e..780db269479 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py +++ b/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py @@ -391,7 +391,7 @@ def test_JobStateUpdateAndJobMonitoringMultuple(self): try: self.assertTrue( res['Value'].get('Received') + - res['Value'].get('Waiting') >= int(len(lfnss) * len(types))) + res['Value'].get('Waiting') >= int(len(lfnss) * len(types))) except TypeError: pass res = jobMonitor.getJobsSummary(jobIDs) From 41fc0d6255c2995f34da49dc57f6ac10a4c64892 Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 12 Jun 2020 12:34:42 +0200 Subject: [PATCH 37/44] sanitize DNs --- ConfigurationSystem/Client/Helpers/Registry.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ConfigurationSystem/Client/Helpers/Registry.py b/ConfigurationSystem/Client/Helpers/Registry.py index eaa337add26..13eeef17da8 100644 --- a/ConfigurationSystem/Client/Helpers/Registry.py +++ b/ConfigurationSystem/Client/Helpers/Registry.py @@ -23,6 +23,7 @@ def getUsernameForDN(dn, usersList=None): :return: S_OK(str)/S_ERROR() """ + dn = dn.strip() if not usersList: result = gConfig.getSections("%s/Users" % gBaseRegistrySection) if not result['OK']: @@ -63,6 +64,7 @@ def getGroupsForDN(dn): :return: S_OK(list)/S_ERROR() -- contain list of groups """ + dn = dn.strip() result = getUsernameForDN(dn) if not result['OK']: return result @@ -128,6 +130,7 @@ def getHostnameForDN(dn): :return: S_OK()/S_ERROR() """ + dn = dn.strip() result = gConfig.getSections("%s/Hosts" % gBaseRegistrySection) if not result['OK']: return result @@ -153,6 +156,7 @@ def findDefaultGroupForDN(dn): :return: S_OK()/S_ERROR() """ + dn = dn.strip() result = getUsernameForDN(dn) if not result['OK']: return result From 816e5d66aeb4a9b9c7282ac11d4fa3ba70159bf7 Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 15 Jun 2020 10:47:14 +0200 Subject: [PATCH 38/44] minor/py3 fixes --- .../Client/BundleDeliveryClient.py | 2 + FrameworkSystem/Client/ProxyManagerClient.py | 90 +++++++++---------- .../Service/ProxyManagerHandler.py | 74 +++++++-------- 3 files changed, 84 insertions(+), 82 deletions(-) diff --git a/FrameworkSystem/Client/BundleDeliveryClient.py b/FrameworkSystem/Client/BundleDeliveryClient.py index eeb72c8c24e..f73b404b69b 100644 --- a/FrameworkSystem/Client/BundleDeliveryClient.py +++ b/FrameworkSystem/Client/BundleDeliveryClient.py @@ -10,6 +10,8 @@ from DIRAC.Core.DISET.TransferClient import TransferClient from DIRAC.Core.Security import Locations, Utilities from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck + + __RCSID__ = "$Id$" diff --git a/FrameworkSystem/Client/ProxyManagerClient.py b/FrameworkSystem/Client/ProxyManagerClient.py index c8a42db8c13..d1df5a017ba 100755 --- a/FrameworkSystem/Client/ProxyManagerClient.py +++ b/FrameworkSystem/Client/ProxyManagerClient.py @@ -1,4 +1,4 @@ -""" ProxyManagementAPI has the functions to "talk" to the ProxyManagement service +""" ProxyManagemerClient has the function to "talk" to the ProxyManagemer service """ import six import os @@ -35,11 +35,11 @@ def __init__(self): def __deleteTemporalFile(self, filename): """ Delete temporal file - :param basestring filename: path to file + :param str filename: path to file """ try: - os.unlink(filename) - except BaseException: + os.remove(filename) + except Exception: pass def clearCaches(self): @@ -89,8 +89,8 @@ def userHasProxy(self, userDN, userGroup, validSeconds=0): """ Check if a user(DN-group) has a proxy in the proxy management Updates internal cache if needed to minimize queries to the service - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param int validSeconds: proxy valid time in a seconds :return: S_OK()/S_ERROR() @@ -110,8 +110,8 @@ def getUserPersistence(self, userDN, userGroup, validSeconds=0): """ Check if a user(DN-group) has a proxy in the proxy management Updates internal cache if needed to minimize queries to the service - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param int validSeconds: proxy valid time in a seconds :return: S_OK()/S_ERROR() @@ -134,8 +134,8 @@ def getUserPersistence(self, userDN, userGroup, validSeconds=0): def setPersistency(self, userDN, userGroup, persistent): """ Set the persistency for user/group - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean persistent: presistent flag :return: S_OK()/S_ERROR() @@ -219,13 +219,13 @@ def downloadProxy(self, userDN, userGroup, limited=False, requiredTimeLeft=1200, cacheTime=14400, proxyToConnect=None, token=None): """ Get a proxy Chain from the proxy management - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds :param X509Chain proxyToConnect: proxy as a chain - :param basestring token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -257,14 +257,14 @@ def downloadProxyToFile(self, userDN, userGroup, limited=False, requiredTimeLeft cacheTime=14400, filePath=None, proxyToConnect=None, token=None): """ Get a proxy Chain from the proxy management and write it to file - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds - :param basestring filePath: path to save proxy + :param str filePath: path to save proxy :param X509Chain proxyToConnect: proxy as a chain - :param basestring token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -284,14 +284,14 @@ def downloadVOMSProxy(self, userDN, userGroup, limited=False, requiredTimeLeft=1 proxyToConnect=None, token=None): """ Download a proxy if needed and transform it into a VOMS one - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds - :param basestring requiredVOMSAttribute: VOMS attr to add to the proxy + :param str requiredVOMSAttribute: VOMS attr to add to the proxy :param X509Chain proxyToConnect: proxy as a chain - :param basestring token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -325,15 +325,15 @@ def downloadVOMSProxyToFile(self, userDN, userGroup, limited=False, requiredTime proxyToConnect=None, token=None): """ Download a proxy if needed, transform it into a VOMS one and write it to file - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds - :param basestring requiredVOMSAttribute: VOMS attr to add to the proxy - :param basestring filePath: path to save proxy + :param str requiredVOMSAttribute: VOMS attr to add to the proxy + :param str filePath: path to save proxy :param X509Chain proxyToConnect: proxy as a chain - :param basestring token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -351,9 +351,9 @@ def downloadVOMSProxyToFile(self, userDN, userGroup, limited=False, requiredTime def getPilotProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft=43200, proxyToConnect=None): """ Download a pilot proxy with VOMS extensions depending on the group - :param basestring userDN: user DN - :param basestring userGroup: user group - :param int requiredTimeLeft: required proxy live time in a seconds + :param str userDN: user DN + :param str userGroup: user group + :param int requiredTimeLeft: required proxy live time in seconds :param X509Chain proxyToConnect: proxy as a chain :return: S_OK(X509Chain)/S_ERROR() @@ -361,7 +361,7 @@ def getPilotProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft=43200, # Assign VOMS attribute vomsAttr = Registry.getVOMSAttributeForGroup(userGroup) if not vomsAttr: - gLogger.verbose("No voms attribute assigned to group %s when requested pilot proxy" % userGroup) + gLogger.warn("No voms attribute assigned to group %s when requested pilot proxy" % userGroup) return self.downloadProxy(userDN, userGroup, limited=False, requiredTimeLeft=requiredTimeLeft, proxyToConnect=proxyToConnect) else: @@ -371,8 +371,8 @@ def getPilotProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft=43200, def getPilotProxyFromVOMSGroup(self, userDN, vomsAttr, requiredTimeLeft=43200, proxyToConnect=None): """ Download a pilot proxy with VOMS extensions depending on the group - :param basestring userDN: user DN - :param basestring vomsAttr: VOMS attribute + :param str userDN: user DN + :param str vomsAttr: VOMS attribute :param int requiredTimeLeft: required proxy live time in a seconds :param X509Chain proxyToConnect: proxy as a chain @@ -395,10 +395,10 @@ def getPilotProxyFromVOMSGroup(self, userDN, vomsAttr, requiredTimeLeft=43200, p def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, token=None, proxyToConnect=None): """ Download a payload proxy with VOMS extensions depending on the group - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param int requiredTimeLeft: required proxy live time in a seconds - :param basestring token: valid token to get a proxy + :param str token: valid token to get a proxy :param X509Chain proxyToConnect: proxy as a chain :return: S_OK(X509Chain)/S_ERROR() @@ -417,9 +417,9 @@ def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, tok def getPayloadProxyFromVOMSGroup(self, userDN, vomsAttr, token, requiredTimeLeft, proxyToConnect=None): """ Download a payload proxy with VOMS extensions depending on the VOMS attr - :param basestring userDN: user DN - :param basestring vomsAttr: VOMS attribute - :param basestring token: valid token to get a proxy + :param str userDN: user DN + :param str vomsAttr: VOMS attribute + :param str token: valid token to get a proxy :param int requiredTimeLeft: required proxy live time in a seconds :param X509Chain proxyToConnect: proxy as a chain @@ -442,10 +442,10 @@ def dumpProxyToFile(self, chain, destinationFile=None, requiredTimeLeft=600): """ Dump a proxy to a file. It's cached so multiple calls won't generate extra files :param X509Chain chain: proxy as a chain - :param basestring destinationFile: path to store proxy + :param str destinationFile: path to store proxy :param int requiredTimeLeft: required proxy live time in a seconds - :return: S_OK(basestring)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ result = chain.hash() if not result['OK']: @@ -477,8 +477,8 @@ def requestToken(self, requesterDN, requesterGroup, numUses=1): """ Request a number of tokens. usesList must be a list of integers and each integer is the number of uses a token must have - :param basestring requesterDN: user DN - :param basestring requesterGroup: user group + :param str requesterDN: user DN + :param str requesterGroup: user group :param int numUses: number of uses :return: S_OK(tuple)/S_ERROR() -- tuple contain token, number uses @@ -573,15 +573,15 @@ def getVOMSAttributes(self, chain): :param X509Chain chain: proxy as a chain - :return: S_OK(basestring)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ return VOMS().getVOMSAttributes(chain) def getUploadedProxyLifeTime(self, DN, group): """ Get the remaining seconds for an uploaded proxy - :param basestring DN: user DN - :param basestring group: group + :param str DN: user DN + :param str group: group :return: S_OK(int)/S_ERROR() """ diff --git a/FrameworkSystem/Service/ProxyManagerHandler.py b/FrameworkSystem/Service/ProxyManagerHandler.py index 8f0b3c0b92e..2aabb0d17a7 100644 --- a/FrameworkSystem/Service/ProxyManagerHandler.py +++ b/FrameworkSystem/Service/ProxyManagerHandler.py @@ -4,7 +4,6 @@ __RCSID__ = "$Id$" -from past.builtins import long import six from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.DISET.RequestHandler import RequestHandler @@ -37,7 +36,8 @@ def initializeHandler(cls, serviceInfoDict): gThreadScheduler.addPeriodicTask(900, cls.__proxyDB.purgeExpiredRequests, elapsedTime=900) gThreadScheduler.addPeriodicTask(21600, cls.__proxyDB.purgeLogs) gThreadScheduler.addPeriodicTask(3600, cls.__proxyDB.purgeExpiredProxies) - gLogger.info("MyProxy: %s\n MyProxy Server: %s" % (useMyProxy, cls.__proxyDB.getMyProxyServer())) + if useMyProxy: + gLogger.info("MyProxy: %s\n MyProxy Server: %s" % (useMyProxy, cls.__proxyDB.getMyProxyServer())) return S_OK() def __generateUserProxiesInfo(self): @@ -86,12 +86,12 @@ def export_getUserProxiesInfo(self): # WARN: Since v7r1 requestDelegationUpload method use only first argument! # WARN: Second argument for compatibility with older versions - types_requestDelegationUpload = [[int, long], [basestring, bool, type(None)]] + types_requestDelegationUpload = [six.integer_types, [six.string_types, bool, type(None)]] def export_requestDelegationUpload(self, requestedUploadTime, diracGroup=None): """ Request a delegation. Send a delegation request to client - :param int,long requestedUploadTime: requested live time + :param int requestedUploadTime: requested live time :return: S_OK(dict)/S_ERROR() -- dict contain id and proxy as string of the request """ @@ -120,13 +120,13 @@ def export_requestDelegationUpload(self, requestedUploadTime, diracGroup=None): gLogger.error("Upload request failed", "by %s:%s : %s" % (userName, userGroup, result['Message'])) return result - types_completeDelegationUpload = [six.integer_types, basestring] + types_completeDelegationUpload = [six.integer_types, six.string_types] def export_completeDelegationUpload(self, requestId, pemChain): """ Upload result of delegation :param int,long requestId: identity number - :param basestring pemChain: certificate as string + :param str pemChain: certificate as string :return: S_OK(dict)/S_ERROR() -- dict contain proxies """ @@ -157,8 +157,8 @@ def export_getRegisteredUsers(self, validSecondsRequired=0): def __checkProperties(self, requestedUserDN, requestedUserGroup): """ Check the properties and return if they can only download limited proxies if authorized - :param basestring requestedUserDN: user DN - :param basestring requestedUserGroup: DIRAC group + :param str requestedUserDN: user DN + :param str requestedUserGroup: DIRAC group :return: S_OK(boolean)/S_ERROR() """ @@ -176,7 +176,7 @@ def __checkProperties(self, requestedUserDN, requestedUserGroup): # Not authorized! return S_ERROR("You can't get proxies!") - types_getProxy = [basestring, basestring, basestring, six.integer_types] + types_getProxy = [six.string_types, six.string_types, six.string_types, six.integer_types] def export_getProxy(self, userDN, userGroup, requestPem, requiredLifetime): """ Get a proxy for a userDN/userGroup @@ -202,13 +202,13 @@ def export_getProxy(self, userDN, userGroup, requestPem, requiredLifetime): def __getProxy(self, userDN, userGroup, requestPem, requiredLifetime, forceLimited): """ Internal to get a proxy - :param basestring userDN: user DN - :param basestring userGroup: DIRAC group - :param basestring requestPem: dump of request certificate - :param int,long requiredLifetime: requested live time of proxy + :param str userDN: user DN + :param str userGroup: DIRAC group + :param str requestPem: dump of request certificate + :param int requiredLifetime: requested live time of proxy :param boolean forceLimited: limited proxy - :return: S_OK(basestring)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ retVal = self.__proxyDB.getProxy(userDN, userGroup, requiredLifeTime=requiredLifetime) if not retVal['OK']: @@ -223,7 +223,9 @@ def __getProxy(self, userDN, userGroup, requestPem, requiredLifetime, forceLimit return retVal return S_OK(retVal['Value']) - types_getVOMSProxy = [basestring, basestring, basestring, six.integer_types, [basestring, type(None), bool]] + types_getVOMSProxy = [six.string_types, six.string_types, + six.string_types, six.integer_types, + [six.string_types, type(None), bool]] def export_getVOMSProxy(self, userDN, userGroup, requestPem, requiredLifetime, vomsAttribute=None): """ Get a proxy for a userDN/userGroup @@ -256,21 +258,17 @@ def __getVOMSProxy(self, userDN, userGroup, requestPem, requiredLifetime, vomsAt chain, secsLeft = retVal['Value'] # If possible we return a proxy 1.5 longer than requested requiredLifetime = int(min(secsLeft, requiredLifetime * self.__maxExtraLifeFactor)) - retVal = chain.generateChainFromRequestString(requestPem, - lifetime=requiredLifetime, - requireLimited=forceLimited) - if not retVal['OK']: - return retVal - _credDict = self.getRemoteCredentials() - return S_OK(retVal['Value']) + return chain.generateChainFromRequestString(requestPem, + lifetime=requiredLifetime, + requireLimited=forceLimited) - types_setPersistency = [basestring, basestring, bool] + types_setPersistency = [six.string_types, six.string_types, bool] def export_setPersistency(self, userDN, userGroup, persistentFlag): """ Set the persistency for a given dn/group - :param basestring userDN: user DN - :param basestring userGroup: DIRAC group + :param str userDN: user DN + :param str userGroup: DIRAC group :param boolean persistentFlag: if proxy persistent :return: S_OK()/S_ERROR() @@ -311,8 +309,8 @@ def export_deleteProxyBundle(self, idList): def export_deleteProxy(self, userDN, userGroup): """ Delete a proxy from the DB - :param basestring userDN: user DN - :param basestring userGroup: DIRAC group + :param str userDN: user DN + :param str userGroup: DIRAC group :return: S_OK()/S_ERROR() """ @@ -333,8 +331,8 @@ def export_getContents(self, selDict, sortDict, start, limit): :param dict selDict: selection fields :param list,tuple sortDict: sorting fields - :param int,long start: search limit start - :param int,long start: search limit amount + :param int start: search limit start + :param int start: search limit amount :return: S_OK(dict)/S_ERROR() -- dict contain fields, record list, total records """ @@ -350,21 +348,21 @@ def export_getLogContents(self, selDict, sortDict, start, limit): :param dict selDict: selection fields :param list,tuple sortDict: search filter - :param int,long start: search limit start - :param int,long start: search limit amount + :param int start: search limit start + :param int start: search limit amount :return: S_OK(dict)/S_ERROR() -- dict contain fields, record list, total records """ return self.__proxyDB.getLogsContent(selDict, sortDict, start, limit) - types_generateToken = [basestring, basestring, six.integer_types] + types_generateToken = [six.string_types, six.string_types, six.integer_types] def export_generateToken(self, requesterDN, requesterGroup, tokenUses): """ Generate tokens for proxy retrieval - :param basestring requesterDN: user DN - :param basestring requesterGroup: DIRAC group - :param int,long tokenUses: number of uses + :param str requesterDN: user DN + :param str requesterGroup: DIRAC group + :param int tokenUses: number of uses :return: S_OK(tuple)/S_ERROR() -- tuple contain token, number uses """ @@ -372,7 +370,7 @@ def export_generateToken(self, requesterDN, requesterGroup, tokenUses): self.__proxyDB.logAction("generate tokens", credDict['DN'], credDict['group'], requesterDN, requesterGroup) return self.__proxyDB.generateToken(requesterDN, requesterGroup, numUses=tokenUses) - types_getProxyWithToken = [basestring, basestring, basestring, six.integer_types, basestring] + types_getProxyWithToken = [six.string_types, six.string_types, six.string_types, six.integer_types, six.string_types] def export_getProxyWithToken(self, userDN, userGroup, requestPem, requiredLifetime, token): """ Get a proxy for a userDN/userGroup @@ -401,7 +399,9 @@ def export_getProxyWithToken(self, userDN, userGroup, requestPem, requiredLifeti self.__proxyDB.logAction("download proxy with token", credDict['DN'], credDict['group'], userDN, userGroup) return self.__getProxy(userDN, userGroup, requestPem, requiredLifetime, True) - types_getVOMSProxyWithToken = [basestring, basestring, basestring, six.integer_types, [basestring, type(None)]] + types_getVOMSProxyWithToken = [six.string_types, six.string_types, + six.string_types, six.integer_types, + [six.string_types, type(None)]] def export_getVOMSProxyWithToken(self, userDN, userGroup, requestPem, requiredLifetime, token, vomsAttribute=None): """ Get a proxy for a userDN/userGroup From 9de4febbac5ff229d6436903127c673be5005d34 Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 15 Jun 2020 11:01:34 +0200 Subject: [PATCH 39/44] inheriting from Client --- FrameworkSystem/Client/BundleDeliveryClient.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/FrameworkSystem/Client/BundleDeliveryClient.py b/FrameworkSystem/Client/BundleDeliveryClient.py index f73b404b69b..364ae701bcf 100644 --- a/FrameworkSystem/Client/BundleDeliveryClient.py +++ b/FrameworkSystem/Client/BundleDeliveryClient.py @@ -6,28 +6,24 @@ import cStringIO from DIRAC import S_OK, gLogger -from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.Core.Base.Client import Client, createClient from DIRAC.Core.DISET.TransferClient import TransferClient from DIRAC.Core.Security import Locations, Utilities +from DIRAC.Core.Utilities.File import mkDir from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck __RCSID__ = "$Id$" -class BundleDeliveryClient(object): +@createClient('Framework/BundleDelivery') +class BundleDeliveryClient(Client): - def __init__(self, rpcClient=False, transferClient=False): - self.rpcClient = rpcClient + def __init__(self, transferClient=False, **kwargs): + Client.__init__(self, **kwargs) self.transferClient = transferClient self.log = gLogger.getSubLogger("BundleDelivery") - def __getRPCClient(self): - if self.rpcClient: - return self.rpcClient - return RPCClient("Framework/BundleDelivery", - skipCACheck=skipCACheck()) - def __getTransferClient(self): if self.transferClient: return self.transferClient @@ -54,7 +50,7 @@ def syncDir(self, bundleID, dirToSyncTo): dirCreated = False if not os.path.isdir(dirToSyncTo): self.log.info("Creating dir %s" % dirToSyncTo) - os.makedirs(dirToSyncTo) + mkDir(dirToSyncTo) dirCreated = True currentHash = self.__getHash(bundleID, dirToSyncTo) self.log.info("Current hash for bundle %s in dir %s is '%s'" % (bundleID, dirToSyncTo, currentHash)) From 846b85160067b258b7b5c2f28ce7c63b4dc2692f Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 15 Jun 2020 11:18:01 +0200 Subject: [PATCH 40/44] some py2 to 3 fixes --- .../Client/BundleDeliveryClient.py | 9 +- FrameworkSystem/Client/ComponentInstaller.py | 88 +++++++++---------- 2 files changed, 46 insertions(+), 51 deletions(-) diff --git a/FrameworkSystem/Client/BundleDeliveryClient.py b/FrameworkSystem/Client/BundleDeliveryClient.py index 364ae701bcf..fd37907b5b5 100644 --- a/FrameworkSystem/Client/BundleDeliveryClient.py +++ b/FrameworkSystem/Client/BundleDeliveryClient.py @@ -2,6 +2,7 @@ """ import os +import io import tarfile import cStringIO @@ -32,7 +33,7 @@ def __getTransferClient(self): def __getHash(self, bundleID, dirToSyncTo): try: - with open(os.path.join(dirToSyncTo, ".dab.%s" % bundleID), "rb") as fd: + with io.open(os.path.join(dirToSyncTo, ".dab.%s" % bundleID), "rb") as fd: bdHash = fd.read().strip() return bdHash except BaseException: @@ -41,7 +42,7 @@ def __getHash(self, bundleID, dirToSyncTo): def __setHash(self, bundleID, dirToSyncTo, bdHash): try: fileName = os.path.join(dirToSyncTo, ".dab.%s" % bundleID) - with open(fileName, "wb") as fd: + with io.open(fileName, "wb") as fd: fd.write(bdHash) except Exception as e: self.log.error("Could not save hash after synchronization", "%s: %s" % (fileName, str(e))) @@ -111,7 +112,7 @@ def getCAs(self): # if we can not found the file, we return the directory, where the file should be transferClient = self.__getTransferClient() casFile = os.path.join(os.path.dirname(retVal['Message']), "cas.pem") - with open(casFile, "w") as fd: + with io.open(casFile, "w") as fd: result = transferClient.receiveFile(fd, 'CAs') if not result['OK']: return result @@ -129,7 +130,7 @@ def getCLRs(self): # if we can not found the file, we return the directory, where the file should be transferClient = self.__getTransferClient() casFile = os.path.join(os.path.dirname(retVal['Message']), "crls.pem") - with open(casFile, "w") as fd: + with io.open(casFile, "w") as fd: result = transferClient.receiveFile(fd, 'CRLs') if not result['OK']: return result diff --git a/FrameworkSystem/Client/ComponentInstaller.py b/FrameworkSystem/Client/ComponentInstaller.py index cf60348151e..6382d9ec459 100644 --- a/FrameworkSystem/Client/ComponentInstaller.py +++ b/FrameworkSystem/Client/ComponentInstaller.py @@ -58,6 +58,7 @@ from __future__ import print_function, absolute_import import os +import io import re import glob import stat @@ -1014,7 +1015,7 @@ def getSoftwareComponents(self, extensions): for agent in agentList: if os.path.splitext(agent)[1] == ".py": agentFile = os.path.join(agentDir, agent) - with open(agentFile, 'r') as afile: + with io.open(agentFile, 'rt') as afile: body = afile.read() if body.find('AgentModule') != -1 or body.find('OptimizerModule') != -1: if system not in agents: @@ -1040,7 +1041,7 @@ def getSoftwareComponents(self, extensions): for executor in executorList: if os.path.splitext(executor)[1] == ".py": executorFile = os.path.join(executorDir, executor) - with open(executorFile, 'r') as afile: + with io.open(executorFile, 'rt') as afile: body = afile.read() if body.find('OptimizerExecutor') != -1: if system not in executors: @@ -1091,9 +1092,8 @@ def getInstalledComponents(self): for component in components: try: runFile = os.path.join(systemDir, component, 'run') - rfile = open(runFile, 'r') - body = rfile.read() - rfile.close() + with io.open(runFile, 'rt') as rFile: + body = rFile.read() for cType in self.componentTypes: if body.find('dirac-%s' % (cType)) != -1: @@ -1126,9 +1126,8 @@ def getSetupComponents(self): for component in componentList: try: runFile = os.path.join(self.startDir, component, 'run') - rfile = open(runFile, 'r') - body = rfile.read() - rfile.close() + with io.open(runFile, 'rt') as rfile: + body = rfile.read() for cType in self.componentTypes: if body.find('dirac-%s' % (cType)) != -1: @@ -1463,9 +1462,8 @@ def getLogTail(self, system, component, length=100): if not os.path.exists(logFileName): retDict[compName] = 'No log file found' else: - logFile = open(logFileName, 'r') - lines = [line.strip() for line in logFile.readlines()] - logFile.close() + with io.open(logFileName, 'rt') as logFile: + lines = [line.strip() for line in logFile.readlines()] if len(lines) < length: retDict[compName] = '\n'.join(lines) @@ -1601,7 +1599,7 @@ def setupSite(self, scriptCfg, cfg=None): if not cmdFound: gLogger.notice('Starting runsvdir ...') - with open(os.devnull, 'w') as devnull: + with io.open(os.devnull, 'w') as devnull: subprocess.Popen(['nohup', 'runsvdir', self.startDir, 'log: DIRAC runsv'], stdout=devnull, stderr=devnull, universal_newlines=True) @@ -1815,16 +1813,16 @@ def _createRunitLog(self, runitCompDir): mkDir(logDir) logConfigFile = os.path.join(logDir, 'config') - with open(logConfigFile, 'w') as fd: + with io.open(logConfigFile, 'w') as fd: fd.write( - """s10000000 + u"""s10000000 n20 """) logRunFile = os.path.join(logDir, 'run') - with open(logRunFile, 'w') as fd: + with io.open(logRunFile, 'w') as fd: fd.write( - """#!/bin/bash + u"""#!/bin/bash # rcfile=%(bashrc)s [ -e $rcfile ] && source $rcfile @@ -1896,31 +1894,29 @@ def installComponent(self, componentType, system, component, extensions, compone try: componentCfg = os.path.join(self.linkedRootPath, 'etc', '%s_%s.cfg' % (system, component)) if not os.path.exists(componentCfg): - fd = open(componentCfg, 'w') - fd.close() + io.open(componentCfg, 'w').close() self._createRunitLog(runitCompDir) runFile = os.path.join(runitCompDir, 'run') - fd = open(runFile, 'w') - fd.write( - """#!/bin/bash - rcfile=%(bashrc)s - [ -e $rcfile ] && source $rcfile - # - exec 2>&1 - # - [ "%(componentType)s" = "agent" ] && renice 20 -p $$ - #%(bashVariables)s - # - exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py %(system)s/%(component)s %(componentCfg)s < /dev/null - """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), - 'bashVariables': bashVars, - 'componentType': componentType, - 'system': system, - 'component': component, - 'componentCfg': componentCfg}) - fd.close() + with io.open(runFile, 'w') as fd: + fd.write( + u"""#!/bin/bash + rcfile=%(bashrc)s + [ -e $rcfile ] && source $rcfile + # + exec 2>&1 + # + [ "%(componentType)s" = "agent" ] && renice 20 -p $$ + #%(bashVariables)s + # + exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py %(system)s/%(component)s %(componentCfg)s < /dev/null + """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), + 'bashVariables': bashVars, + 'componentType': componentType, + 'system': system, + 'component': component, + 'componentCfg': componentCfg}) os.chmod(runFile, self.gDefaultPerms) @@ -1930,8 +1926,8 @@ def installComponent(self, componentType, system, component, extensions, compone stopFile = os.path.join(runitCompDir, 'control', 't') # This is, e.g., /opt/dirac/control/WorkfloadManagementSystem/Matcher/ controlDir = self.runitDir.replace('runit', 'control') - with open(stopFile, 'w') as fd: - fd.write("""#!/bin/bash + with io.open(stopFile, 'w') as fd: + fd.write(u"""#!/bin/bash echo %(controlDir)s/%(system)s/%(component)s/stop_%(type)s touch %(controlDir)s/%(system)s/%(component)s/stop_%(type)s """ % {'controlDir': controlDir, @@ -2093,9 +2089,9 @@ def installPortal(self): try: self._createRunitLog(runitWebAppDir) runFile = os.path.join(runitWebAppDir, 'run') - with open(runFile, 'w') as fd: + with io.open(runFile, 'w') as fd: fd.write( - """#!/bin/bash + u"""#!/bin/bash rcfile=%(bashrc)s [ -e $rcfile ] && source $rcfile # @@ -2405,9 +2401,8 @@ def _createMySQLCMDLines(self, dbFile): cmdLines = [] - fd = open(dbFile) - dbLines = fd.readlines() - fd.close() + with io.open(dbFile, 'rt') as fd: + dbLines = fd.readlines() for line in dbLines: # Should we first source an SQL file (is this sql file an extension)? @@ -2415,9 +2410,8 @@ def _createMySQLCMDLines(self, dbFile): sourcedDBbFileName = line.split(' ')[1].replace('\n', '') gLogger.info("Found file to source: %s" % sourcedDBbFileName) sourcedDBbFile = os.path.join(rootPath, sourcedDBbFileName) - fdSourced = open(sourcedDBbFile) - dbLinesSourced = fdSourced.readlines() - fdSourced.close() + with io.open(sourcedDBbFile, 'rt') as fdSourced: + dbLinesSourced = fdSourced.readlines() for lineSourced in dbLinesSourced: if lineSourced.strip(): cmdLines.append(lineSourced.strip()) From d5175ac120632d6516f52234481267d05980bd2e Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 15 Jun 2020 14:51:00 +0200 Subject: [PATCH 41/44] fix sh --- FrameworkSystem/Client/ComponentInstaller.py | 48 ++++++++++---------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/FrameworkSystem/Client/ComponentInstaller.py b/FrameworkSystem/Client/ComponentInstaller.py index 6382d9ec459..609dba57537 100644 --- a/FrameworkSystem/Client/ComponentInstaller.py +++ b/FrameworkSystem/Client/ComponentInstaller.py @@ -1823,12 +1823,11 @@ def _createRunitLog(self, runitCompDir): with io.open(logRunFile, 'w') as fd: fd.write( u"""#!/bin/bash - # - rcfile=%(bashrc)s - [ -e $rcfile ] && source $rcfile - # - exec svlogd . +rcfile=%(bashrc)s +[[ -e $rcfile ]] && source ${rcfile} +# +exec svlogd . """ % {'bashrc': os.path.join(self.instancePath, 'bashrc')}) os.chmod(logRunFile, self.gDefaultPerms) @@ -1902,15 +1901,16 @@ def installComponent(self, componentType, system, component, extensions, compone with io.open(runFile, 'w') as fd: fd.write( u"""#!/bin/bash - rcfile=%(bashrc)s - [ -e $rcfile ] && source $rcfile - # - exec 2>&1 - # - [ "%(componentType)s" = "agent" ] && renice 20 -p $$ - #%(bashVariables)s - # - exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py %(system)s/%(component)s %(componentCfg)s < /dev/null + +rcfile=%(bashrc)s +[[ -e $rcfile ]] && source ${rcfile} +# +exec 2>&1 +# +[[ "%(componentType)s" = "agent" ]] && renice 20 -p $$ +#%(bashVariables)s +# +exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py %(system)s/%(component)s %(componentCfg)s < /dev/null """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), 'bashVariables': bashVars, 'componentType': componentType, @@ -1925,9 +1925,10 @@ def installComponent(self, componentType, system, component, extensions, compone # This is, e.g., /opt/dirac/runit/WorkfloadManagementSystem/Matcher/control/t stopFile = os.path.join(runitCompDir, 'control', 't') # This is, e.g., /opt/dirac/control/WorkfloadManagementSystem/Matcher/ - controlDir = self.runitDir.replace('runit', 'control') + controlDir = self.runitDir.replace('runit', 'control') with io.open(stopFile, 'w') as fd: fd.write(u"""#!/bin/bash + echo %(controlDir)s/%(system)s/%(component)s/stop_%(type)s touch %(controlDir)s/%(system)s/%(component)s/stop_%(type)s """ % {'controlDir': controlDir, @@ -2090,16 +2091,17 @@ def installPortal(self): self._createRunitLog(runitWebAppDir) runFile = os.path.join(runitWebAppDir, 'run') with io.open(runFile, 'w') as fd: - fd.write( + fd.write( u"""#!/bin/bash - rcfile=%(bashrc)s - [ -e $rcfile ] && source $rcfile - # - exec 2>&1 - # - exec python %(DIRAC)s/WebAppDIRAC/scripts/dirac-webapp-run.py -p < /dev/null + +rcfile=%(bashrc)s +[[ -e $rcfile ]] && source $rcfile +# +exec 2>&1 +# +exec python %(DIRAC)s/WebAppDIRAC/scripts/dirac-webapp-run.py -p < /dev/null """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), - 'DIRAC': self.linkedRootPath}) + 'DIRAC': self.linkedRootPath}) os.chmod(runFile, self.gDefaultPerms) except Exception: From 92e679136f27c768b7db36a85472207d02efc052 Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 29 Jun 2020 14:07:43 +0200 Subject: [PATCH 42/44] .keys -> list() --- WorkloadManagementSystem/Agent/SiteDirector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WorkloadManagementSystem/Agent/SiteDirector.py b/WorkloadManagementSystem/Agent/SiteDirector.py index 6546b2e26d6..10a31b0aeb6 100644 --- a/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/WorkloadManagementSystem/Agent/SiteDirector.py @@ -536,7 +536,7 @@ def submitPilots(self): totalWaitingPilots = 0 manyWaitingPilotsFlag = False if self.pilotWaitingFlag: - tqIDList = additionalInfo.keys() + tqIDList = list(additionalInfo) result = pilotAgentsDB.countPilots({'TaskQueueID': tqIDList, 'Status': WAITING_PILOT_STATUS}, None) @@ -659,7 +659,7 @@ def _ifAndWhereToSubmit(self): def monitorJobsQueuesPilots(self, matchingTQs): """ Just printout of jobs queues and pilots status in TQ """ - tqIDList = matchingTQs.keys() + tqIDList = list(matchingTQs) result = pilotAgentsDB.countPilots({'TaskQueueID': tqIDList, 'Status': WAITING_PILOT_STATUS}, None) From 2079340afbd457a23ca2cc6a4090b7cb5f1d2cba Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 29 Jun 2020 14:22:19 +0200 Subject: [PATCH 43/44] use super for Clients --- DataManagementSystem/Client/FTS3Client.py | 2 +- .../Client/S3GatewayClient.py | 2 +- .../Client/BundleDeliveryClient.py | 3 +- FrameworkSystem/Client/ComponentInstaller.py | 56 ++++++------- FrameworkSystem/Client/ProxyManagerClient.py | 82 +++++++++---------- .../Client/SystemAdministratorClient.py | 2 +- .../Service/ProxyManagerHandler.py | 52 ++++++------ ProductionSystem/Client/ProductionClient.py | 5 +- .../Client/StorageManagerClient.py | 2 +- .../Client/TransformationClient.py | 2 +- .../Agent/SiteDirector.py | 2 +- .../Client/PilotsLoggingClient.py | 2 +- 12 files changed, 107 insertions(+), 105 deletions(-) diff --git a/DataManagementSystem/Client/FTS3Client.py b/DataManagementSystem/Client/FTS3Client.py index 29f653e1016..79fdb5f4ba2 100644 --- a/DataManagementSystem/Client/FTS3Client.py +++ b/DataManagementSystem/Client/FTS3Client.py @@ -11,7 +11,7 @@ class FTS3Client(Client): def __init__(self, url=None, **kwargs): """ Constructor function. """ - Client.__init__(self, **kwargs) + super(FTS3Client, self).__init__(**kwargs) self.setServer('DataManagement/FTS3Manager') if url: self.setServer(url) diff --git a/DataManagementSystem/Client/S3GatewayClient.py b/DataManagementSystem/Client/S3GatewayClient.py index 04bd31396a3..263b77b49bb 100644 --- a/DataManagementSystem/Client/S3GatewayClient.py +++ b/DataManagementSystem/Client/S3GatewayClient.py @@ -11,7 +11,7 @@ class S3GatewayClient(Client): def __init__(self, url=None, **kwargs): """ Constructor function. """ - Client.__init__(self, **kwargs) + super(S3GatewayClient, self).__init__(**kwargs) self.setServer('DataManagement/S3Gateway') if url: self.setServer(url) diff --git a/FrameworkSystem/Client/BundleDeliveryClient.py b/FrameworkSystem/Client/BundleDeliveryClient.py index fd37907b5b5..5667199a351 100644 --- a/FrameworkSystem/Client/BundleDeliveryClient.py +++ b/FrameworkSystem/Client/BundleDeliveryClient.py @@ -21,7 +21,8 @@ class BundleDeliveryClient(Client): def __init__(self, transferClient=False, **kwargs): - Client.__init__(self, **kwargs) + super(BundleDeliveryClient, self).__init__(**kwargs) + self.setServer('Framework/BundleDelivery') self.transferClient = transferClient self.log = gLogger.getSubLogger("BundleDelivery") diff --git a/FrameworkSystem/Client/ComponentInstaller.py b/FrameworkSystem/Client/ComponentInstaller.py index 609dba57537..764bb0a4d72 100644 --- a/FrameworkSystem/Client/ComponentInstaller.py +++ b/FrameworkSystem/Client/ComponentInstaller.py @@ -1015,7 +1015,7 @@ def getSoftwareComponents(self, extensions): for agent in agentList: if os.path.splitext(agent)[1] == ".py": agentFile = os.path.join(agentDir, agent) - with io.open(agentFile, 'rt') as afile: + with io.open(agentFile, 'rt') as afile: body = afile.read() if body.find('AgentModule') != -1 or body.find('OptimizerModule') != -1: if system not in agents: @@ -1041,7 +1041,7 @@ def getSoftwareComponents(self, extensions): for executor in executorList: if os.path.splitext(executor)[1] == ".py": executorFile = os.path.join(executorDir, executor) - with io.open(executorFile, 'rt') as afile: + with io.open(executorFile, 'rt') as afile: body = afile.read() if body.find('OptimizerExecutor') != -1: if system not in executors: @@ -1092,8 +1092,8 @@ def getInstalledComponents(self): for component in components: try: runFile = os.path.join(systemDir, component, 'run') - with io.open(runFile, 'rt') as rFile: - body = rFile.read() + with io.open(runFile, 'rt') as rFile: + body = rFile.read() for cType in self.componentTypes: if body.find('dirac-%s' % (cType)) != -1: @@ -1126,8 +1126,8 @@ def getSetupComponents(self): for component in componentList: try: runFile = os.path.join(self.startDir, component, 'run') - with io.open(runFile, 'rt') as rfile: - body = rfile.read() + with io.open(runFile, 'rt') as rfile: + body = rfile.read() for cType in self.componentTypes: if body.find('dirac-%s' % (cType)) != -1: @@ -1462,8 +1462,8 @@ def getLogTail(self, system, component, length=100): if not os.path.exists(logFileName): retDict[compName] = 'No log file found' else: - with io.open(logFileName, 'rt') as logFile: - lines = [line.strip() for line in logFile.readlines()] + with io.open(logFileName, 'rt') as logFile: + lines = [line.strip() for line in logFile.readlines()] if len(lines) < length: retDict[compName] = '\n'.join(lines) @@ -1599,7 +1599,7 @@ def setupSite(self, scriptCfg, cfg=None): if not cmdFound: gLogger.notice('Starting runsvdir ...') - with io.open(os.devnull, 'w') as devnull: + with io.open(os.devnull, 'w') as devnull: subprocess.Popen(['nohup', 'runsvdir', self.startDir, 'log: DIRAC runsv'], stdout=devnull, stderr=devnull, universal_newlines=True) @@ -1815,14 +1815,14 @@ def _createRunitLog(self, runitCompDir): logConfigFile = os.path.join(logDir, 'config') with io.open(logConfigFile, 'w') as fd: fd.write( - u"""s10000000 + u"""s10000000 n20 """) logRunFile = os.path.join(logDir, 'run') with io.open(logRunFile, 'w') as fd: fd.write( - u"""#!/bin/bash + u"""#!/bin/bash rcfile=%(bashrc)s [[ -e $rcfile ]] && source ${rcfile} @@ -1893,14 +1893,14 @@ def installComponent(self, componentType, system, component, extensions, compone try: componentCfg = os.path.join(self.linkedRootPath, 'etc', '%s_%s.cfg' % (system, component)) if not os.path.exists(componentCfg): - io.open(componentCfg, 'w').close() + io.open(componentCfg, 'w').close() self._createRunitLog(runitCompDir) runFile = os.path.join(runitCompDir, 'run') with io.open(runFile, 'w') as fd: - fd.write( - u"""#!/bin/bash + fd.write( + u"""#!/bin/bash rcfile=%(bashrc)s [[ -e $rcfile ]] && source ${rcfile} @@ -1912,11 +1912,11 @@ def installComponent(self, componentType, system, component, extensions, compone # exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py %(system)s/%(component)s %(componentCfg)s < /dev/null """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), - 'bashVariables': bashVars, - 'componentType': componentType, - 'system': system, - 'component': component, - 'componentCfg': componentCfg}) + 'bashVariables': bashVars, + 'componentType': componentType, + 'system': system, + 'component': component, + 'componentCfg': componentCfg}) os.chmod(runFile, self.gDefaultPerms) @@ -1925,9 +1925,9 @@ def installComponent(self, componentType, system, component, extensions, compone # This is, e.g., /opt/dirac/runit/WorkfloadManagementSystem/Matcher/control/t stopFile = os.path.join(runitCompDir, 'control', 't') # This is, e.g., /opt/dirac/control/WorkfloadManagementSystem/Matcher/ - controlDir = self.runitDir.replace('runit', 'control') - with io.open(stopFile, 'w') as fd: - fd.write(u"""#!/bin/bash + controlDir = self.runitDir.replace('runit', 'control') + with io.open(stopFile, 'w') as fd: + fd.write(u"""#!/bin/bash echo %(controlDir)s/%(system)s/%(component)s/stop_%(type)s touch %(controlDir)s/%(system)s/%(component)s/stop_%(type)s @@ -2090,9 +2090,9 @@ def installPortal(self): try: self._createRunitLog(runitWebAppDir) runFile = os.path.join(runitWebAppDir, 'run') - with io.open(runFile, 'w') as fd: - fd.write( - u"""#!/bin/bash + with io.open(runFile, 'w') as fd: + fd.write( + u"""#!/bin/bash rcfile=%(bashrc)s [[ -e $rcfile ]] && source $rcfile @@ -2101,7 +2101,7 @@ def installPortal(self): # exec python %(DIRAC)s/WebAppDIRAC/scripts/dirac-webapp-run.py -p < /dev/null """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), - 'DIRAC': self.linkedRootPath}) + 'DIRAC': self.linkedRootPath}) os.chmod(runFile, self.gDefaultPerms) except Exception: @@ -2412,8 +2412,8 @@ def _createMySQLCMDLines(self, dbFile): sourcedDBbFileName = line.split(' ')[1].replace('\n', '') gLogger.info("Found file to source: %s" % sourcedDBbFileName) sourcedDBbFile = os.path.join(rootPath, sourcedDBbFileName) - with io.open(sourcedDBbFile, 'rt') as fdSourced: - dbLinesSourced = fdSourced.readlines() + with io.open(sourcedDBbFile, 'rt') as fdSourced: + dbLinesSourced = fdSourced.readlines() for lineSourced in dbLinesSourced: if lineSourced.strip(): cmdLines.append(lineSourced.strip()) diff --git a/FrameworkSystem/Client/ProxyManagerClient.py b/FrameworkSystem/Client/ProxyManagerClient.py index d1df5a017ba..67a7ddbf71d 100755 --- a/FrameworkSystem/Client/ProxyManagerClient.py +++ b/FrameworkSystem/Client/ProxyManagerClient.py @@ -35,7 +35,7 @@ def __init__(self): def __deleteTemporalFile(self, filename): """ Delete temporal file - :param str filename: path to file + :param str filename: path to file """ try: os.remove(filename) @@ -89,8 +89,8 @@ def userHasProxy(self, userDN, userGroup, validSeconds=0): """ Check if a user(DN-group) has a proxy in the proxy management Updates internal cache if needed to minimize queries to the service - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param int validSeconds: proxy valid time in a seconds :return: S_OK()/S_ERROR() @@ -110,8 +110,8 @@ def getUserPersistence(self, userDN, userGroup, validSeconds=0): """ Check if a user(DN-group) has a proxy in the proxy management Updates internal cache if needed to minimize queries to the service - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param int validSeconds: proxy valid time in a seconds :return: S_OK()/S_ERROR() @@ -134,8 +134,8 @@ def getUserPersistence(self, userDN, userGroup, validSeconds=0): def setPersistency(self, userDN, userGroup, persistent): """ Set the persistency for user/group - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean persistent: presistent flag :return: S_OK()/S_ERROR() @@ -219,13 +219,13 @@ def downloadProxy(self, userDN, userGroup, limited=False, requiredTimeLeft=1200, cacheTime=14400, proxyToConnect=None, token=None): """ Get a proxy Chain from the proxy management - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds :param X509Chain proxyToConnect: proxy as a chain - :param str token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -257,14 +257,14 @@ def downloadProxyToFile(self, userDN, userGroup, limited=False, requiredTimeLeft cacheTime=14400, filePath=None, proxyToConnect=None, token=None): """ Get a proxy Chain from the proxy management and write it to file - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds - :param str filePath: path to save proxy + :param str filePath: path to save proxy :param X509Chain proxyToConnect: proxy as a chain - :param str token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -284,14 +284,14 @@ def downloadVOMSProxy(self, userDN, userGroup, limited=False, requiredTimeLeft=1 proxyToConnect=None, token=None): """ Download a proxy if needed and transform it into a VOMS one - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds - :param str requiredVOMSAttribute: VOMS attr to add to the proxy + :param str requiredVOMSAttribute: VOMS attr to add to the proxy :param X509Chain proxyToConnect: proxy as a chain - :param str token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -325,15 +325,15 @@ def downloadVOMSProxyToFile(self, userDN, userGroup, limited=False, requiredTime proxyToConnect=None, token=None): """ Download a proxy if needed, transform it into a VOMS one and write it to file - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds - :param str requiredVOMSAttribute: VOMS attr to add to the proxy - :param str filePath: path to save proxy + :param str requiredVOMSAttribute: VOMS attr to add to the proxy + :param str filePath: path to save proxy :param X509Chain proxyToConnect: proxy as a chain - :param str token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -351,9 +351,9 @@ def downloadVOMSProxyToFile(self, userDN, userGroup, limited=False, requiredTime def getPilotProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft=43200, proxyToConnect=None): """ Download a pilot proxy with VOMS extensions depending on the group - :param str userDN: user DN - :param str userGroup: user group - :param int requiredTimeLeft: required proxy live time in seconds + :param str userDN: user DN + :param str userGroup: user group + :param int requiredTimeLeft: required proxy live time in seconds :param X509Chain proxyToConnect: proxy as a chain :return: S_OK(X509Chain)/S_ERROR() @@ -371,8 +371,8 @@ def getPilotProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft=43200, def getPilotProxyFromVOMSGroup(self, userDN, vomsAttr, requiredTimeLeft=43200, proxyToConnect=None): """ Download a pilot proxy with VOMS extensions depending on the group - :param str userDN: user DN - :param str vomsAttr: VOMS attribute + :param str userDN: user DN + :param str vomsAttr: VOMS attribute :param int requiredTimeLeft: required proxy live time in a seconds :param X509Chain proxyToConnect: proxy as a chain @@ -395,10 +395,10 @@ def getPilotProxyFromVOMSGroup(self, userDN, vomsAttr, requiredTimeLeft=43200, p def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, token=None, proxyToConnect=None): """ Download a payload proxy with VOMS extensions depending on the group - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param int requiredTimeLeft: required proxy live time in a seconds - :param str token: valid token to get a proxy + :param str token: valid token to get a proxy :param X509Chain proxyToConnect: proxy as a chain :return: S_OK(X509Chain)/S_ERROR() @@ -417,9 +417,9 @@ def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, tok def getPayloadProxyFromVOMSGroup(self, userDN, vomsAttr, token, requiredTimeLeft, proxyToConnect=None): """ Download a payload proxy with VOMS extensions depending on the VOMS attr - :param str userDN: user DN - :param str vomsAttr: VOMS attribute - :param str token: valid token to get a proxy + :param str userDN: user DN + :param str vomsAttr: VOMS attribute + :param str token: valid token to get a proxy :param int requiredTimeLeft: required proxy live time in a seconds :param X509Chain proxyToConnect: proxy as a chain @@ -442,10 +442,10 @@ def dumpProxyToFile(self, chain, destinationFile=None, requiredTimeLeft=600): """ Dump a proxy to a file. It's cached so multiple calls won't generate extra files :param X509Chain chain: proxy as a chain - :param str destinationFile: path to store proxy + :param str destinationFile: path to store proxy :param int requiredTimeLeft: required proxy live time in a seconds - :return: S_OK(str)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ result = chain.hash() if not result['OK']: @@ -477,8 +477,8 @@ def requestToken(self, requesterDN, requesterGroup, numUses=1): """ Request a number of tokens. usesList must be a list of integers and each integer is the number of uses a token must have - :param str requesterDN: user DN - :param str requesterGroup: user group + :param str requesterDN: user DN + :param str requesterGroup: user group :param int numUses: number of uses :return: S_OK(tuple)/S_ERROR() -- tuple contain token, number uses @@ -573,15 +573,15 @@ def getVOMSAttributes(self, chain): :param X509Chain chain: proxy as a chain - :return: S_OK(str)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ return VOMS().getVOMSAttributes(chain) def getUploadedProxyLifeTime(self, DN, group): """ Get the remaining seconds for an uploaded proxy - :param str DN: user DN - :param str group: group + :param str DN: user DN + :param str group: group :return: S_OK(int)/S_ERROR() """ diff --git a/FrameworkSystem/Client/SystemAdministratorClient.py b/FrameworkSystem/Client/SystemAdministratorClient.py index 784d7b7b824..0e69080403a 100644 --- a/FrameworkSystem/Client/SystemAdministratorClient.py +++ b/FrameworkSystem/Client/SystemAdministratorClient.py @@ -16,7 +16,7 @@ class SystemAdministratorClient(Client): def __init__(self, host, port=None, **kwargs): """ Constructor function. Takes a mandatory host parameter """ - Client.__init__(self, **kwargs) + super(SystemAdministratorClient, self).__init__(**kwargs) if not port: port = SYSADMIN_PORT self.setServer('dips://%s:%s/Framework/SystemAdministrator' % (host, port)) diff --git a/FrameworkSystem/Service/ProxyManagerHandler.py b/FrameworkSystem/Service/ProxyManagerHandler.py index 2aabb0d17a7..eaee0640179 100644 --- a/FrameworkSystem/Service/ProxyManagerHandler.py +++ b/FrameworkSystem/Service/ProxyManagerHandler.py @@ -91,7 +91,7 @@ def export_getUserProxiesInfo(self): def export_requestDelegationUpload(self, requestedUploadTime, diracGroup=None): """ Request a delegation. Send a delegation request to client - :param int requestedUploadTime: requested live time + :param int requestedUploadTime: requested live time :return: S_OK(dict)/S_ERROR() -- dict contain id and proxy as string of the request """ @@ -126,7 +126,7 @@ def export_completeDelegationUpload(self, requestId, pemChain): """ Upload result of delegation :param int,long requestId: identity number - :param str pemChain: certificate as string + :param str pemChain: certificate as string :return: S_OK(dict)/S_ERROR() -- dict contain proxies """ @@ -157,8 +157,8 @@ def export_getRegisteredUsers(self, validSecondsRequired=0): def __checkProperties(self, requestedUserDN, requestedUserGroup): """ Check the properties and return if they can only download limited proxies if authorized - :param str requestedUserDN: user DN - :param str requestedUserGroup: DIRAC group + :param str requestedUserDN: user DN + :param str requestedUserGroup: DIRAC group :return: S_OK(boolean)/S_ERROR() """ @@ -202,13 +202,13 @@ def export_getProxy(self, userDN, userGroup, requestPem, requiredLifetime): def __getProxy(self, userDN, userGroup, requestPem, requiredLifetime, forceLimited): """ Internal to get a proxy - :param str userDN: user DN - :param str userGroup: DIRAC group - :param str requestPem: dump of request certificate - :param int requiredLifetime: requested live time of proxy + :param str userDN: user DN + :param str userGroup: DIRAC group + :param str requestPem: dump of request certificate + :param int requiredLifetime: requested live time of proxy :param boolean forceLimited: limited proxy - :return: S_OK(str)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ retVal = self.__proxyDB.getProxy(userDN, userGroup, requiredLifeTime=requiredLifetime) if not retVal['OK']: @@ -224,8 +224,8 @@ def __getProxy(self, userDN, userGroup, requestPem, requiredLifetime, forceLimit return S_OK(retVal['Value']) types_getVOMSProxy = [six.string_types, six.string_types, - six.string_types, six.integer_types, - [six.string_types, type(None), bool]] + six.string_types, six.integer_types, + [six.string_types, type(None), bool]] def export_getVOMSProxy(self, userDN, userGroup, requestPem, requiredLifetime, vomsAttribute=None): """ Get a proxy for a userDN/userGroup @@ -259,16 +259,16 @@ def __getVOMSProxy(self, userDN, userGroup, requestPem, requiredLifetime, vomsAt # If possible we return a proxy 1.5 longer than requested requiredLifetime = int(min(secsLeft, requiredLifetime * self.__maxExtraLifeFactor)) return chain.generateChainFromRequestString(requestPem, - lifetime=requiredLifetime, - requireLimited=forceLimited) + lifetime=requiredLifetime, + requireLimited=forceLimited) types_setPersistency = [six.string_types, six.string_types, bool] def export_setPersistency(self, userDN, userGroup, persistentFlag): """ Set the persistency for a given dn/group - :param str userDN: user DN - :param str userGroup: DIRAC group + :param str userDN: user DN + :param str userGroup: DIRAC group :param boolean persistentFlag: if proxy persistent :return: S_OK()/S_ERROR() @@ -309,8 +309,8 @@ def export_deleteProxyBundle(self, idList): def export_deleteProxy(self, userDN, userGroup): """ Delete a proxy from the DB - :param str userDN: user DN - :param str userGroup: DIRAC group + :param str userDN: user DN + :param str userGroup: DIRAC group :return: S_OK()/S_ERROR() """ @@ -331,8 +331,8 @@ def export_getContents(self, selDict, sortDict, start, limit): :param dict selDict: selection fields :param list,tuple sortDict: sorting fields - :param int start: search limit start - :param int start: search limit amount + :param int start: search limit start + :param int start: search limit amount :return: S_OK(dict)/S_ERROR() -- dict contain fields, record list, total records """ @@ -348,8 +348,8 @@ def export_getLogContents(self, selDict, sortDict, start, limit): :param dict selDict: selection fields :param list,tuple sortDict: search filter - :param int start: search limit start - :param int start: search limit amount + :param int start: search limit start + :param int start: search limit amount :return: S_OK(dict)/S_ERROR() -- dict contain fields, record list, total records """ @@ -360,9 +360,9 @@ def export_getLogContents(self, selDict, sortDict, start, limit): def export_generateToken(self, requesterDN, requesterGroup, tokenUses): """ Generate tokens for proxy retrieval - :param str requesterDN: user DN - :param str requesterGroup: DIRAC group - :param int tokenUses: number of uses + :param str requesterDN: user DN + :param str requesterGroup: DIRAC group + :param int tokenUses: number of uses :return: S_OK(tuple)/S_ERROR() -- tuple contain token, number uses """ @@ -400,8 +400,8 @@ def export_getProxyWithToken(self, userDN, userGroup, requestPem, requiredLifeti return self.__getProxy(userDN, userGroup, requestPem, requiredLifetime, True) types_getVOMSProxyWithToken = [six.string_types, six.string_types, - six.string_types, six.integer_types, - [six.string_types, type(None)]] + six.string_types, six.integer_types, + [six.string_types, type(None)]] def export_getVOMSProxyWithToken(self, userDN, userGroup, requestPem, requiredLifetime, token, vomsAttribute=None): """ Get a proxy for a userDN/userGroup diff --git a/ProductionSystem/Client/ProductionClient.py b/ProductionSystem/Client/ProductionClient.py index b77bc1ca9d1..09404335569 100644 --- a/ProductionSystem/Client/ProductionClient.py +++ b/ProductionSystem/Client/ProductionClient.py @@ -3,10 +3,11 @@ __RCSID__ = "$Id$" from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Core.Base.Client import Client +from DIRAC.Core.Base.Client import Client, createClient from DIRAC.ProductionSystem.Utilities.StateMachine import ProductionsStateMachine +@createClient('Framework/BundleDelivery') class ProductionClient(Client): """ Exposes the functionality available in the ProductionSystem/ProductionManagerHandler @@ -16,7 +17,7 @@ def __init__(self, **kwargs): """ Simple constructor """ - Client.__init__(self, **kwargs) + super(ProductionClient, self).__init__(**kwargs) self.setServer('Production/ProductionManager') self.prodDescription = {} self.stepCounter = 1 diff --git a/StorageManagementSystem/Client/StorageManagerClient.py b/StorageManagementSystem/Client/StorageManagerClient.py index d40f6060b53..fccbc359a23 100644 --- a/StorageManagementSystem/Client/StorageManagerClient.py +++ b/StorageManagementSystem/Client/StorageManagerClient.py @@ -223,5 +223,5 @@ class StorageManagerClient(Client): """ def __init__(self, **kwargs): - Client.__init__(self, **kwargs) + super(StorageManagerClient, self).__init__(**kwargs) self.setServer('StorageManagement/StorageManager') diff --git a/TransformationSystem/Client/TransformationClient.py b/TransformationSystem/Client/TransformationClient.py index 012ab9345c9..1489717ef6d 100644 --- a/TransformationSystem/Client/TransformationClient.py +++ b/TransformationSystem/Client/TransformationClient.py @@ -61,7 +61,7 @@ def __init__(self, **kwargs): """ Simple constructor """ - Client.__init__(self, **kwargs) + super(TransformationClient, self).__init__(**kwargs) opsH = Operations() self.maxResetCounter = opsH.getValue('Transformations/FilesMaxResetCounter', 10) diff --git a/WorkloadManagementSystem/Agent/SiteDirector.py b/WorkloadManagementSystem/Agent/SiteDirector.py index 10a31b0aeb6..922b2cfbbf8 100644 --- a/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/WorkloadManagementSystem/Agent/SiteDirector.py @@ -536,7 +536,7 @@ def submitPilots(self): totalWaitingPilots = 0 manyWaitingPilotsFlag = False if self.pilotWaitingFlag: - tqIDList = list(additionalInfo) + tqIDList = list(additionalInfo) result = pilotAgentsDB.countPilots({'TaskQueueID': tqIDList, 'Status': WAITING_PILOT_STATUS}, None) diff --git a/WorkloadManagementSystem/Client/PilotsLoggingClient.py b/WorkloadManagementSystem/Client/PilotsLoggingClient.py index 87eb9ca17ca..2c3c03a494d 100644 --- a/WorkloadManagementSystem/Client/PilotsLoggingClient.py +++ b/WorkloadManagementSystem/Client/PilotsLoggingClient.py @@ -12,7 +12,7 @@ class PilotsLoggingClient(Client): """ def __init__(self, **kwargs): - Client.__init__(self, **kwargs) + super(PilotsLoggingClient, self).__init__(**kwargs) self.setServer('WorkloadManagement/PilotsLogging') def addPilotsLogging(self, pilotUUID, timestamp, source, phase, status, messageContent): From da6eb9cb104140596df11746c0c517c80afa395e Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Mon, 29 Jun 2020 16:04:24 +0200 Subject: [PATCH 44/44] v7r1p6 notes and tags --- __init__.py | 2 +- release.notes | 15 +++++++++++++++ setup.py | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/__init__.py b/__init__.py index 312d4bef72f..efb0160832a 100755 --- a/__init__.py +++ b/__init__.py @@ -92,7 +92,7 @@ majorVersion = 7 minorVersion = 1 -patchLevel = 5 +patchLevel = 6 preVersion = 0 version = "v%sr%s" % (majorVersion, minorVersion) diff --git a/release.notes b/release.notes index 6564bb704da..a53f64286c6 100644 --- a/release.notes +++ b/release.notes @@ -1,3 +1,18 @@ +[v7r1p6] + +*Framework +CHANGE: (#4643) BundleDeliveryClient: inheriting from Client + +*SMS +FIX: (#4645) StorageManagementDB - drop the tables in the correct order to avoid foreign key errors + +*TS +NEW: (#4641) TransformationCleaningAgent will (re)clean very old transformations that are still in the system + +*tests +FIX: (#4645) dropDBs uses real array; +FIX: (#4645) remove readonly variables + [v7r1p5] *Framework diff --git a/setup.py b/setup.py index 04c89a4eca8..5a0f96c84dd 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name="DIRAC", - version="7.1.5", + version="7.1.6", url="https://github.com/DIRACGRID/DIRAC", license="GPLv3", package_dir=package_dir,