FIX: _checkFloat checks INTEGERS, not datetimes; pyLint compliant corrections

ResourceStatusSystem/Agent/CacheCleanerAgent.py

@@ -1,18 +1,22 @@
-################################################################################
 # $HeadURL$
-################################################################################
-__RCSID__  = "$Id$"
-AGENT_NAME = 'ResourceStatus/CleanerAgent'
+''' CacheCleanerAgent
+
+  This agent cleans the history tables, and the cache ones if entries older
+  than a certan period.
 
-from datetime                                             import datetime,timedelta
+'''
 
-from DIRAC                                                import S_OK, S_ERROR
-from DIRAC.Core.Base.AgentModule                          import AgentModule
+from datetime import datetime, timedelta
 
-from DIRAC.ResourceStatusSystem                           import ValidRes  
+from DIRAC                                                      import S_OK, S_ERROR
+from DIRAC.Core.Base.AgentModule                                import AgentModule
+from DIRAC.ResourceStatusSystem                                 import ValidRes  
 from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient     import ResourceStatusClient
 from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient
 
+__RCSID__  = '$Id: $'
+AGENT_NAME = 'ResourceStatus/CleanerAgent'
+
 class CacheCleanerAgent( AgentModule ):
   '''
   The CacheCleanerAgent tidies up the ResourceStatusDB, namely:
@@ -30,13 +34,19 @@ class CacheCleanerAgent( AgentModule ):
   If you want to know more about the CacheCleanerAgent, scroll down to the end of the 
   file.   
   '''
+
+  # Too many public methods
+  # pylint: disable-msg=R0904
 
   def initialize( self ):
 
+    # Attribute defined outside __init__  
+    # pylint: disable-msg=W0201
+
     try:
 
-      self.rsClient         = ResourceStatusClient()
-      self.rmClient         = ResourceManagementClient()  
+      self.rsClient      = ResourceStatusClient()
+      self.rmClient      = ResourceManagementClient()  
       self.historyTables = [ '%sHistory' % x for x in ValidRes ]
 
       return S_OK()
@@ -57,12 +67,12 @@ def execute( self ):
       now          = datetime.utcnow().replace( microsecond = 0, second = 0 )
       sixMonthsAgo = now - timedelta( days = 180 )
 
-      for g in ValidRes:
+      for granularity in ValidRes:
         #deleter = getattr( self.rsClient, 'delete%sHistory' % g )
 
         kwargs = { 'meta' : { 'minor' : { 'DateEnd' : sixMonthsAgo } } }
-        self.log.info( 'Deleting %sHistory older than %s' % ( g, sixMonthsAgo ) )
-        res = self.rsClient.deleteElementHistory( g, **kwargs )
+        self.log.info( 'Deleting %sHistory older than %s' % ( granularity, sixMonthsAgo ) )
+        res = self.rsClient.deleteElementHistory( granularity, **kwargs )
         if not res[ 'OK' ]:
           self.log.error( res[ 'Message' ] )            
 

ResourceStatusSystem/Agent/CacheFeederAgent.py

@@ -1,41 +1,50 @@
-################################################################################
 # $HeadURL:  $
-################################################################################
-__RCSID__  = "$Id:  $"
-AGENT_NAME = 'ResourceStatus/CacheFeederAgent'
+''' CacheFeederAgent
+
+  This agent feeds the Cache tables with the outputs of the cache commands.
+
+'''
 
 from datetime import datetime
 
 from DIRAC                                                      import S_OK, S_ERROR
 from DIRAC.Core.Base.AgentModule                                import AgentModule
-
 from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient
 from DIRAC.ResourceStatusSystem.Command.CommandCaller           import CommandCaller
 from DIRAC.ResourceStatusSystem.Command.ClientsInvoker          import ClientsInvoker
 from DIRAC.ResourceStatusSystem.Command.knownAPIs               import initAPIs
 
+__RCSID__  = '$Id: $'
+AGENT_NAME = 'ResourceStatus/CacheFeederAgent'
+
 class CacheFeederAgent( AgentModule ):
   '''
   The CacheFeederAgent feeds the cache tables for the client and the accounting.
   It runs periodically a set of commands, and stores it's results on the
   tables.
   '''
 
+  # Too many public methods
+  # pylint: disable-msg=R0904  
+
   def initialize( self ):
 
+    # Attribute defined outside __init__ 
+    # pylint: disable-msg=W0201
+
     try:
 
       self.rmClient       = ResourceManagementClient()
       self.clientsInvoker = ClientsInvoker()
 
-      commandsList_ClientsCache = [
+      commandsListClientsCache = [
         ( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'     ),
         ( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ),
         ( 'ClientsCache_Command', 'DTEverySites_Command'              ),
         ( 'ClientsCache_Command', 'DTEveryResources_Command'          )
         ]
 
-      commandsList_AccountingCache =  [
+      commandsListAccountingCache =  [
         ( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command',     ( 2, ),    'Always' ),
         ( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command',   ( 2, ),    'Always' ),
         ( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ),   'Hourly' ),
@@ -51,33 +60,33 @@ def initialize( self ):
         ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command',         ( 8760, ), 'Daily'  ),
         ]
 
-      self.commandObjectsList_ClientsCache    = []
-      self.commandObjectsList_AccountingCache = []
+      self.commandObjectsListClientsCache    = []
+      self.commandObjectsListAccountingCache = []
 
       cc = CommandCaller()
 
       # We know beforehand which APIs are we going to need, so we initialize them
       # first, making everything faster.
-      APIs = [ 'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator',
-               'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient' ]
-      APIs = initAPIs( APIs, {} )
+      knownAPIs = [ 'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator',
+                    'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient' ]
+      knownAPIs = initAPIs( knownAPIs, {} )
 
-      for command in commandsList_ClientsCache:
+      for command in commandsListClientsCache:
 
         cObj = cc.setCommandObject( command )
-        for apiName, apiInstance in APIs.items():
+        for apiName, apiInstance in knownAPIs.items():
           cc.setAPI( cObj, apiName, apiInstance )
 
-        self.commandObjectsList_ClientsCache.append( ( command, cObj ) )
+        self.commandObjectsListClientsCache.append( ( command, cObj ) )
 
-      for command in commandsList_AccountingCache:
+      for command in commandsListAccountingCache:
 
         cObj = cc.setCommandObject( command )
-        for apiName, apiInstance in APIs.items():
+        for apiName, apiInstance in knownAPIs.items():
           cc.setAPI( cObj, apiName, apiInstance )
         cArgs = command[ 2 ]
 
-        self.commandObjectsList_AccountingCache.append( ( command, cObj, cArgs ) )
+        self.commandObjectsListAccountingCache.append( ( command, cObj, cArgs ) )
 
       return S_OK()
 
@@ -92,10 +101,10 @@ def execute( self ):
 
     try:
 
-      for co in self.commandObjectsList_ClientsCache:
+      for co in self.commandObjectsListClientsCache:
 
         commandName = co[0][1].split( '_' )[0]
-        gLogger.info( 'Executed %s' % commandName )
+        self.log.info( 'Executed %s' % commandName )
         try:
           self.clientsInvoker.setCommand( co[1] )
           res = self.clientsInvoker.doCommand()['Result']
@@ -140,7 +149,7 @@ def execute( self ):
 
       now = datetime.utcnow().replace( microsecond = 0 )
 
-      for co in self.commandObjectsList_AccountingCache:
+      for co in self.commandObjectsListAccountingCache:
 
         if co[0][3] == 'Hourly':
           if now.minute >= 10:

ResourceStatusSystem/Agent/RSInspectorAgent.py

@@ -1,8 +1,9 @@
-################################################################################
 # $HeadURL:  $
-################################################################################
-__RCSID__  = "$Id:  $"
-AGENT_NAME = 'ResourceStatus/RSInspectorAgent'
+''' RSInspectorAgent
+
+  This agent inspect Resources, and evaluates policies that apply.
+
+'''
 
 import Queue, time
 
@@ -16,6 +17,9 @@
 from DIRAC.ResourceStatusSystem.Utilities.Utils             import where
 from DIRAC.ResourceStatusSystem.Utilities                   import CS
 
+__RCSID__  = '$Id: $'
+AGENT_NAME = 'ResourceStatus/RSInspectorAgent'
+
 class RSInspectorAgent( AgentModule ):
   """
     The RSInspector agent ( ResourceInspectorAgent ) is one of the four
@@ -28,13 +32,19 @@ class RSInspectorAgent( AgentModule ):
     end of the file.
   """
 
+  # Too many public methods
+  # pylint: disable-msg=R0904
+
   def initialize( self ):
 
+    # Attribute defined outside __init__ 
+    # pylint: disable-msg=W0201
+
     try:
       self.rsClient             = ResourceStatusClient()
-      self.ResourcesFreqs       = CS.getTypedDictRootedAt( 'CheckingFreqs/ResourcesFreqs' )
-      self.ResourcesToBeChecked = Queue.Queue()
-      self.ResourceNamesInCheck = []
+      self.resourcesFreqs       = CS.getTypedDictRootedAt( 'CheckingFreqs/ResourcesFreqs' )
+      self.resourcesToBeChecked = Queue.Queue()
+      self.resourceNamesInCheck = []
 
       self.maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 )
       self.threadPool         = ThreadPool( self.maxNumberOfThreads,
@@ -53,9 +63,6 @@ def initialize( self ):
       self.log.exception( errorStr )
       return S_ERROR( errorStr )
 
-################################################################################
-################################################################################
-
   def execute( self ):
 
     try:
@@ -66,7 +73,7 @@ def execute( self ):
                                     'TokenOwner' ]
       kwargs[ 'tokenOwner' ]    = 'RS_SVC'
 
-      resQuery = self.rsClient.getStuffToCheck( 'Resource', self.ResourcesFreqs, **kwargs )
+      resQuery = self.rsClient.getStuffToCheck( 'Resource', self.resourcesFreqs, **kwargs )
       if not resQuery[ 'OK' ]:
         self.log.error( resQuery[ 'Message' ] )
         return resQuery
@@ -76,40 +83,43 @@ def execute( self ):
 
       for resourceTuple in resQuery:
 
-        if ( resourceTuple[ 0 ], resourceTuple[ 1 ] ) in self.ResourceNamesInCheck:
+        if ( resourceTuple[ 0 ], resourceTuple[ 1 ] ) in self.resourceNamesInCheck:
           self.log.info( '%s(%s) discarded, already on the queue' % ( resourceTuple[ 0 ], resourceTuple[ 1 ] ) )
           continue
 
         resourceL = [ 'Resource' ] + resourceTuple
 
-        self.ResourceNamesInCheck.insert( 0, ( resourceTuple[ 0 ], resourceTuple[ 1 ] ) )
-        self.ResourcesToBeChecked.put( resourceL )
+        self.resourceNamesInCheck.insert( 0, ( resourceTuple[ 0 ], resourceTuple[ 1 ] ) )
+        self.resourcesToBeChecked.put( resourceL )
 
       return S_OK()
 
     except Exception, x:
       errorStr = where( self, self.execute )
-      self.log.exception( errorStr,lException=x )
+      self.log.exception( errorStr, lException = x )
       return S_ERROR( errorStr )
 
-################################################################################
-################################################################################
-
   def finalize( self ):
-    if self.ResourceNamesInCheck:
+    '''
+      Method executed at the end of the last cycle. It waits until the queue
+      is empty.
+    '''   
+    if self.resourceNamesInCheck:
       _msg = "Wait for queue to get empty before terminating the agent (%d tasks)"
-      _msg = _msg % len( self.ResourceNamesInCheck )
+      _msg = _msg % len( self.resourceNamesInCheck )
       self.log.info( _msg )
-      while self.ResourceNamesInCheck:
+      while self.resourceNamesInCheck:
         time.sleep( 2 )
       self.log.info( "Queue is empty, terminating the agent..." )
     return S_OK()
 
-################################################################################
 ################################################################################
 
   def _executeCheck( self, _arg ):
-
+    '''
+      Method executed by the threads in the pool. Picks one element from the
+      common queue, and enforces policies on that element.
+    '''
     # Init the APIs beforehand, and reuse them.
     __APIs__ = [ 'ResourceStatusClient', 'ResourceManagementClient' ]
     clients = knownAPIs.initAPIs( __APIs__, {} )
@@ -118,7 +128,7 @@ def _executeCheck( self, _arg ):
 
     while True:
 
-      toBeChecked  = self.ResourcesToBeChecked.get()
+      toBeChecked  = self.resourcesToBeChecked.get()
 
       pepDict = { 'granularity'  : toBeChecked[ 0 ],
                   'name'         : toBeChecked[ 1 ],
@@ -131,24 +141,24 @@ def _executeCheck( self, _arg ):
 
       try:
 
-        gLogger.info( "Checking Resource %s, with type/status: %s/%s" % \
+        self.log.info( "Checking Resource %s, with type/status: %s/%s" % \
                       ( pepDict['name'], pepDict['statusType'], pepDict['status'] ) )
 
         pepRes =  pep.enforce( **pepDict )
         if pepRes.has_key( 'PolicyCombinedResult' ) and pepRes[ 'PolicyCombinedResult' ].has_key( 'Status' ):
           pepStatus = pepRes[ 'PolicyCombinedResult' ][ 'Status' ]
           if pepStatus != pepDict[ 'status' ]:
-            gLogger.info( 'Updated Site %s (%s) from %s to %s' %
+            self.log.info( 'Updated Site %s (%s) from %s to %s' %
                           ( pepDict['name'], pepDict['statusType'], pepDict['status'], pepStatus ))
 
         # remove from InCheck list
-        self.ResourceNamesInCheck.remove( ( pepDict[ 'name' ], pepDict[ 'statusType' ] ) )
+        self.resourceNamesInCheck.remove( ( pepDict[ 'name' ], pepDict[ 'statusType' ] ) )
 
       except Exception:
         self.log.exception( "RSInspector._executeCheck Checking Resource %s, with type/status: %s/%s" % \
                       ( pepDict['name'], pepDict['statusType'], pepDict['status'] ) )
         try:
-          self.ResourceNamesInCheck.remove( ( pepDict[ 'name' ], pepDict[ 'statusType' ] ) )
+          self.resourceNamesInCheck.remove( ( pepDict[ 'name' ], pepDict[ 'statusType' ] ) )
         except IndexError:
           pass