[v7r0] Minor fixes + doc on Bdii

ConfigurationSystem/Agent/Bdii2CSAgent.py

@@ -3,6 +3,14 @@
     present in the CS and notifies the administrators.
     For the CEs and SEs already present in the CS, the agent is updating
     if necessary settings which were changed in the BDII recently
+
+The following options can be set for the Bdii2CSAgent.
+
+.. literalinclude:: ../ConfigTemplate.cfg
+  :start-after: ##BEGIN Bdii2CSAgent
+  :end-before: ##END
+  :dedent: 2
+  :caption: Bdii2CSAgent options
 """
 
 __RCSID__ = "$Id$"

ConfigurationSystem/Agent/GOCDB2CSAgent.py

@@ -2,6 +2,15 @@
 This module provides GOCDB2CSAgent code.
 
 The agent is used to synchronize information between GOCDB and DIRAC configuration System (CS)
+Right now it only adds perfosonar endpoints
+
+The following options can be set for the GOCDB2CSAgent.
+
+.. literalinclude:: ../ConfigTemplate.cfg
+  :start-after: ##BEGIN GOCDB2CSAgent
+  :end-before: ##END
+  :dedent: 2
+  :caption: GOCDB2CSAgent options
 """
 
 __RCSID__ = "$Id$"

ConfigurationSystem/ConfigTemplate.cfg

@@ -16,24 +16,33 @@ Services
 }
 Agents
 {
+  ##BEGIN Bdii2CSAgent
   Bdii2CSAgent
   {
+    #Time between cycles in seconds
+    PollingTime = 14400
     BannedCEs =
     BannedSEs =
     SelectedSites =
     ProcessCEs = yes
     ProcessSEs = no
+    # Mail Notifications options
     MailTo =
     MailFrom =
     VirtualOrganization =
+    # Flag to turn to False if you want this agent to write in the CS
     DryRun = True
     Host = lcg-bdii.cern.ch:2170
+    # URLs for Glue2, if filled and GLUE2Only is False, the agent will look under theses URLs for Glue2 information
     GLUE2URLs =
+    # If True, only look for Glue2 information. If True, uses URLs from the Host option
     GLUE2Only = False
   }
+  ##END
   ##BEGIN VOMS2CSAgent
   VOMS2CSAgent
   {
+    #Time between cycles in seconds
     PollingTime = 14400
     MailFrom = noreply@dirac.system
     AutoAddUsers = True
@@ -43,13 +52,17 @@ Agents
     DetailedReport = True
     MakeHomeDirectory = False
     VO = Any
+    # Flag to turn to False if you want this agent to write in the CS (more granularity within other options)
     DryRun = True
   }
   ##END
+  ##BEGIN GOCDB2CSAgent
   GOCDB2CSAgent
   {
-    Cycles = 0
+    #Time between cycles in seconds
     PollingTime = 14400
+    # Flag to turn to False if you want this agent to write in the CS
     DryRun = True
   }
+  ##END
 }

Core/Utilities/StateMachine.py

@@ -1,12 +1,10 @@
 ########################################################################
-# $HeadURL $
-# File: State.py
 # Author: Krzysztof.Ciba@NOSPAMgmail.com
 # Date: 2013/07/03 10:33:02
 ########################################################################
-""" 
-:mod: State 
- 
+"""
+:mod: State
+
 .. module: State
 
 
@@ -17,113 +15,104 @@
 state machine
 """
 __RCSID__ = "$Id: $"
-##
-# @file State.py
-# @author Krzysztof.Ciba@NOSPAMgmail.com
-# @date 2013/07/03 10:33:07
-# @brief Definition of State class.
-
-## imports 
 
-#from DIRACSingleton import DIRACSingleton
 
-########################################################################
-class State( object ):
+class State(object):
   """
   .. class:: State
-  
+
   single state
   """
-  def __str__( self ):
+
+  def __str__(self):
     """ str() op """
     return self.__class__.__name__
 
-class StateMachine( object ):
+
+class StateMachine(object):
   """
   .. class:: StateMachine
 
   simple state machine
-  
+
   """
-  def __init__( self, state = None, transTable=None ):
+
+  def __init__(self, state=None, transTable=None):
     """ c'tor
 
     :param self: self reference
     :param mixed state: initial state
     :param dict transTable: transition table
     """
-    
-    if not issubclass( state.__class__, State ):
+
+    if not issubclass(state.__class__, State):
       raise TypeError("state should be inherited from State")
     self.__state = state
-    self.transTable = transTable if type(transTable) == dict else {}
+    self.transTable = transTable if isinstance(transTable, dict) else {}
 
-  def setState( self, state ):
+  def setState(self, state):
     """ set state """
-    assert issubclass( state.__class__, State )
+    assert issubclass(state.__class__, State)
     self.__state = state
 
-  def addTransition( self, fromState, toState, condition ):
+  def addTransition(self, fromState, toState, condition):
     """ add transtion rule from :fromState: to :toState: upon condition :condition: """
-    if not callable( condition ):
-      raise TypeError("condition should be callable" )
-    if not issubclass( fromState.__class__, State ):
-      raise TypeError("fromState should be inherited from State" )
-    if not issubclass( toState.__class__, State ):
-      raise TypeError("toState should be inherited from State" )
-    
+    if not callable(condition):
+      raise TypeError("condition should be callable")
+    if not issubclass(fromState.__class__, State):
+      raise TypeError("fromState should be inherited from State")
+    if not issubclass(toState.__class__, State):
+      raise TypeError("toState should be inherited from State")
+
     if fromState not in self.transTable:
       self.transTable[fromState] = {}
     self.transTable[fromState][toState] = condition
 
-  def next( self, *args, **kwargs ):
-    """ make transition to the next state 
-    
+  def next(self, *args, **kwargs):
+    """ make transition to the next state
+
     :param tuple args: args passed to condition
-    :param dict kwargs: kwargs passed to condition 
+    :param dict kwargs: kwargs passed to condition
     """
     for nextState, condition in self.transTable[self.__state].items():
-      if condition( *args, **kwargs ):
+      if condition(*args, **kwargs):
         self.__state = nextState
         break
 
   @property
-  def state( self ):
+  def state(self):
     """ get current state """
     return self.__state
 
   @state.setter
-  def state( self, state ):
-    assert issubclass( state.__class__, State ) 
+  def state(self, state):
+    assert issubclass(state.__class__, State)
     self.__state = state
 
 
-
-
-#class Waiting( State ):
+# class Waiting( State ):
 #  pass
-  
-#class Done( State ):
+
+# class Done( State ):
 #  pass
 
-#class Failed( State ):
+# class Failed( State ):
 #  pass
 
-#class Scheduled( State ):
+# class Scheduled( State ):
 #  pass
 
-#waiting = Waiting()
-#done = Done()
-#failed = Failed()
-#scheduled = Scheduled()
+# waiting = Waiting()
+# done = Done()
+# failed = Failed()
+# scheduled = Scheduled()
 
-#def toDone( slist ):
+# def toDone( slist ):
 #  return list(set(slist)) == [ "Done" ]
 
-#def toFailed( slist ):
+# def toFailed( slist ):
 #  return "Failed" in slist
-
-#def toWaiting( slist ):
+# def toWaiting( slist ):
 #  for st in slist:
 #    if st == "Done":
 #      continue
@@ -132,8 +121,7 @@ def state( self, state ):
 #    if st == "Waiting":
 #      return True
 #  return False
-
-#def toScheduled( slist ):
+# def toScheduled( slist ):
 #  for st in slist:
 #    if st == "Done":
 #      continue
@@ -143,21 +131,21 @@ def state( self, state ):
 #      return True
 #  return False
 """
-tr = { waiting: { done: toDone, 
-                  failed: toFailed, 
-                  scheduled: toScheduled, 
+tr = { waiting: { done: toDone,
+                  failed: toFailed,
+                  scheduled: toScheduled,
                   waiting: toWaiting },
-       scheduled: { failed: toFailed, 
-                    waiting: toWaiting, 
-                    done: toDone, 
+       scheduled: { failed: toFailed,
+                    waiting: toWaiting,
+                    done: toDone,
                     scheduled: toScheduled },
-       done: { waiting: toWaiting, 
+       done: { waiting: toWaiting,
                failed: toFailed,
-               scheduled: toScheduled, 
+               scheduled: toScheduled,
                done: toDone },
-       failed: { waiting: toWaiting, 
-                 scheduled: toScheduled, 
-                 done: toDone, 
+       failed: { waiting: toWaiting,
+                 scheduled: toScheduled,
+                 done: toDone,
                  failed: toFailed } }
 
 sm = StateMachine( waiting, tr )
@@ -178,5 +166,5 @@ def state( self, state ):
 sm.next( slist = [ "Done", "Done", "Done", "Failed" ] )
 print sm.state
 
-  
+
 """

ResourceStatusSystem/PolicySystem/PDP.py

@@ -276,7 +276,7 @@ def _combineSinglePolicyResults(self, singlePolicyRes):
 
     # We set the rssMachine on the current state ( ensures it is a valid one )
     # FIXME: probably this check can be done at takeDecision
-    machineStatus = self.rssMachine.setState(self.decisionParams['status'])
+    machineStatus = self.rssMachine.setState(self.decisionParams['status'], noWarn=True)
     if not machineStatus['OK']:
       return machineStatus
 

ResourceStatusSystem/PolicySystem/StateMachine.py

@@ -118,16 +118,16 @@ def levelOfState(self, state):
       return -1
     return self.states[state].level
 
-  def setState(self, candidateState):
+  def setState(self, candidateState, noWarn=False):
     """ Makes sure the state is either None or known to the machine, and that it is a valid state to move into.
         Final states are also checked.
 
     examples:
-      >>> sm0.setState( None )[ 'OK' ]
+      >>> sm0.setState(None)['OK']
           True
-      >>> sm0.setState( 'Nirvana' )[ 'OK' ]
+      >>> sm0.setState('Nirvana')['OK']
           True
-      >>> sm0.setState( 'AnotherState' )[ 'OK' ]
+      >>> sm0.setState('AnotherState')['OK']
           False
 
     :param state: state which will be set as current state of the StateMachine
@@ -140,9 +140,11 @@ def setState(self, candidateState):
 
     if candidateState is None:
       self.state = candidateState
-    elif candidateState in self.states.keys():
+    elif candidateState in self.states:
       if not self.states[self.state].stateMap:
-        gLogger.warn("Final state, won't move")
+        if not noWarn:
+          gLogger.warn("Final state, won't move",
+                       "(%s, asked to move to %s)" % (self.state, candidateState))
         return S_OK(self.state)
       if candidateState not in self.states[self.state].stateMap:
         gLogger.warn("Can't move from %s to %s, choosing a good one" % (self.state, candidateState))

docs/source/AdministratorGuide/Resources/agents2CS.rst

@@ -0,0 +1,34 @@
+.. _agents2CS:
+
+=================================
+Agents for discovery of resources
+=================================
+
+DIRAC has 2 agents that can be used for discovering and updating resources (e.g. computing elements).
+These agents are the:
+
+* :mod:`~DIRAC.ConfigurationSystem.Agent.Bdii2CSAgent`
+* :mod:`~DIRAC.ConfigurationSystem.Agent.GOCDB2CSAgent`
+
+Bdii2CSAgent
+------------
+
+The Bdii2CSAgent routinely contacts the BDII for availability of Computing and Storage Elements for a given VO,
+or for a few. It detects resources not yet present in the CS and notifies the administrators via email.
+It can also be configured to update the Configuration automatically.
+For the CEs and SEs already present in the CS, the agent is updating the existing entries if they were changed in the BDII recently.
+
+The Bdii2CSAgent can understand both Glue1 and Glue2 specifications.
+You can install more than one of such agents in parallel: a typical case is when some sites publish information
+only on Glue2, but not on Glue1.
+
+The agent is by default installed with a ``DryRun`` option which is set to True.
+Remove it, or set to False for automatic updating of the CS.
+
+GOCSB2CSAgent
+-------------
+
+The GOCDB2CSAgent looks in GOCDB for published resources. As of today, the agent only looks for Perfsonar endpoints.
+
+The agent is by default installed with a ``DryRun`` option which is set to True.
+Remove it, or set to False for automatic updating of the CS.

docs/source/AdministratorGuide/Resources/index.rst

@@ -20,6 +20,7 @@ contributing with their computing and storage capacity, available as conventiona
    computingelements
    messagequeues
    storage
+   agents2CS