diff --git a/ConfigurationSystem/Agent/Bdii2CSAgent.py b/ConfigurationSystem/Agent/Bdii2CSAgent.py index 54475342723..d9c880c1dd3 100644 --- a/ConfigurationSystem/Agent/Bdii2CSAgent.py +++ b/ConfigurationSystem/Agent/Bdii2CSAgent.py @@ -3,6 +3,14 @@ present in the CS and notifies the administrators. For the CEs and SEs already present in the CS, the agent is updating if necessary settings which were changed in the BDII recently + +The following options can be set for the Bdii2CSAgent. + +.. literalinclude:: ../ConfigTemplate.cfg + :start-after: ##BEGIN Bdii2CSAgent + :end-before: ##END + :dedent: 2 + :caption: Bdii2CSAgent options """ __RCSID__ = "$Id$" diff --git a/ConfigurationSystem/Agent/GOCDB2CSAgent.py b/ConfigurationSystem/Agent/GOCDB2CSAgent.py index bc079b0fbee..16813967539 100644 --- a/ConfigurationSystem/Agent/GOCDB2CSAgent.py +++ b/ConfigurationSystem/Agent/GOCDB2CSAgent.py @@ -2,6 +2,15 @@ This module provides GOCDB2CSAgent code. The agent is used to synchronize information between GOCDB and DIRAC configuration System (CS) +Right now it only adds perfosonar endpoints + +The following options can be set for the GOCDB2CSAgent. + +.. literalinclude:: ../ConfigTemplate.cfg + :start-after: ##BEGIN GOCDB2CSAgent + :end-before: ##END + :dedent: 2 + :caption: GOCDB2CSAgent options """ __RCSID__ = "$Id$" diff --git a/ConfigurationSystem/ConfigTemplate.cfg b/ConfigurationSystem/ConfigTemplate.cfg index 994dea2b368..356b39f790d 100644 --- a/ConfigurationSystem/ConfigTemplate.cfg +++ b/ConfigurationSystem/ConfigTemplate.cfg @@ -16,24 +16,33 @@ Services } Agents { + ##BEGIN Bdii2CSAgent Bdii2CSAgent { + #Time between cycles in seconds + PollingTime = 14400 BannedCEs = BannedSEs = SelectedSites = ProcessCEs = yes ProcessSEs = no + # Mail Notifications options MailTo = MailFrom = VirtualOrganization = + # Flag to turn to False if you want this agent to write in the CS DryRun = True Host = lcg-bdii.cern.ch:2170 + # URLs for Glue2, if filled and GLUE2Only is False, the agent will look under theses URLs for Glue2 information GLUE2URLs = + # If True, only look for Glue2 information. If True, uses URLs from the Host option GLUE2Only = False } + ##END ##BEGIN VOMS2CSAgent VOMS2CSAgent { + #Time between cycles in seconds PollingTime = 14400 MailFrom = noreply@dirac.system AutoAddUsers = True @@ -43,13 +52,17 @@ Agents DetailedReport = True MakeHomeDirectory = False VO = Any + # Flag to turn to False if you want this agent to write in the CS (more granularity within other options) DryRun = True } ##END + ##BEGIN GOCDB2CSAgent GOCDB2CSAgent { - Cycles = 0 + #Time between cycles in seconds PollingTime = 14400 + # Flag to turn to False if you want this agent to write in the CS DryRun = True } + ##END } diff --git a/Core/Utilities/StateMachine.py b/Core/Utilities/StateMachine.py index c6e52d22cf1..4ae2d5e1e74 100644 --- a/Core/Utilities/StateMachine.py +++ b/Core/Utilities/StateMachine.py @@ -1,12 +1,10 @@ ######################################################################## -# $HeadURL $ -# File: State.py # Author: Krzysztof.Ciba@NOSPAMgmail.com # Date: 2013/07/03 10:33:02 ######################################################################## -""" -:mod: State - +""" +:mod: State + .. module: State @@ -17,113 +15,104 @@ state machine """ __RCSID__ = "$Id: $" -## -# @file State.py -# @author Krzysztof.Ciba@NOSPAMgmail.com -# @date 2013/07/03 10:33:07 -# @brief Definition of State class. - -## imports -#from DIRACSingleton import DIRACSingleton -######################################################################## -class State( object ): +class State(object): """ .. class:: State - + single state """ - def __str__( self ): + + def __str__(self): """ str() op """ return self.__class__.__name__ -class StateMachine( object ): + +class StateMachine(object): """ .. class:: StateMachine simple state machine - + """ - def __init__( self, state = None, transTable=None ): + + def __init__(self, state=None, transTable=None): """ c'tor :param self: self reference :param mixed state: initial state :param dict transTable: transition table """ - - if not issubclass( state.__class__, State ): + + if not issubclass(state.__class__, State): raise TypeError("state should be inherited from State") self.__state = state - self.transTable = transTable if type(transTable) == dict else {} + self.transTable = transTable if isinstance(transTable, dict) else {} - def setState( self, state ): + def setState(self, state): """ set state """ - assert issubclass( state.__class__, State ) + assert issubclass(state.__class__, State) self.__state = state - def addTransition( self, fromState, toState, condition ): + def addTransition(self, fromState, toState, condition): """ add transtion rule from :fromState: to :toState: upon condition :condition: """ - if not callable( condition ): - raise TypeError("condition should be callable" ) - if not issubclass( fromState.__class__, State ): - raise TypeError("fromState should be inherited from State" ) - if not issubclass( toState.__class__, State ): - raise TypeError("toState should be inherited from State" ) - + if not callable(condition): + raise TypeError("condition should be callable") + if not issubclass(fromState.__class__, State): + raise TypeError("fromState should be inherited from State") + if not issubclass(toState.__class__, State): + raise TypeError("toState should be inherited from State") + if fromState not in self.transTable: self.transTable[fromState] = {} self.transTable[fromState][toState] = condition - def next( self, *args, **kwargs ): - """ make transition to the next state - + def next(self, *args, **kwargs): + """ make transition to the next state + :param tuple args: args passed to condition - :param dict kwargs: kwargs passed to condition + :param dict kwargs: kwargs passed to condition """ for nextState, condition in self.transTable[self.__state].items(): - if condition( *args, **kwargs ): + if condition(*args, **kwargs): self.__state = nextState break @property - def state( self ): + def state(self): """ get current state """ return self.__state @state.setter - def state( self, state ): - assert issubclass( state.__class__, State ) + def state(self, state): + assert issubclass(state.__class__, State) self.__state = state - - -#class Waiting( State ): +# class Waiting( State ): # pass - -#class Done( State ): + +# class Done( State ): # pass -#class Failed( State ): +# class Failed( State ): # pass -#class Scheduled( State ): +# class Scheduled( State ): # pass -#waiting = Waiting() -#done = Done() -#failed = Failed() -#scheduled = Scheduled() +# waiting = Waiting() +# done = Done() +# failed = Failed() +# scheduled = Scheduled() -#def toDone( slist ): +# def toDone( slist ): # return list(set(slist)) == [ "Done" ] -#def toFailed( slist ): +# def toFailed( slist ): # return "Failed" in slist - -#def toWaiting( slist ): +# def toWaiting( slist ): # for st in slist: # if st == "Done": # continue @@ -132,8 +121,7 @@ def state( self, state ): # if st == "Waiting": # return True # return False - -#def toScheduled( slist ): +# def toScheduled( slist ): # for st in slist: # if st == "Done": # continue @@ -143,21 +131,21 @@ def state( self, state ): # return True # return False """ -tr = { waiting: { done: toDone, - failed: toFailed, - scheduled: toScheduled, +tr = { waiting: { done: toDone, + failed: toFailed, + scheduled: toScheduled, waiting: toWaiting }, - scheduled: { failed: toFailed, - waiting: toWaiting, - done: toDone, + scheduled: { failed: toFailed, + waiting: toWaiting, + done: toDone, scheduled: toScheduled }, - done: { waiting: toWaiting, + done: { waiting: toWaiting, failed: toFailed, - scheduled: toScheduled, + scheduled: toScheduled, done: toDone }, - failed: { waiting: toWaiting, - scheduled: toScheduled, - done: toDone, + failed: { waiting: toWaiting, + scheduled: toScheduled, + done: toDone, failed: toFailed } } sm = StateMachine( waiting, tr ) @@ -178,5 +166,5 @@ def state( self, state ): sm.next( slist = [ "Done", "Done", "Done", "Failed" ] ) print sm.state - + """ diff --git a/ResourceStatusSystem/PolicySystem/PDP.py b/ResourceStatusSystem/PolicySystem/PDP.py index 39df1a7ea48..ec1137130c3 100644 --- a/ResourceStatusSystem/PolicySystem/PDP.py +++ b/ResourceStatusSystem/PolicySystem/PDP.py @@ -276,7 +276,7 @@ def _combineSinglePolicyResults(self, singlePolicyRes): # We set the rssMachine on the current state ( ensures it is a valid one ) # FIXME: probably this check can be done at takeDecision - machineStatus = self.rssMachine.setState(self.decisionParams['status']) + machineStatus = self.rssMachine.setState(self.decisionParams['status'], noWarn=True) if not machineStatus['OK']: return machineStatus diff --git a/ResourceStatusSystem/PolicySystem/StateMachine.py b/ResourceStatusSystem/PolicySystem/StateMachine.py index 7dae89a9466..142e4492fef 100644 --- a/ResourceStatusSystem/PolicySystem/StateMachine.py +++ b/ResourceStatusSystem/PolicySystem/StateMachine.py @@ -118,16 +118,16 @@ def levelOfState(self, state): return -1 return self.states[state].level - def setState(self, candidateState): + def setState(self, candidateState, noWarn=False): """ Makes sure the state is either None or known to the machine, and that it is a valid state to move into. Final states are also checked. examples: - >>> sm0.setState( None )[ 'OK' ] + >>> sm0.setState(None)['OK'] True - >>> sm0.setState( 'Nirvana' )[ 'OK' ] + >>> sm0.setState('Nirvana')['OK'] True - >>> sm0.setState( 'AnotherState' )[ 'OK' ] + >>> sm0.setState('AnotherState')['OK'] False :param state: state which will be set as current state of the StateMachine @@ -140,9 +140,11 @@ def setState(self, candidateState): if candidateState is None: self.state = candidateState - elif candidateState in self.states.keys(): + elif candidateState in self.states: if not self.states[self.state].stateMap: - gLogger.warn("Final state, won't move") + if not noWarn: + gLogger.warn("Final state, won't move", + "(%s, asked to move to %s)" % (self.state, candidateState)) return S_OK(self.state) if candidateState not in self.states[self.state].stateMap: gLogger.warn("Can't move from %s to %s, choosing a good one" % (self.state, candidateState)) diff --git a/docs/source/AdministratorGuide/Resources/agents2CS.rst b/docs/source/AdministratorGuide/Resources/agents2CS.rst new file mode 100644 index 00000000000..3fb4dbc00b7 --- /dev/null +++ b/docs/source/AdministratorGuide/Resources/agents2CS.rst @@ -0,0 +1,34 @@ +.. _agents2CS: + +================================= +Agents for discovery of resources +================================= + +DIRAC has 2 agents that can be used for discovering and updating resources (e.g. computing elements). +These agents are the: + +* :mod:`~DIRAC.ConfigurationSystem.Agent.Bdii2CSAgent` +* :mod:`~DIRAC.ConfigurationSystem.Agent.GOCDB2CSAgent` + +Bdii2CSAgent +------------ + +The Bdii2CSAgent routinely contacts the BDII for availability of Computing and Storage Elements for a given VO, +or for a few. It detects resources not yet present in the CS and notifies the administrators via email. +It can also be configured to update the Configuration automatically. +For the CEs and SEs already present in the CS, the agent is updating the existing entries if they were changed in the BDII recently. + +The Bdii2CSAgent can understand both Glue1 and Glue2 specifications. +You can install more than one of such agents in parallel: a typical case is when some sites publish information +only on Glue2, but not on Glue1. + +The agent is by default installed with a ``DryRun`` option which is set to True. +Remove it, or set to False for automatic updating of the CS. + +GOCSB2CSAgent +------------- + +The GOCDB2CSAgent looks in GOCDB for published resources. As of today, the agent only looks for Perfsonar endpoints. + +The agent is by default installed with a ``DryRun`` option which is set to True. +Remove it, or set to False for automatic updating of the CS. diff --git a/docs/source/AdministratorGuide/Resources/index.rst b/docs/source/AdministratorGuide/Resources/index.rst index 8a1d437dc42..a5adf2b0814 100644 --- a/docs/source/AdministratorGuide/Resources/index.rst +++ b/docs/source/AdministratorGuide/Resources/index.rst @@ -20,6 +20,7 @@ contributing with their computing and storage capacity, available as conventiona computingelements messagequeues storage + agents2CS diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index 18a9f127535..725fa0aeb5c 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -345,7 +345,7 @@ function installDIRAC(){ } >> "$CLIENTINSTALLDIR/dirac-ci-install.cfg" fi - if ! ./dirac-install -r $DIRAC_RELEASE -t client "${INSTALLOPTIONS[@]}" "$CLIENTINSTALLDIR/dirac-ci-install.cfg" $DEBUG; then + if ! ./dirac-install -r $DIRAC_RELEASE -t client $INSTALLOPTIONS "$CLIENTINSTALLDIR/dirac-ci-install.cfg" $DEBUG; then echo 'ERROR: DIRAC client installation failed' exit 1 fi diff --git a/tests/System/client_core.sh b/tests/System/client_core.sh index cffaa0c3bcf..0bcacb9d425 100755 --- a/tests/System/client_core.sh +++ b/tests/System/client_core.sh @@ -58,8 +58,8 @@ then fi echo " " -echo "dirac-proxy-init -g dirac_prod" -dirac-proxy-init -g dirac_prod +echo "dirac-proxy-init" +dirac-proxy-init if [ $? -ne 0 ] then exit $?