diff --git a/VERSION b/VERSION index 15a2799..1809198 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.0 +3.4.0 diff --git a/bin/Console.php b/bin/Console.php index 3b27e46..4f0f5a9 100755 --- a/bin/Console.php +++ b/bin/Console.php @@ -2,7 +2,7 @@ isRedisEnabled() || $Config->storeLiveDateInArchive()) { for ($i = 0; $i < $Config->getNumberOfHoststatusWorkers(); $i++) { - $Syslog->info('Fork new host status worker'); - $HoststatusChild = new Statusengine\HoststatusChild( - $Config, - $ParentPid, - $Syslog - ); - $hoststatusChildPid = $HoststatusChild->fork(); - $pids[] = $hoststatusChildPid; + $pids[] = $ChildFactory->forkHoststatusChild(); } } @@ -70,74 +65,29 @@ if ($Config->isCrateEnabled() || $Config->isMysqlEnabled()) { for ($i = 0; $i < $Config->getNumberOfLogentryWorkers(); $i++) { - $Syslog->info('Fork new log entry worker'); - $LogentryChild = new Statusengine\LogentryChild( - $Config, - $ParentPid, - $Syslog - ); - $logentryChildPid = $LogentryChild->fork(); - $pids[] = $logentryChildPid; + $pids[] = $ChildFactory->forkLogentryChild(); } for ($i = 0; $i < $Config->getNumberOfStatechangeWorkers(); $i++) { - $Syslog->info('Fork new state change worker'); - $StatechangeConfig = new Statusengine\Config\Statechange(); - $StatechangeSignalHandler = new \Statusengine\ChildSignalHandler(); - $StatechangeStatistics = new \Statusengine\Redis\Statistics($Config, $Syslog); - $StatechangeChild = new Statusengine\StatechangeChild( - $Config, - $ParentPid, - $Syslog - ); - $statechangeChildPid = $StatechangeChild->fork(); - $pids[] = $statechangeChildPid; + $pids[] = $ChildFactory->forkStatechangeChild(); } for ($i = 0; $i < $Config->getNumberOfHostcheckWorkers(); $i++) { - $Syslog->info('Fork new host check worker'); - $HostcheckChild = new Statusengine\HostcheckChild( - $Config, - $ParentPid, - $Syslog - ); - $hostcheckChildPid = $HostcheckChild->fork(); - $pids[] = $hostcheckChildPid; + $pids[] = $ChildFactory->forkHostcheckChild(); } for ($i = 0; $i < $Config->getNumberOfServicecheckWorkers(); $i++) { - $Syslog->info('Fork new service check worker'); - $ServicecheckChild = new Statusengine\ServicecheckChild( - $Config, - $ParentPid, - $Syslog - ); - $servicecheckChildPid = $ServicecheckChild->fork(); - $pids[] = $servicecheckChildPid; + $pids[] = $ChildFactory->forkServicecheckChild(); } for ($i = 0; $i < $Config->getNumberOfMiscWorkers(); $i++) { - $Syslog->info('Fork new misc worker'); - $MiscChild = new Statusengine\MiscChild( - $Config, - $ParentPid, - $Syslog - ); - $miscChildPid = $MiscChild->fork(); - $pids[] = $miscChildPid; + $pids[] = $ChildFactory->forkMiscChild(); } } if ($Config->isProcessPerfdataEnabled() && $Config->isOnePerfdataBackendEnabled()) { for ($i = 0; $i < $Config->getNumberOfPerfdataWorkers(); $i++) { - $Syslog->info('Fork new performance data worker'); - $PerfdataChild = new Statusengine\PerfdataChild( - $Config, - $ParentPid, - $Syslog - ); - $perfdataChildPid = $PerfdataChild->fork(); - $pids[] = $perfdataChildPid; + $pids[] = $ChildFactory->forkPerfdataChild(); } } @@ -160,7 +110,8 @@ $TaskManager, $Syslog, $MonitoringRestartConfig, - $StorageBackend + $StorageBackend, + $ChildFactory ); foreach ($pids as $Pid) { $ParentProcess->addChildPid($Pid); diff --git a/bootstrap.php b/bootstrap.php index fb05abe..69632ed 100644 --- a/bootstrap.php +++ b/bootstrap.php @@ -1,7 +1,7 @@ nodeName; } - if($startTime === null){ + if ($startTime === null) { $startTime = time(); } @@ -119,7 +119,7 @@ public function saveNodeName($nodeName = null, $startTime = null) { /** * @return array */ - public function getNodes(){ + public function getNodes() { $this->connect(); $query = $this->Connection->prepare('SELECT * FROM statusengine_nodes ORDER BY node_name ASC'); @@ -131,7 +131,7 @@ public function getNodes(){ } $this->disconnect(); $nodes = []; - foreach($result as $record){ + foreach ($result as $record) { $nodes[] = NodeName::fromCrateDb($record); } return $nodes; @@ -140,7 +140,7 @@ public function getNodes(){ /** * @param string $nodeName */ - public function deleteNodeByName($nodeName){ + public function deleteNodeByName($nodeName) { $this->connect(); $query = $this->Connection->prepare('DELETE FROM statusengine_nodes WHERE node_name=?'); $query->bindValue(1, $nodeName); @@ -172,6 +172,9 @@ public function connect() { $this->Connection->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); } catch (\Exception $e) { $this->Syslog->error($e->getMessage()); + + //rethrow exception that the parent process will not die. + throw $e; } return $this->Connection; } @@ -179,7 +182,7 @@ public function connect() { /** * @param int $timeout in seconds */ - public function setTimeout($timeout){ + public function setTimeout($timeout) { $this->Connection->setAttribute(PDO::ATTR_TIMEOUT, $timeout); } @@ -379,13 +382,13 @@ public function deleteTaskByUuids($uuids = []) { public function deleteHostchecksOlderThan($timestamp) { $partitions = $this->getPartitionsByTableName('statusengine_hostchecks'); $daysToDelete = []; - foreach($partitions as $record){ - if(isset($record['values']['day']) && $record['values']['day'] < $timestamp){ + foreach ($partitions as $record) { + if (isset($record['values']['day']) && $record['values']['day'] < $timestamp) { $daysToDelete[] = $record['values']['day']; } } - foreach($daysToDelete as $partition){ + foreach ($daysToDelete as $partition) { $this->dropPartitionsFromTableByTableNameAndDayValue('statusengine_hostchecks', $partition); } } @@ -408,13 +411,13 @@ public function deleteHostAcknowledgementsOlderThan($timestamp) { public function deleteHostNotificationsOlderThan($timestamp) { $partitions = $this->getPartitionsByTableName('statusengine_host_notifications'); $daysToDelete = []; - foreach($partitions as $record){ - if(isset($record['values']['day']) && $record['values']['day'] < $timestamp){ + foreach ($partitions as $record) { + if (isset($record['values']['day']) && $record['values']['day'] < $timestamp) { $daysToDelete[] = $record['values']['day']; } } - foreach($daysToDelete as $partition){ + foreach ($daysToDelete as $partition) { $this->dropPartitionsFromTableByTableNameAndDayValue('statusengine_host_notifications', $partition); } } @@ -425,13 +428,13 @@ public function deleteHostNotificationsOlderThan($timestamp) { public function deleteHostStatehistoryOlderThan($timestamp) { $partitions = $this->getPartitionsByTableName('statusengine_host_statehistory'); $daysToDelete = []; - foreach($partitions as $record){ - if(isset($record['values']['day']) && $record['values']['day'] < $timestamp){ + foreach ($partitions as $record) { + if (isset($record['values']['day']) && $record['values']['day'] < $timestamp) { $daysToDelete[] = $record['values']['day']; } } - foreach($daysToDelete as $partition){ + foreach ($daysToDelete as $partition) { $this->dropPartitionsFromTableByTableNameAndDayValue('statusengine_host_statehistory', $partition); } } @@ -454,13 +457,13 @@ public function deleteHostDowntimeHistoryOlderThan($timestamp) { public function deleteServicechecksOlderThan($timestamp) { $partitions = $this->getPartitionsByTableName('statusengine_servicechecks'); $daysToDelete = []; - foreach($partitions as $record){ - if(isset($record['values']['day']) && $record['values']['day'] < $timestamp){ + foreach ($partitions as $record) { + if (isset($record['values']['day']) && $record['values']['day'] < $timestamp) { $daysToDelete[] = $record['values']['day']; } } - foreach($daysToDelete as $partition){ + foreach ($daysToDelete as $partition) { $this->dropPartitionsFromTableByTableNameAndDayValue('statusengine_servicechecks', $partition); } } @@ -483,13 +486,13 @@ public function deleteServiceAcknowledgementsOlderThan($timestamp) { public function deleteServiceNotificationsOlderThan($timestamp) { $partitions = $this->getPartitionsByTableName('statusengine_service_notifications'); $daysToDelete = []; - foreach($partitions as $record){ - if(isset($record['values']['day']) && $record['values']['day'] < $timestamp){ + foreach ($partitions as $record) { + if (isset($record['values']['day']) && $record['values']['day'] < $timestamp) { $daysToDelete[] = $record['values']['day']; } } - foreach($daysToDelete as $partition){ + foreach ($daysToDelete as $partition) { $this->dropPartitionsFromTableByTableNameAndDayValue('statusengine_service_notifications', $partition); } } @@ -531,13 +534,13 @@ public function deleteServiceDowntimeHistoryOlderThan($timestamp) { public function deleteLogentriesOlderThan($timestamp) { $partitions = $this->getPartitionsByTableName('statusengine_logentries'); $daysToDelete = []; - foreach($partitions as $record){ - if(isset($record['values']['day']) && $record['values']['day'] < $timestamp){ + foreach ($partitions as $record) { + if (isset($record['values']['day']) && $record['values']['day'] < $timestamp) { $daysToDelete[] = $record['values']['day']; } } - foreach($daysToDelete as $partition){ + foreach ($daysToDelete as $partition) { $this->dropPartitionsFromTableByTableNameAndDayValue('statusengine_logentries', $partition); } } @@ -558,18 +561,18 @@ public function deleteTasksOlderThan($timestamp) { * @param int $timestamp * @return bool */ - public function deletePerfdataOlderThan($timestamp){ + public function deletePerfdataOlderThan($timestamp) { $timestamp = $timestamp * 1000; $partitions = $this->getPartitionsByTableName('statusengine_perfdata'); $daysToDelete = []; - foreach($partitions as $record){ - if(isset($record['values']['day']) && $record['values']['day'] < $timestamp){ - $daysToDelete[] = $record['values']['day']; - } + foreach ($partitions as $record) { + if (isset($record['values']['day']) && $record['values']['day'] < $timestamp) { + $daysToDelete[] = $record['values']['day']; + } } - foreach($daysToDelete as $partition){ + foreach ($daysToDelete as $partition) { $this->dropPartitionsFromTableByTableNameAndDayValue('statusengine_perfdata', $partition); } } @@ -578,7 +581,7 @@ public function deletePerfdataOlderThan($timestamp){ * @param string $tablename * @return array */ - public function getPartitionsByTableName($tablename){ + public function getPartitionsByTableName($tablename) { $query = $this->prepare( 'SELECT * FROM information_schema.table_partitions WHERE table_name=?' ); @@ -592,7 +595,7 @@ public function getPartitionsByTableName($tablename){ * @param int $dayValue * @return bool */ - public function dropPartitionsFromTableByTableNameAndDayValue($tableName, $dayValue){ + public function dropPartitionsFromTableByTableNameAndDayValue($tableName, $dayValue) { $query = $this->prepare(sprintf('DELETE FROM %s WHERE DAY = ?', $tableName)); $query->bindValue(1, $dayValue); return $query->execute(); @@ -626,7 +629,7 @@ public function getServiceScheduleddowntimeBackend() { return new CrateServiceScheduleddowntime($this, $this->nodeName); } - public function monitoringengineWasRestarted(){ + public function monitoringengineWasRestarted() { $this->connect(); $Hoststatus = new CrateHoststatus($this, $this->BulkInsertObjectStore, $this->nodeName); $Hoststatus->truncate(); diff --git a/src/Backends/Crate/CrateModel.php b/src/Backends/Crate/CrateModel.php index 3312cf1..6abc04f 100644 --- a/src/Backends/Crate/CrateModel.php +++ b/src/Backends/Crate/CrateModel.php @@ -1,7 +1,7 @@ Connection->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); } catch (\Exception $e) { $this->Syslog->error($e->getMessage()); + + //rethrow exception that the parent process will not die. + throw $e; } //Enable UTF-8 - $query = $this->Connection->prepare('SET NAMES utf8'); - $query->execute(); + try { + $query = $this->Connection->prepare('SET NAMES utf8'); + $query->execute(); + } catch (\Exception $e) { + $this->Syslog->error($e->getMessage()); + //rethrow exception that the parent process will not die. + throw $e; + } return $this->Connection; } @@ -310,7 +319,7 @@ public function deleteTaskByUuids($uuids = []) { * @param int $timestamp * @return bool */ - public function deletePerfdataOlderThan($timestamp){ + public function deletePerfdataOlderThan($timestamp) { $query = $this->prepare( 'DELETE FROM statusengine_perfdata WHERE timestamp_unix < ?' ); diff --git a/src/Backends/MySQL/MysqlModel.php b/src/Backends/MySQL/MysqlModel.php index 3d0c93b..b5b3e26 100644 --- a/src/Backends/MySQL/MysqlModel.php +++ b/src/Backends/MySQL/MysqlModel.php @@ -1,7 +1,7 @@ Pid = new Pid(getmypid()); + $this->Pid = new Pid(getmypid(), $this->childName); $this->setup(); @@ -64,7 +69,7 @@ public function fork() { } //Return back to the parent process - return new Pid($pid); + return new Pid($pid, $this->childName); } /** diff --git a/src/ChildFactory.php b/src/ChildFactory.php new file mode 100644 index 0000000..cdc2f8c --- /dev/null +++ b/src/ChildFactory.php @@ -0,0 +1,178 @@ +. + */ + +namespace Statusengine; + + +use Statusengine\ValueObjects\Pid; + +class ChildFactory { + + /** + * @var Config + */ + private $Config; + + /** + * @var Syslog + */ + private $Syslog; + + /** + * @var Pid + */ + private $ParentPid; + + public function __construct(Config $Config, Syslog $Syslog, Pid $ParentPid) { + $this->Config = $Config; + $this->Syslog = $Syslog; + $this->ParentPid = $ParentPid; + } + + /** + * @return Pid + */ + public function forkHoststatusChild() { + $this->Syslog->info('Fork new host status worker'); + $HoststatusChild = new HoststatusChild( + $this->Config, + $this->ParentPid, + $this->Syslog + ); + $Pid = $HoststatusChild->fork(); + return $Pid; + } + + /** + * @return Pid + */ + public function forkServicestatusChild() { + $this->Syslog->info('Fork new service status worker'); + $ServicestatusChild = new ServicestatusChild( + $this->Config, + $this->ParentPid, + $this->Syslog + ); + $Pid = $ServicestatusChild->fork(); + return $Pid; + } + + /** + * @return Pid + */ + public function forkLogentryChild() { + $this->Syslog->info('Fork new log entry worker'); + $LogentryChild = new LogentryChild( + $this->Config, + $this->ParentPid, + $this->Syslog + ); + $Pid = $LogentryChild->fork(); + return $Pid; + } + + /** + * @return Pid + */ + public function forkStatechangeChild() { + $this->Syslog->info('Fork new state change worker'); + $StatechangeChild = new StatechangeChild( + $this->Config, + $this->ParentPid, + $this->Syslog + ); + $Pid = $StatechangeChild->fork(); + return $Pid; + } + + /** + * @return Pid + */ + public function forkHostcheckChild() { + $this->Syslog->info('Fork new host check worker'); + $HostcheckChild = new HostcheckChild( + $this->Config, + $this->ParentPid, + $this->Syslog + ); + $Pid = $HostcheckChild->fork(); + return $Pid; + } + + /** + * @return Pid + */ + public function forkServicecheckChild() { + $this->Syslog->info('Fork new service check worker'); + $ServicecheckChild = new ServicecheckChild( + $this->Config, + $this->ParentPid, + $this->Syslog + ); + $Pid = $ServicecheckChild->fork(); + return $Pid; + } + + /** + * @return Pid + */ + public function forkMiscChild() { + $this->Syslog->info('Fork new misc worker'); + $MiscChild = new MiscChild( + $this->Config, + $this->ParentPid, + $this->Syslog + ); + $Pid = $MiscChild->fork(); + return $Pid; + } + + /** + * @return Pid + */ + public function forkPerfdataChild() { + $this->Syslog->info('Fork new performance data worker'); + $PerfdataChild = new PerfdataChild( + $this->Config, + $this->ParentPid, + $this->Syslog + ); + $Pid = $PerfdataChild->fork(); + return $Pid; + } + + /** + * @param string $childName + * @return bool + */ + public function canChildBeReborn($childName) { + $methodName = 'fork' . $childName; + return method_exists($this, $methodName); + } + + /** + * @param $childName + * @return Pid + */ + public function respawn($childName) { + $methodName = 'fork' . $childName; + return call_user_func([$this, $methodName]); + } + +} diff --git a/src/ChildSignalHandler.php b/src/ChildSignalHandler.php index 63060eb..75f2849 100644 --- a/src/ChildSignalHandler.php +++ b/src/ChildSignalHandler.php @@ -1,7 +1,7 @@ StatisticCollector = $StatisticCollector; $this->Config = $Config; @@ -101,6 +112,7 @@ public function __construct( $this->Syslog = $Syslog; $this->MonitoringRestartConfig = $MonitoringRestartConfig; $this->StorageBackend = $StorageBackend; + $this->ChildFactory = $ChildFactory; $this->QueueingEngine = new QueueingEngine($this->Config, $this->MonitoringRestartConfig); $this->Queue = $this->QueueingEngine->getQueue(); @@ -121,18 +133,43 @@ public function loop() { $this->checkForDeadChilds(); if ($this->checkForCommands) { - $this->TaskManager->checkAndProcessTasks(); + try { + $this->TaskManager->checkAndProcessTasks(); + }catch (\Exception $exception){ + $this->Syslog->error($exception->getMessage()); + } } //Also replaces sleep(1) $jobData = $this->Queue->getJob(); if ($jobData !== null) { //Monitoring engine was restarted - if($jobData->object_type == 102){ + if ($jobData->object_type == 102) { $this->Syslog->info('Catch monitoring restart. Trigger callbacks...'); $this->StorageBackend->monitoringengineWasRestarted(); } } + + //Check for dead childs + if (!empty($this->deadChilds)) { + /** @var Pid $deadChild */ + foreach ($this->deadChilds as $deadChild) { + $this->removePid($deadChild); + if ($this->ChildFactory->canChildBeReborn($deadChild->getChildName())) { + $this->Syslog->info('Respawn dead child'); + $newPid = $this->ChildFactory->respawn($deadChild->getChildName()); + $this->addChildPid($newPid); + } else { + $this->Syslog->error(sprintf( + 'Can not respawn child of type %s', + $deadChild->getChildName() + )); + } + } + $this->deadChilds = []; + //Update StatisticCollector with the new PIDs + $this->StatisticCollector->setPids($this->getChildPids()); + } } } @@ -150,8 +187,12 @@ public function getChildPids() { return $this->pids; } + /** + * @return Pid + * @throws Exception\NotNumericValueException + */ public function getPid() { - return new Pid(getmypid()); + return new Pid(getmypid(), 'ParentProcess'); } public function checkForDeadChilds() { @@ -160,11 +201,21 @@ public function checkForDeadChilds() { if (pcntl_waitpid($Pid->getPid(), $status, WNOHANG) == 0) { //Child still alive $pidsAlive[] = $Pid; - }else{ + } else { $this->Syslog->alert(sprintf('Child with pid %s is dead!!', $Pid->getPid())); + $this->deadChilds[] = $Pid; } } $this->pids = $pidsAlive; } + private function removePid(Pid $PidToRemove) { + /** @var Pid $Pid */ + foreach ($this->pids as $index => $Pid) { + if ($Pid->getPid() === $PidToRemove->getPid()) { + unset($this->pids[$index]); + } + } + } + } \ No newline at end of file diff --git a/src/ParentSignalHandler.php b/src/ParentSignalHandler.php index c1540d4..047172c 100644 --- a/src/ParentSignalHandler.php +++ b/src/ParentSignalHandler.php @@ -1,7 +1,7 @@ pid = $pid; + $this->childName = $childName; } /** @@ -46,4 +53,12 @@ public function __construct($pid) { public function getPid() { return $this->pid; } -} \ No newline at end of file + + /** + * @return string + */ + public function getChildName() { + return $this->childName; + } + +} diff --git a/src/ValueObjects/Servicecheck.php b/src/ValueObjects/Servicecheck.php index 6438960..ac9e9b1 100644 --- a/src/ValueObjects/Servicecheck.php +++ b/src/ValueObjects/Servicecheck.php @@ -1,7 +1,7 @@