#!/usr/bin/perl -w
#
# monitord 1.93
# Copyright (C) 2000-2007 Risto Vaarandi
# 
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

use strict;

# Global variables

use vars qw(
  $appl
  $cmdcol
  $conffile
  $daemon
  $dfcmd
  %disk_limits
  @exec_scripts
  $facility
  $freekbcol
  $freeprcol
  $interval
  $keepalive
  $keepalivecounter
  $keepalivefactor
  $keepalivemsg
  $level
  %load_limits
  $maxstdout
  $mountpcol
  $normalsev
  $notifyprog
  $pidfile
  %proc_limits
  $pscmd
  $refresh
  $syslogavail
  $terminate
  $uptimecmd
);

# Load the modules

use Getopt::Long;
use POSIX qw(setsid);
$syslogavail = eval { require Sys::Syslog };

# Add bin-directories to the PATH environment variable

if (exists($ENV{"PATH"}) && length($ENV{"PATH"})) {
  $ENV{"PATH"} .= ":/bin:/usr/bin:/usr/local/bin";
} else {
  $ENV{"PATH"} = "/bin:/usr/bin:/usr/local/bin";
}

# Set commands and flags for monitoring the state of the system

if ($^O =~ /linux/i) {

  $dfcmd = "df -lP";
  $mountpcol = 6;
  $freeprcol = 5;
  $freekbcol = 4;

  $pscmd = "ps -eo args";
  $cmdcol = 0;

  $uptimecmd = "uptime";

} elsif ($^O =~ /solaris/i) {

  $dfcmd = "df -lk";
  $mountpcol = 6;
  $freeprcol = 5;
  $freekbcol = 4;

  $pscmd = "ps -eo args";
  $cmdcol = 0;

  $uptimecmd = "uptime";

} elsif ($^O =~ /bsd/i || $^O =~ /darwin/i) {

  $dfcmd = "df -lk";
  $mountpcol = 6;
  $freeprcol = 5;
  $freekbcol = 4;

  $pscmd = "ps -axo command";
  $cmdcol = 0;

  $uptimecmd = "uptime";

} else {

  $dfcmd = "df -lk";
  $mountpcol = 6;
  $freeprcol = 5;
  $freekbcol = 4;

  $pscmd = "ps -e";
  $cmdcol = -1;

  $uptimecmd = "uptime";

}

# -------------------------------------------------------------------------

# Function that prints version information

sub print_version {

print STDERR << "VERSIONINFO";

monitord 1.93
Copyright (C) 2000-2007 Risto Vaarandi
This is free software.  You may redistribute copies of it under the terms of
the GNU General Public License <http://www.gnu.org/licenses/gpl.html>.
There is NO WARRANTY, to the extent permitted by law.

VERSIONINFO
}

# Function that prints usage information

sub print_usage {

print STDERR << "USAGEINFO";

Usage: $0 [OPTION]... CONFFILE

Options:
  --pid-file=<file>  store process ID to <file>
                     (default is to run without creating the pid file)

  --poll-interval=<int>  check the system after <int> second intervals
                         (default is 60 seconds)

  --keepalive-interval=<int>  send keepalives after <int> second intervals;
                              if <int> is not a multiple of the value given
                              with the --poll-interval option, it is rounded
                              down to the nearest multiple
                              (default is 300 seconds)

  --keepalive-message=<text>  use <text> for keepalive notification text
                              (default is "keepalive message")

  --notify-program=<prog>  all notifications are sent with program <prog>
                           (default is to write "application=%a object=%o 
                            severity=%s text=%t" to standard output)

  --severity-for-normal=<sev>  notifications for normal system conditions
                               (e.g., keepalives) have the severity <sev> 
                               (default is "normal")

  --max-stdout=<int>  take at most <int> lines from the standard output
                      of the monitoring script when forming a notification
                      text (default is 10)

  --program-name=<name>  use <name> for program name when logging to syslog
                         and sending notifications 
                         (default is $0 without the leading path)

  --syslog=<pri>  log program internal errors with syslog priority <pri>
                  (default is "user.err")

  --daemon | --nodaemon  run as a daemon | don't run as a daemon
                         (default is --nodaemon)

  --help, -?  print the usage information

  --version  print the version information

Notify program (see the --notify-program option) may contain the following 
variables which will be substituted with the real values when sending 
the notification:

%a - application that sent the notification (see the --program-name option)
%o - object that the notification is about (e.g., 'sendmail', '/usr', etc.)
%s - severity of the notification (e.g., 'minor', 'major', etc.)
%t - notification text (e.g., 'Too few sendmail running', 
                              'Filesystem /usr: less than 10% free', etc.) 

Note that before substitution all values will be put inside apostrophes,
and all apostrophes inside the values will be masked.


An example command line for starting monitord:

/usr/bin/monitord --daemon --poll-interval=60 --severity-for-normal=notice \\
  --notify-prog='logger -i -t %a -p daemon.%s %t' /etc/monitord.conf


Configuration file directives:

proc <regexp>  <min> <max>  <severity>  <resend time>
disk <mountpoint>  <disklimit> <diskreset>  <severity>  <resend time>
load <loadtype>  <loadlimit> <loadreset>  <severity>  <resend time>
exec <label> <progname> <flag1> ... <flagN>  <severity>  <resend time>

	where <severity> = severity string (e.g., 'minor', 'major', etc.)
              <resend time> = if set to non-zero, notifications are resent 
                after every <resend time> seconds during the error condition.
                If set to 0, every notification is sent only once when
                the error condition is discovered.
              <regexp> = regular expression for matching process command lines
	      <disklimit> = integer followed by the optional % symbol
	      <diskreset> = integer followed by the optional % symbol. Reset
                value must be specified in the same units as the limit value.
	      <loadtype> = 1, 5, 15
	      <loadlimit> = float or integer  
	      <loadreset> = float or integer
	      <label> = unique string describing the nature of the script


An example configuration file:

# Issue a critical notification if there is less than 1 or more than 2 "ls -l"
# processes in the process table. If the number of "ls -l" processes becomes 
# 1 or 2, an "error clear" notification is issued (its severity can be set 
# with the --severity-for-normal option).

proc ls\\s+-l 1 2 critical 0

# Issue a critical notification if there is no sshd process running. Continue 
# issuing critical notifications after every 30 seconds until sshd appears in
# the process table again. When sshd is running again, issue an "error clear"
# notification.

proc sshd 1 - critical 30

# Issue a major notification if there is less than 10% (10Mb) free space in 
# /tmp. If more than 30% (40Mb) of /tmp becomes free, an "error clear" 
# notification is issued. 

disk /tmp 10% 30% major 0
disk /tmp 10240 40960 major 0

# Issue a minor notification if the 15 minute load exceeds 2.0. Repeat minor
# notifications after every 60 seconds as long as the load stays above 2.0. 
# If the 15 minute load drops below 1.0, issue an "error clear" notification.

load 15 2.0 1.0 minor 60

# Issue a warning notification if the disk_monitor.sh script returns non-zero. 
# First N lines of the script's standard output (see the --max-stdout option)
# will be concatenated and the result is used as a notification text. After 
# the first non-zero is returned every different non-zero value returned by 
# disk_monitor.sh is considered a different error condition and a new warning 
# notification is produced. If the script finally returns 0, an "error clear"
# notification is issued.

exec diskmon disk_monitor.sh warning 0

USAGEINFO
}

# Function that reads options from command line and processes them

sub read_options {

  my($help, $version);
  my($priority);

  # set defaults for command line options

  $conffile = undef;
  $pidfile = undef;
  $interval = 60;
  $keepalive = 300;
  $keepalivemsg = "keepalive message";
  $notifyprog = undef;
  $normalsev = "normal";
  $maxstdout = 10;
  $appl = (split(/\//, $0))[-1];
  $priority = "user.err";
  $daemon = 0;

  # parse command line options

  GetOptions( "pid-file=s" => \$pidfile,
              "poll-interval=i" => \$interval,
              "keepalive-interval=i" => \$keepalive,
              "keepalive-message=s" => \$keepalivemsg,
              "notify-program=s" => \$notifyprog,
              "severity-for-normal=s" => \$normalsev,
              "max-stdout=i" => \$maxstdout,
              "program-name=s" => \$appl,
              "syslog=s" => \$priority,
              "daemon!" => \$daemon,
              "help|?" => \$help,
              "version" => \$version );

  # print version info, if requested

  if ($version) {
    print_version();
    exit(0);
  }
 
  # print usage info, if requested

  if ($help) {
    print_usage();
    exit(0);
  }
 
  # set the name of the configuration file, if given in command line,
  # otherwise print usage info

  if (defined($ARGV[0])) { 
    $conffile = $ARGV[0];
  } else {
    print_usage();
    exit(0);
  }

  # check numeric parameters for range errors
 
  if ($interval < 1) {
    print STDERR "Poll interval must be a positive integer\n"; 
    exit(1);
  }

  if ($keepalive < $interval) {
    print STDERR "Keepalive interval can not be smaller than poll interval\n";
    exit(1);
  }

  if ($maxstdout < 1) {
    print STDERR "# of stdout lines must be a positive integer\n"; 
    exit(1);
  }

  # split syslog-priority into facility and level

  ($facility, $level) = split(/\./, $priority);
}

# Function for logging internal errors of the program

sub logerr { 

  my($msg) = $_[0];

  if ($syslogavail) {
    $msg =~ s/%/%%/g;
    eval { Sys::Syslog::syslog($level, $msg) };
  }

  if (!$daemon) { print STDERR "$msg\n"; }
}

# Function will put the input value inside apostrophes

sub quote_value {
  $_[0] =~ s/'/'\\''/g;
  $_[0] = "'" . $_[0] . "'";
}

# Function for sending a notification

sub send_msg {

  my($object) = $_[0];
  my($severity) = $_[1];
  my($text) = $_[2];
  my($app) = $appl;
  my($cmdline, $ret);
  my(%variables);

  if (!defined($notifyprog)) {
    print "application=$app object=$object severity=$severity text=$text\n";
    return;
  }

  $variables{"a"} = quote_value($app);
  $variables{"o"} = quote_value($object);
  $variables{"s"} = quote_value($severity);
  $variables{"t"} = quote_value($text);

  $cmdline = $notifyprog;
  $cmdline =~ s/\%([aost])/$variables{$1}/g;

  $ret = system($cmdline);
  $ret = $ret >> 8;

  if ($ret) { logerr("Execution of '$cmdline' failed (exit=$ret)!"); }
}

# Functions for checking if the parameter is an integer or real number

sub is_integer {

  my($value) = $_[0];
  return !($value =~ tr/[0-9]//cd);
}

sub is_real {

  my($value) = $_[0];
  my($pos, $int, $frac);

  $pos = index($value, ".");

  if ($pos != -1) {
    $int = substr($value, 0, $pos);
    $frac = substr($value, $pos+1);
    if (is_integer($int) && is_integer($frac))  { return 1; }
  } else {
    if (is_integer($value))  { return 1; }
  }

  return 0;
}

# Function for reading in configuration

sub read_config {

  my($line, $len, $sev, $resend);
  my($ps, $min, $max);
  my($mountpoint, $is_prcnt, $is_prcnt2); 
  my($limit, $reset, $loadtype);
  my($label, $script);
  my(@comp);

  if (open(CONFFILE, "$conffile")) {

    %proc_limits = ();
    %disk_limits = ();
    %load_limits = ();
    @exec_scripts = ();

    while (<CONFFILE>) {

      # skip empty lines and comments
      if (/^\s*(.*\S)/)  { $line = $1; }  else  { next; }
      if (substr($line, 0, 1) eq '#')  { next; }

      # split the line into fields
      @comp = split(/\s+/, $line);

      $comp[0] = lc($comp[0]);
      $len = scalar(@comp);

      $sev = lc($comp[$len-2]);
      $resend = $comp[$len-1];

      if (!is_real($resend)) {
        logerr("Invalid resend value '$resend' in $conffile at line $.");
        next;
      }

      # handle the 'proc' directive

      if ($comp[0] eq "proc") {

        if ($len < 6) { 
          logerr("Invalid number of parameters in $conffile at line $.");
          next; 
        }

        $ps = join(" ", @comp[1..$len-5]);
        eval { "" =~ /$ps/; };
        if ($@) {
          logerr("Invalid regular expression '$ps' in $conffile at line $.");
          next; 
        }

        $min = $comp[$len-4];
        $max = $comp[$len-3];

        if (!is_integer($min) || ($max ne "-" &&  !is_integer($max))) {
          logerr("Minimum or maximum is not integer in $conffile at line $.");
          next; 
        }

        if ($max ne "-"  &&  $min > $max) { 
          logerr("Minimum exceeds maximum in $conffile at line $.");
          next; 
        }

        if (!exists($proc_limits{$ps}))  { $proc_limits{$ps} = []; }

        push @{$proc_limits{$ps}}, $min, $max, $sev, 0, $resend, 0;
      }

      # handle the 'disk' directive

      elsif ($comp[0] eq "disk") {

        if ($len < 6) { 
          logerr("Invalid number of parameters in $conffile at line $.");
          next; 
        }

        $mountpoint = $comp[1];

        if ($comp[2] =~ /^(\d+)(%?)$/) { 
          $limit = $1; 
          $is_prcnt = length($2); 
        } else { 
          logerr("Invalid limit value '$comp[2]' in $conffile at line $.");
          next; 
        }

        if ($comp[3] =~ /^(\d+)(%?)$/) { 
          $reset = $1; 
          $is_prcnt2 = length($2); 
        } else { 
          logerr("Invalid reset value '$comp[3]' in $conffile at line $.");
          next; 
        }

        if ($limit > $reset) { 
          logerr("Limit exceeds reset in $conffile at line $.");
          next; 
        }

        if (!exists($disk_limits{$mountpoint}))  
                  { $disk_limits{$mountpoint} = []; }

        push @{$disk_limits{$mountpoint}}, $is_prcnt, $limit, 
               $is_prcnt2, $reset, $sev, 0, $resend, 0;
      }

      # handle the 'load' directive

      elsif ($comp[0] eq "load") {

        if ($len < 6) {
          logerr("Invalid number of parameters in $conffile at line $.");
          next; 
        }

        if ($comp[1] eq "1" || $comp[1] eq "5" || $comp[1] eq "15") {
          $loadtype = $comp[1];
          $limit = $comp[2];
          $reset = $comp[3];
        } else {
          logerr("Invalid load type '$comp[1]' in $conffile at line $.");
          next; 
        }

        if (!is_real($limit) || !is_real($reset)) {
          logerr("Invalid limit or reset value in $conffile at line $.");
          next;
        }

        if ($limit < $reset) { 
          logerr("Reset exceeds limit in $conffile at line $.");
          next; 
        }

        if (!exists($load_limits{$loadtype}))  
                  { $load_limits{$loadtype} = []; }

        push @{$load_limits{$loadtype}}, $limit, $reset, $sev, 0, $resend, 0;
      }

      # handle the 'exec' directive

      elsif ($comp[0] eq "exec") {

        if ($len < 5) {
          logerr("Invalid number of parameters in $conffile at line $.");
          next; 
        }

        $label = $comp[1];
        $script = join(" ", @comp[2..$len-3]);

        push @exec_scripts, $label, $script, $sev, 0, $resend, 0;
      }

    }

    close(CONFFILE);

  } else {
    logerr("Can't open configuration file $conffile ($!)!");
    exit(1);
  }

}

# Function for process monitoring

sub check_proc {

  my(@processes, @comp, $process);
  my($exit_value, $index, $pos, $i, $len);
  my(@regexp, $key, $ref);
  my($min, $max, $sev, $status, $resend, $time);
  my($message, $newstatus);
  my(%ps_count) = ();

  # assign to @regexp all regular expressions that are used for searching
  # processes from the process table, and initialize %ps_count with zeros

  @regexp = keys(%proc_limits);
  foreach $key (@regexp) { $ps_count{$key} = 0; }

  # assign the content of the process table to @processes and set
  # the column which contains the full command line of each process

  @processes = `$pscmd`;

  $exit_value  = $? >> 8;

  if ($exit_value) {
    logerr("Command '$pscmd' failed (exit=$exit_value)!");
    return;
  }

  chomp(@processes);

  if ($cmdcol > 0)  { $index = $cmdcol - 1; }
  elsif ($cmdcol == 0)  { $index = undef; }
  else { $index = $cmdcol; }

  # search the process table and find the number of matches for each
  # regular expression

  $len = scalar(@processes);

  for ($i = 1; $i < $len; ++$i) {

    @comp = split(/\s+/, $processes[$i]);

    if (!defined($index))  { $process = $processes[$i]; }
    elsif (!length($comp[0]) && $index >= 0)  { $process = $comp[$index+1]; }
    else { $process = $comp[$index]; }

    foreach $key (@regexp) {
      if ($process =~ /$key/)  { ++$ps_count{$key}; } 
    }

  }

  # search the array of monitored processes and if the status of a process
  # has changed, generate a notification
 
  while(($key, $ref) = each(%proc_limits)) {

    $len = scalar(@{$ref});

    for ($i = 0; $i < $len; $i += 6) {

      $min = $ref->[$i];
      $max = $ref->[$i+1];
      $sev = $ref->[$i+2];
      $status = $ref->[$i+3];
      $resend = $ref->[$i+4];
      $time = $ref->[$i+5];

      if ($max ne "-"  &&  $ps_count{$key} > $max) {
        $message = sprintf("Too many '%s' running (\# = %d)",
        $key, $ps_count{$key});
        $newstatus = 1;
      }
      elsif ($ps_count{$key} < $min) {
        $message = sprintf("Too few '%s' running (\# = %d)",
        $key, $ps_count{$key});
        $newstatus = 1;
      }
      else {
        $message = sprintf("Process '%s' OK", $key);
        $newstatus = 0;
      }

      # if the current status and the previous status are different,
      # or if resend timer has expired, send the notification

      if ($status != $newstatus ||
          ($newstatus && $resend && time() - $time >= $resend)) {

        if ($newstatus) { send_msg($key, $sev, $message); }
          else { send_msg($key, $normalsev, $message); }

        $ref->[$i+5] = time();
      }

      $ref->[$i+3] = $newstatus;
    }
  }

}

# Function for file system monitoring

sub check_disk {

  my(@disks, @comp);
  my($exit_value, $mountpoint, $freepr, $freekb);
  my($ref, $i, $len, $j, $len2);
  my($is_prcnt, $is_prcnt2, $limit, $reset, $sev, $status);
  my($resend, $time, $message, $newstatus);

  # assign to @disks the current status of file systems

  @disks = `$dfcmd`;

  $exit_value  = $? >> 8;

  if ($exit_value) {
    logerr("Command '$dfcmd' failed (exit=$exit_value)!");
    return;
  }

  chomp(@disks);

  # search the file systems for systems having not enough free space

  $len = scalar(@disks);

  for ($i = 1; $i < $len; ++$i) {

    # split the $dfcmd output into fields and fetch the file system name,
    # the percentage of used space, and the amount of free space

    @comp = split(/\s+/, $disks[$i]);

    if (length($comp[0])) {
      $mountpoint = $comp[$mountpcol - 1];
      $freepr = $comp[$freeprcol - 1];
      $freekb = $comp[$freekbcol - 1];
    } else {
      $mountpoint = $comp[$mountpcol];
      $freepr = $comp[$freeprcol];
      $freekb = $comp[$freekbcol];
    }

    # convert the percentage of used space to the percentage of free space 

    $freepr =~ tr/%//d;
    $freepr = 100 - $freepr;

    # if the array of monitored file systems does not contain the current
    # file system but has the wildcard (*) record, add the file system to
    # the array of monitored file systems with the parameters of the
    # wildcard record

    if (!exists($disk_limits{$mountpoint})) {
      if (!exists($disk_limits{"*"}))  { next; }
      $disk_limits{$mountpoint} = [ @{$disk_limits{"*"}} ];
    }

    $ref = $disk_limits{$mountpoint};

    # search the array of monitored file systems and if the status of
    # a file system has changed, generate a notification
 
    $len2 = scalar(@{$ref});

    for ($j = 0; $j < $len2; $j += 8) {

      $is_prcnt = $ref->[$j];
      $limit = $ref->[$j+1];
      $is_prcnt2 = $ref->[$j+2];
      $reset = $ref->[$j+3];
      $sev = $ref->[$j+4];
      $status = $ref->[$j+5];
      $resend = $ref->[$j+6];
      $time = $ref->[$j+7];

      if ($is_prcnt  &&  $freepr <= $limit) {
        $message = sprintf("Filesystem %s: less than %d%% free (= %d%%)",
        $mountpoint, $limit, $freepr);
        $newstatus = 1;
      }
      elsif ($freekb <= $limit) {
        $message = sprintf("Filesystem %s: less than %dK free (= %dK)",
        $mountpoint, $limit, $freekb);
        $newstatus = 1;
      } 
      elsif ($is_prcnt2  &&  $freepr > $reset) {
        $message = sprintf("Filesystem %s OK", $mountpoint);
        $newstatus = 0;
      }
      elsif ($freekb > $reset) {
        $message = sprintf("Filesystem %s OK", $mountpoint);
        $newstatus = 0;
      }
      else { next; }

      # if the current status and the previous status are different,
      # or if resend timer has expired, send the notification

      if ($status != $newstatus ||
          ($newstatus && $resend && time() - $time >= $resend)) {

        if ($newstatus) { send_msg($mountpoint, $sev, $message); }
          else { send_msg($mountpoint, $normalsev, $message); }

        $ref->[$j+7] = time();
      }

      $ref->[$j+5] = $newstatus;
    }
  }

}

# Function for load average monitoring

sub check_load {

  my($exit_value, $load, $key, $ref, $len, $i);
  my($limit, $reset, $sev, $status);
  my($resend, $time, $newstatus, $message);
  my(%loads) = ();

  # assign to $load the current load average of the system

  $load = `$uptimecmd`;
  chomp($load);

  $exit_value  = $? >> 8;

  if ($exit_value) {
    logerr("Command '$uptimecmd' failed (exit=$exit_value)!");
    return;
  }

  if ($load !~ /load averages?:\s*([\d\.]+),\s*([\d\.]+),\s*([\d\.]+)/) {
    logerr("Can't parse '$uptimecmd' output!");
    exit(1);
  }

  $loads{"1"} = $1;
  $loads{"5"} = $2;
  $loads{"15"} = $3;

  # search the array of monitored load average types and if the status of 
  # the system wrt a load average type has changed, generate a notification

  while(($key, $ref) = each(%load_limits)) {

    $len = scalar(@{$ref});

    for ($i = 0; $i < $len; $i += 6) {

      $limit = $ref->[$i];
      $reset = $ref->[$i+1];
      $sev = $ref->[$i+2];
      $status = $ref->[$i+3];
      $resend = $ref->[$i+4];
      $time = $ref->[$i+5];

      if ($loads{$key} >= $limit) {
        $message = sprintf("%d min Load Average too high (= %.2f)",
        $key, $loads{$key});
        $newstatus = 1;
      } 
      elsif ($loads{$key} < $reset) {
        $message = sprintf("Load %d min OK", $key);
        $newstatus = 0;
      }
      else { next; }

      # if the current status and the previous status are different,
      # or if resend timer has expired, send the notification

      if ($status != $newstatus ||
          ($newstatus && $resend && time() - $time >= $resend)) {

        if ($newstatus) { send_msg("Load-$key", $sev, $message); }
          else { send_msg("Load-$key", $normalsev, $message); }

        $ref->[$i+5] = time();
      }

      $ref->[$i+3] = $newstatus;
    }
  }

}

# Function for system monitoring with custom scripts

sub check_exec {

  my($len, $i, @output, $index);
  my($label, $cmdline, $sev, $status);
  my($resend, $time, $newstatus, $message);  

  # execute custom scripts and fetch their output

  $len = scalar(@exec_scripts);

  for ($i = 0; $i < $len; $i += 6) {

    $label = $exec_scripts[$i];
    $cmdline = $exec_scripts[$i+1];
    $sev = $exec_scripts[$i+2];
    $status = $exec_scripts[$i+3];
    $resend = $exec_scripts[$i+4];
    $time = $exec_scripts[$i+5];

    @output = `$cmdline`;
    $newstatus = $? >> 8;

    # if the current status and the previous status are different,
    # or if resend timer has expired, send the notification

    if ($status != $newstatus ||
        ($newstatus && $resend && time() - $time >= $resend)) {

      $index = scalar(@output);
      if ($index > $maxstdout) { $index = $maxstdout; }

      chomp(@output);
      $message = join(" ", @output[0..$index-1]);

      if (!length($message)) { $message = "empty message"; }

      if ($newstatus) { send_msg($label, $sev, $message); }
        else { send_msg($label, $normalsev, $message); }

      $exec_scripts[$i+5] = time();
    }

    $exec_scripts[$i+3] = $newstatus;
  }
}

# Function for daemonization

sub daemonize {

  local $SIG{HUP} = 'IGNORE'; # ignore SIGHUP inside this function
  my($pid);

  # fork a new copy of the process and exit from the parent

  $pid = fork();

  if (!defined($pid)) {
    logerr("Can't fork a new process for daemonization ($!)!");
    exit(1);
  }

  if ($pid) { exit(0); }

  # create a new session and process group

  if (!POSIX::setsid()) {
    logerr("Can't start a new session ($!)!");
    exit(1);
  }

  # fork a second copy of the process and exit from the parent - the parent
  # as a session leader might deliver the SIGHUP signal to child when it 
  # exits, but SIGHUP is ignored inside this function

  $pid = fork();

  if (!defined($pid)) {
    logerr("Can't fork a new process for daemonization ($!)!");
    exit(1);
  }

  if ($pid) { exit(0); }

  # connect stdin, stdout, and stderr to /dev/null

  if (!open(STDIN, '/dev/null')) {
    logerr("Can't connect stdin to /dev/null ($!)!");
    exit(1);
  }

  if (!open(STDOUT, '>/dev/null')) {
    logerr("Can't connect stdout to /dev/null ($!)!");
    exit(1);
  }

  if (!open(STDERR, '>&STDOUT')) {
    logerr("Can't connect stderr to stdout with dup ($!)!"); 
    exit(1);
  }
}

# Functions for signal handling

sub hup_handler {
  $SIG{HUP} = \&hup_handler;
  $refresh = 1;
}

sub term_handler {
  $SIG{TERM} = \&term_handler;
  $terminate = 1;
}

# -------------------------------------------------------------

$SIG{HUP} = \&hup_handler;
$refresh = 0;

$SIG{TERM} = \&term_handler;
$terminate = 0;

read_options();

if ($daemon) { daemonize(); }

if ($syslogavail) { 
  eval { Sys::Syslog::openlog($appl, 'cons,pid', $facility) }; 
}

read_config();

$keepalivefactor = int($keepalive / $interval) - 1;
$keepalivecounter = 0;

if (defined($pidfile)) {
  if (open(PIDFILE, ">$pidfile")) {
    print PIDFILE $$;
    close(PIDFILE);
  } else {
    logerr("Can't open pid file $pidfile ($!)!");
  }
}

for (;;) {

  # start with a sleep - if the program was started at the system boot,
  # some of the monitored processes might not have been started yet

  if (!$terminate)  { sleep($interval); }

  if ($terminate) {
    logerr("Exiting on SIGTERM!");
    exit(0);
  }

  check_proc();
  check_disk();
  check_load();
  check_exec();

  if ($refresh) {

    if ($syslogavail) { 
      eval { Sys::Syslog::closelog() };
      eval { Sys::Syslog::openlog($appl, 'cons,pid', $facility) }; 
    }

    read_config();
    $refresh = 0;
  }

  if (!$keepalivecounter) {
    send_msg($keepalive, $normalsev, $keepalivemsg);
    $keepalivecounter = $keepalivefactor;
  } else { 
    --$keepalivecounter; 
  }

}

