bundles/icinga2: send notifications

bundles/icinga2/files/icinga2/groups.conf

@@ -13,3 +13,15 @@ object ServiceGroup "bundle-${bundle}" {
   assign where service.vars.bundle == "${bundle}"
 }
 % endfor
+
+object ServiceGroup "checks_with_mail" {
+  display_name = "Checks which send E-Mails"
+
+  assign where service.vars.notification.mail == true
+}
+
+object ServiceGroup "checks_with_sms" {
+  display_name = "Checks which send SMS"
+
+  assign where service.vars.notification.sms == true
+}

bundles/icinga2/files/icinga2/hosts.conf

@@ -9,6 +9,8 @@ object Host "${monitored_node.name}" {
   vars.period = "${sla_info[monitored_node.metadata.get('sla', '24x7')]}"
   vars.location = "${monitored_node.metadata.get('location', 'unknown')}"
   vars.bw_groups = [ "${'", "'.join(sorted({group.name for group in monitored_node.groups}))}" ]
+  vars.notification.sms = ${str(monitored_node.metadata.get('icinga_options', {}).get('vars.notification.sms', True)).lower()}
+  vars.notification.mail = true
 }
 % endfor
 

bundles/icinga2/files/icinga2/notification_commands.conf

@@ -0,0 +1,74 @@
+object NotificationCommand "send-host-notification" {
+  import "plugin-notification-command"
+
+  command = [ ConfigDir + "/scripts/icinga_notification_wrapper" ]
+
+  arguments = {
+    "--mail" = {
+      set_if = {{
+        host_vars = host.vars
+        if (host_vars.notification.mail == true) {
+          return true
+        } else {
+          return false
+        }
+      }}
+      value = "$user.email$"
+    }
+    "--sms" = {
+      set_if = {{
+        host_vars = host.vars
+        if (host_vars.notification.sms == true) {
+          return true
+        } else {
+          return false
+        }
+      }}
+      value = "$user.vars.mobile$"
+    }
+    "--host_name" = "$host.display_name$"
+    "--state" = "$host.state$"
+    "--output" = "$host.output$"
+    "--escalation" = {
+      set_if = "$escalation$"
+    }
+  }
+}
+
+object NotificationCommand "send-service-notification" {
+  import "plugin-notification-command"
+
+  command = [ ConfigDir + "/scripts/icinga_notification_wrapper" ]
+
+  arguments = {
+    "--mail" = {
+      set_if = {{
+        svc_vars = service.vars
+        if (svc_vars.notification.mail == true) {
+          return true
+        } else {
+          return false
+        }
+      }}
+      value = "$user.email$"
+    }
+    "--sms" = {
+      set_if = {{
+        svc_vars = service.vars
+        if (svc_vars.notification.sms == true) {
+          return true
+        } else {
+          return false
+        }
+      }}
+      value = "$user.vars.mobile$"
+    }
+    "--host_name" = "$host.display_name$"
+    "--service_name" = "$service.display_name$"
+    "--state" = "$service.state$"
+    "--output" = "$service.output$"
+    "--escalation" = {
+      set_if = "$escalation$"
+    }
+  }
+}

bundles/icinga2/files/icinga2/notifications.conf

@@ -0,0 +1,61 @@
+apply Notification "notify_host-notification" to Host {
+  import "host-notification"
+  interval = 30m
+
+  times = {
+    begin = 1m
+    end = 30m
+  }
+
+  assign where host.vars.notification.sms == true
+}
+
+
+apply Notification "notify_service-notification" to Service {
+  import "service-notification"
+  interval = 30m
+
+  times = {
+    begin = 1m
+    end = 30m
+  }
+
+  assign where service.vars.notification.sms == true
+}
+
+
+apply Notification "notify_host-notification-escalation" to Host {
+  import "host-notification"
+  interval = 20m
+
+  times = {
+    begin = 30m
+  }
+
+  vars.escalation = true
+
+  assign where host.vars.notification.sms == true
+}
+
+
+apply Notification "notify_service-notification-escalation" to Service {
+  import "service-notification"
+  interval = 20m
+
+  times = {
+    begin = 30m
+  }
+
+  vars.escalation = true
+
+  assign where service.vars.notification.sms == true
+}
+
+
+apply Notification "notify_service-notification-mail" to Service {
+  import "service-notification"
+  interval = 0
+
+  assign where service.vars.notification.mail == true
+  ignore where service.vars.notification.sms == true
+}

bundles/icinga2/files/icinga2/templates.conf

@@ -7,9 +7,6 @@ template Host "generic-host" {
     enable_event_handler = true
     enable_flapping = false
     enable_perfdata = false
-    vars.notification.sms = true
-    vars.notification_type = "sms"
-
 }
 
 template Host "host-active" {
@@ -36,7 +33,6 @@ template Service "generic-service" {
     max_check_attempts = 4
     check_interval = 5m
     retry_interval = 2m
-    vars.notification.sms = true
     enable_notifications = true
     enable_event_handler = true
     enable_flapping = false
@@ -63,8 +59,6 @@ template Notification "host-notification" {
     types = [ Problem, Recovery, Custom ]
     user_groups = [ "on-call_sms" ]
     period = host.vars.period
-
-    vars.notification_type = "sms"
 }
 
 template Notification "service-notification" {
@@ -73,8 +67,6 @@ template Notification "service-notification" {
     types = [ Problem, Recovery, Custom ]
     user_groups = [ "on-call_sms" ]
 
-    vars.notification_type = "sms"
-
     if(service.vars.period) {
       period = service.vars.period
     } else {

bundles/icinga2/files/icinga2/users.conf

@@ -7,7 +7,7 @@ object User "${username}" {
   display_name = "${username}"
   enable_notifications = true
   period = "24x7"
-  states = [ OK, Critical, Up, Down ]
+  states = [ OK, Warning, Critical, Up, Down ]
   types = [ Problem, Recovery ]
 
 % if config['is_admin']:

bundles/icinga2/files/scripts/icinga_notification_wrapper

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+
+import email.mime.text
+import smtplib
+from argparse import ArgumentParser
+from requests import get
+from subprocess import run
+from sys import argv
+
+parser = ArgumentParser(
+    prog='icinga_notification_wrapper',
+    description='Icinga2 Notification Wrapper',
+)
+parser.add_argument(
+    '--host_name',
+    type=str,
+    required=True,
+)
+parser.add_argument(
+    '--service_name',
+    type=str,
+)
+parser.add_argument(
+    '--output',
+    type=str,
+    required=True,
+)
+parser.add_argument(
+    '--state',
+    type=str,
+    required=True,
+)
+parser.add_argument(
+    '--sms',
+    type=str,
+)
+parser.add_argument(
+    '--mail',
+    type=str,
+)
+parser.add_argument(
+    '--escalation',
+    action='store_true',
+)
+
+def log_to_syslog(message):
+    try:
+        msg = '{}/{}: {}'.format(args.host_name, args.service_name, message)
+        run(['logger', '-t', 'icinga_notification_wrapper', msg])
+    except:
+        # We don't expect this to fail. However, *if* it fails, we don't
+        # want it to get in the way of other notifications we may have
+        # to send.
+        pass
+
+
+def notify_per_sms():
+    log_to_syslog('SMS requested, but not implemented yet!') # FIXME TODO
+    return
+    msg = 'ICINGA: {host}/{service} is {state}: {output}'.format(
+        host=args.host_name,
+        service=args.service_name,
+        state=args.state,
+        output=args.output
+    )
+
+
+def notify_per_mail():
+    text = """
+    _      _
+   (_)____(_)___  ____ _____ _
+  / / ___/ / __ \/ __ `/ __ `/
+ / / /__/ / / / / /_/ / /_/ /
+/_/\___/_/_/ /_/\__, /\__,_/
+               /____/
+
+Host: {host}"""
+
+    if args.service_name:
+        text += """
+Service: {service}"""
+
+    text += """
+State: {state}
+
+{output}"""
+
+    mail = email.mime.text.MIMEText(text.format(
+            host=args.host_name,
+            service=args.service_name,
+            state=args.state,
+            output=args.output
+        ),
+        'plain',
+        'utf-8',
+    )
+
+    if args.service_name:
+        mail['Subject'] = '[ICINGA] {}/{} is {}'.format(args.host_name, args.service_name, args.state)
+    else:
+        mail['Subject'] = '[ICINGA] {} is {}'.format(args.host_name, args.state)
+
+    mail['To'] = args.mail
+    mail['From'] = 'noreply+icinga@kunbox.net'
+
+    try:
+        s = smtplib.SMTP('localhost')
+        s.sendmail(mail['From'], [args.mail], mail.as_string())
+        s.quit()
+
+        log_to_syslog('Sent mail to "{}"'.format(args.mail))
+    except Exception as e:
+        log_to_syslog('Sending mail to "{}" failed: {}'.format(args.mail, repr(e)))
+
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+
+    log_to_syslog(' '.join(argv))
+
+    if args.mail and not args.escalation:
+        notify_per_mail()
+
+    if args.sms:
+        notify_per_sms()

bundles/icinga2/items.py

@@ -91,6 +91,10 @@ files = {
             'svc_systemd:icinga2:restart',
         },
     },
+    '/etc/icinga2/scripts/icinga_notification_wrapper': {
+        'source': 'scripts/icinga_notification_wrapper',
+        'mode': '0755',
+    },
     '/etc/icinga2/features-available/ido-pgsql.conf': {
         'source': 'icinga2/ido-pgsql.conf',
         'content_type': 'mako',
@@ -129,6 +133,24 @@ files = {
             'svc_systemd:icinga2:restart',
         },
     },
+    '/etc/icinga2/conf.d/notification_commands.conf': {
+        'source': 'icinga2/notification_commands.conf',
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
+    '/etc/icinga2/conf.d/notifications.conf': {
+        'source': 'icinga2/notifications.conf',
+        'needs': {
+            'pkg_apt:icinga2',
+        },
+        'triggers': {
+            'svc_systemd:icinga2:restart',
+        },
+    },
     '/etc/icinga2/conf.d/templates.conf': {
         'source': 'icinga2/templates.conf',
         'needs': {

bundles/nginx/metadata.py

@@ -106,12 +106,14 @@ def monitoring(metadata):
                 'check_command': 'check_http_wget',
                 'vars.http_wget_contains': vconfig['website_check_string'],
                 'vars.http_wget_url': '{}://{}{}'.format(scheme, domain, vconfig['website_check_path']),
+                'vars.notification.sms': True,
             }
 
         if vconfig.get('check_ssl', False):
             services['NGINX VHOST {} CERTIFICATE'.format(vname)] = {
                 'check_command': 'check_vhost_https_cert_at_url',
                 'vars.domain': domain,
+                'vars.notification.mail': True,
             }
 
     max_connections = metadata.get('nginx/worker_connections') * metadata.get('nginx/worker_processes')

bundles/postfix/metadata.py

@@ -28,6 +28,7 @@ if node.has_bundle('postfixadmin'):
     defaults['icinga2_api']['postfix']['services'].update({
         'SMTP CONNECT': {
             'check_command': 'check_smtp',
+            'vars.notification.sms': True,
         },
         'SMTP SUBMISSION CONNECT': {
             'check_command': 'check_smtp',

bundles/powerdns/metadata.py

@@ -14,6 +14,7 @@ defaults = {
             'services': {
                 'POWERDNS PROCESS': {
                     'command_on_monitored_host': '/usr/lib/nagios/plugins/check_procs -C pdns_server -c 1:',
+                    'vars.notification.mail': True,
                 },
             },
         },

bundles/rspamd/items.py

@@ -65,7 +65,6 @@ actions = {
     },
 }
 
-# TODO manage this using bundlewrap
 if 'dkim' in node.metadata.get('rspamd', {}):
     for i in {'arc', 'dkim_signing'}:
         files[f'/etc/rspamd/local.d/{i}.conf'] = {

bundles/sshmon/metadata.py

@@ -96,6 +96,7 @@ def default_checks(metadata):
                             '-A -I "^/dev$" -I "^/run" -I "^/sys" -i "/sys/kernel/debug/tracing" '
                             f'{disk_space_ignore_patterns_string}',
                         ),
+                        'vars.notification.mail': True,
                     },
                     'MOUNTS': {
                         'command_on_monitored_host': f'sudo /usr/local/share/icinga/plugins/check_mounts {mounts_options}',

bundles/vmhost/metadata.py

@@ -12,6 +12,7 @@ defaults = {
             'services': {
                 'QEMU VM STATUS': {
                     'command_on_monitored_host': 'sudo /usr/local/share/icinga/plugins/check_vm_status',
+                    'vars.notification.mail': True,
                 },
             },
         },

bundles/zfs/items.py

@@ -122,17 +122,3 @@ directories = {
 #
 #else:
 #    files["/mnt/zfs-snapshot-backup"] = {'delete': True}
-
-# TODO when we start using telegraf
-#if node.has_bundle('telegraf'):
-#    files['/etc/telegraf-zfs-dataset.conf'] = {
-#        'content': dumps(
-#            node.metadata.get('zfs', {}),
-#            cls=MetadataJSONEncoder,
-#            indent=4,
-#            sort_keys=True,
-#        ) + '\n',
-#    }
-#    files['/usr/local/bin/telegraf-zfs-dataset'] = {
-#        'mode': '0775',
-#    }

bundles/zfs/metadata.py

@@ -76,6 +76,7 @@ if node.has_bundle('sshmon'):
                     },
                     'ZFS MOUNTED VOLUMES': {
                         'command_on_monitored_host': 'sudo /usr/local/share/icinga/plugins/check_zfs_volumes',
+                        'vars.notification.mail': True,
                     },
                     'ZFS OLD SNAPSHOTS': {
                         'command_on_monitored_host': 'sudo /usr/local/share/icinga/plugins/check_zfs_old_snapshots',
@@ -147,10 +148,12 @@ def monitoring(metadata):
     for poolname, pool_options in metadata.get('zfs/pools').items():
         services['ZFS ZPOOL ONLINE {}'.format(poolname)] = {
             'command_on_monitored_host': 'sudo /usr/local/share/icinga/plugins/check_zpool_online {}'.format(poolname),
+            'vars.notification.mail': True,
         }
 
         services['ZFS ZPOOL SPACE ' + poolname] = {
-            'command_on_monitored_host': 'sudo /usr/local/share/icinga/plugins/check_zpool_space {} 90'.format(poolname)
+            'command_on_monitored_host': 'sudo /usr/local/share/icinga/plugins/check_zpool_space {} 90'.format(poolname),
+            'vars.notification.mail': True,
         }
 
     return {

nodes/htz-cloud/sewfile.py

@@ -10,6 +10,9 @@ nodes['htz-cloud.sewfile'] = {
         'webserver',
     },
     'metadata': {
+        'icinga_options': {
+            'vars.notification.sms': False,
+        },
         'interfaces': {
             'eth0': {
                 'ips': {

nodes/htz/ex42-1048908.py

@@ -41,9 +41,7 @@ nodes['htz.ex42-1048908'] = {
                 'redis': {},
 
                 # No need to create a bundle just to install packages,
-                # configs will be managed by users nevertheless. Maybe
-                # this will be a FIXME once we start managing backups
-                # via bundlewrap.
+                # configs will be managed by users nevertheless.
                 'weechat': {},
                 'weechat-core': {},
                 'weechat-curses': {},