lib/god/hub.rb

   1 module God
   2
   3   class Hub
   4     class << self
   5       # directory to hold conditions and their corresponding metric
   6       # {condition => metric}
   7       attr_accessor :directory
   8     end
   9
  10     self.directory = {}
  11
  12     def self.attach(condition, metric)
  13       self.directory[condition] = metric
  14       condition.reset
  15
  16       case condition
  17         when PollCondition
  18           Timer.get.schedule(condition, 0)
  19         when EventCondition, TriggerCondition
  20           condition.register
  21       end
  22     end
  23
  24     def self.detach(condition)
  25       self.directory.delete(condition)
  26
  27       case condition
  28         when PollCondition
  29           Timer.get.unschedule(condition)
  30         when EventCondition, TriggerCondition
  31           condition.deregister
  32       end
  33     end
  34
  35     def self.trigger(condition)
  36       case condition
  37         when PollCondition
  38           self.handle_poll(condition)
  39         when EventCondition, TriggerCondition
  40           self.handle_event(condition)
  41       end
  42     end
  43
  44     def self.handle_poll(condition)
  45       Thread.new do
  46         begin
  47           metric = self.directory[condition]
  48
  49           # it's possible that the timer will trigger an event before it can be cleared
  50           # by an exiting metric, in which case it should be ignored
  51           unless metric.nil?
  52             watch = metric.watch
  53
  54             watch.mutex.synchronize do
  55               # run the test
  56               result = condition.test
  57
  58               # log
  59               messages = self.log(watch, metric, condition, result)
  60
  61               # notify
  62               if condition.notify && self.trigger?(metric, result)
  63                 self.notify(condition, messages.last)
  64               end
  65
  66               # after-condition
  67               condition.after
  68
  69               # get the destination
  70               dest =
  71               if result && condition.transition
  72                 # condition override
  73                 condition.transition
  74               else
  75                 # regular
  76                 metric.destination && metric.destination[result]
  77               end
  78
  79               # transition or reschedule
  80               if dest
  81                 # transition
  82                 begin
  83                   watch.move(dest)
  84                 rescue EventRegistrationFailedError
  85                   msg = watch.name + ' Event registration failed, moving back to previous state'
  86                   applog(watch, :info, msg)
  87
  88                   dest = watch.state
  89                   retry
  90                 end
  91               else
  92                 # reschedule
  93                 Timer.get.schedule(condition)
  94               end
  95             end
  96           end
  97         rescue Exception => e
  98           message = format("Unhandled exception (%s): %s\n%s",
  99                            e.class, e.message, e.backtrace.join("\n"))
 100           applog(nil, :fatal, message)
 101         end
 102       end
 103     end
 104
 105     def self.handle_event(condition)
 106       Thread.new do
 107         begin
 108           metric = self.directory[condition]
 109
 110           unless metric.nil?
 111             watch = metric.watch
 112
 113             watch.mutex.synchronize do
 114               # log
 115               messages = self.log(watch, metric, condition, true)
 116
 117               # notify
 118               if condition.notify && self.trigger?(metric, true)
 119                 self.notify(condition, messages.last)
 120               end
 121
 122               # get the destination
 123               dest =
 124               if condition.transition
 125                 # condition override
 126                 condition.transition
 127               else
 128                 # regular
 129                 metric.destination && metric.destination[true]
 130               end
 131
 132               if dest
 133                 watch.move(dest)
 134               end
 135             end
 136           end
 137         rescue Exception => e
 138           message = format("Unhandled exception (%s): %s\n%s",
 139                            e.class, e.message, e.backtrace.join("\n"))
 140           applog(nil, :fatal, message)
 141         end
 142       end
 143     end
 144
 145     # helpers
 146
 147     def self.trigger?(metric, result)
 148       (metric.destination && metric.destination.keys.size == 2) || result == true
 149     end
 150
 151     def self.log(watch, metric, condition, result)
 152       status =
 153       if self.trigger?(metric, result)
 154         "[trigger]"
 155       else
 156         "[ok]"
 157       end
 158
 159       messages = []
 160
 161       # log info if available
 162       if condition.info
 163         Array(condition.info).each do |condition_info|
 164           messages << "#{watch.name} #{status} #{condition_info} (#{condition.base_name})"
 165           applog(watch, :info, messages.last)
 166         end
 167       else
 168         messages << "#{watch.name} #{status} (#{condition.base_name})"
 169         applog(watch, :info, messages.last)
 170       end
 171
 172       # log
 173       debug_message = watch.name + ' ' + condition.base_name + " [#{result}] " + self.dest_desc(metric, condition)
 174       applog(watch, :debug, debug_message)
 175
 176       messages
 177     end
 178
 179     def self.dest_desc(metric, condition)
 180       if condition.transition
 181         {true => condition.transition}.inspect
 182       else
 183         if metric.destination
 184           metric.destination.inspect
 185         else
 186           'none'
 187         end
 188       end
 189     end
 190
 191     def self.notify(condition, message)
 192       spec = Contact.normalize(condition.notify)
 193       unmatched = []
 194
 195       # resolve contacts
 196       resolved_contacts =
 197       spec[:contacts].inject([]) do |acc, contact_name_or_group|
 198         cons = Array(God.contacts[contact_name_or_group] || God.contact_groups[contact_name_or_group])
 199         unmatched << contact_name_or_group if cons.empty?
 200         acc += cons
 201         acc
 202       end
 203
 204       # warn about unmatched contacts
 205       unless unmatched.empty?
 206         msg = "#{condition.watch.name} no matching contacts for '#{unmatched.join(", ")}'"
 207         applog(condition.watch, :warn, msg)
 208       end
 209
 210       # notify each contact
 211       resolved_contacts.each do |c|
 212         host = `hostname`.chomp rescue 'none'
 213         c.notify(message, Time.now, spec[:priority], spec[:category], host)
 214
 215         msg = "#{condition.watch.name} #{c.info ? c.info : "notification sent for contact: #{c.name}"} (#{c.base_name})"
 216
 217         applog(condition.watch, :info, msg % [])
 218       end
 219     end
 220   end
 221
 222 end