lib/god/conditions/flapping.rb

   1 module God
   2   module Conditions
   3
   4     # Condition Symbol :flapping
   5     # Type: Trigger
   6     #
   7     # Trigger when a Task transitions to or from a state or states a given number
   8     # of times within a given period.
   9     #
  10     # Paramaters
  11     #   Required
  12     #     +times+ is the number of times that the Task must transition before
  13     #             triggering.
  14     #     +within+ is the number of seconds within which the Task must transition
  15     #              the specified number of times before triggering. You may use
  16     #              the sugar methods #seconds, #minutes, #hours, #days to clarify
  17     #              your code (see examples).
  18     #     --one or both of--
  19     #     +from_state+ is the state (as a Symbol) from which the transition must occur.
  20     #     +to_state is the state (as a Symbol) to which the transition must occur.
  21     #
  22     #   Optional:
  23     #     +retry_in+ is the number of seconds after which to re-monitor the Task after
  24     #                it has been disabled by the condition.
  25     #     +retry_times+ is the number of times after which to permanently unmonitor
  26     #                   the Task.
  27     #     +retry_within+ is the number of seconds within which
  28     #
  29     # Examples
  30     #
  31     # Trigger if
  32     class Flapping < TriggerCondition
  33       attr_accessor :times,
  34                     :within,
  35                     :from_state,
  36                     :to_state,
  37                     :retry_in,
  38                     :retry_times,
  39                     :retry_within
  40
  41       def initialize
  42         self.info = "process is flapping"
  43       end
  44
  45       def prepare
  46         @timeline = Timeline.new(self.times)
  47         @retry_timeline = Timeline.new(self.retry_times)
  48       end
  49
  50       def valid?
  51         valid = true
  52         valid &= complain("Attribute 'times' must be specified", self) if self.times.nil?
  53         valid &= complain("Attribute 'within' must be specified", self) if self.within.nil?
  54         valid &= complain("Attributes 'from_state', 'to_state', or both must be specified", self) if self.from_state.nil? && self.to_state.nil?
  55         valid
  56       end
  57
  58       def process(event, payload)
  59         begin
  60           if event == :state_change
  61             event_from_state, event_to_state = *payload
  62
  63             from_state_match = !self.from_state || self.from_state && Array(self.from_state).include?(event_from_state)
  64             to_state_match = !self.to_state || self.to_state && Array(self.to_state).include?(event_to_state)
  65
  66             if from_state_match && to_state_match
  67               @timeline << Time.now
  68
  69               concensus = (@timeline.size == self.times)
  70               duration = (@timeline.last - @timeline.first) < self.within
  71
  72               if concensus && duration
  73                 @timeline.clear
  74                 trigger
  75                 retry_mechanism
  76               end
  77             end
  78           end
  79         rescue => e
  80           puts e.message
  81           puts e.backtrace.join("\n")
  82         end
  83       end
  84
  85       private
  86
  87       def retry_mechanism
  88         if self.retry_in
  89           @retry_timeline << Time.now
  90
  91           concensus = (@retry_timeline.size == self.retry_times)
  92           duration = (@retry_timeline.last - @retry_timeline.first) < self.retry_within
  93
  94           if concensus && duration
  95             # give up
  96             Thread.new do
  97               sleep 1
  98
  99               # log
 100               msg = "#{self.watch.name} giving up"
 101               applog(self.watch, :info, msg)
 102             end
 103           else
 104             # try again later
 105             Thread.new do
 106               sleep 1
 107
 108               # log
 109               msg = "#{self.watch.name} auto-reenable monitoring in #{self.retry_in} seconds"
 110               applog(self.watch, :info, msg)
 111
 112               sleep self.retry_in
 113
 114               # log
 115               msg = "#{self.watch.name} auto-reenabling monitoring"
 116               applog(self.watch, :info, msg)
 117
 118               if self.watch.state == :unmonitored
 119                 self.watch.monitor
 120               end
 121             end
 122           end
 123         end
 124       end
 125     end
 126
 127   end
 128 end