From b0320b4390f91d673b317d6af191b548c4350ceb Mon Sep 17 00:00:00 2001 From: tom Date: Wed, 2 Jan 2008 17:22:31 -0800 Subject: [PATCH] prevent carry-over conditions --- History.txt | 3 +- lib/god/condition.rb | 2 +- lib/god/conditions/cpu_usage.rb | 12 +-- lib/god/conditions/memory_usage.rb | 12 +-- lib/god/conditions/process_exits.rb | 14 ++-- lib/god/conditions/process_running.rb | 22 +++-- lib/god/hub.rb | 133 ++++++++++++++++-------------- lib/god/process.rb | 13 +++ lib/god/task.rb | 5 +- lib/god/timer.rb | 21 +++-- lib/god/watch.rb | 2 +- test/configs/child_polls/simple_server.rb | 2 +- test/test_conditions_process_running.rb | 10 ++- test/test_hub.rb | 47 +++++++---- test/test_process.rb | 28 +++++++ test/test_timer.rb | 28 ++++--- 16 files changed, 223 insertions(+), 131 deletions(-) diff --git a/History.txt b/History.txt index 863ab62..4c503dd 100644 --- a/History.txt +++ b/History.txt @@ -1,7 +1,8 @@ == 0.6.4 / * Bug Fixes * Refactor Hub to clarify mutexing - * Eliminated potential iteration problem in Timer + * Eliminate potential iteration problem in Timer + * Add caching PID accessor to process to solve event deregistration failure == 0.6.3 / 2007-12-18 * Minor Enhancements diff --git a/lib/god/condition.rb b/lib/god/condition.rb index e56d962..3305099 100644 --- a/lib/god/condition.rb +++ b/lib/god/condition.rb @@ -1,7 +1,7 @@ module God class Condition < Behavior - attr_accessor :transition, :notify, :info + attr_accessor :transition, :notify, :info, :phase # Generate a Condition of the given kind. The proper class if found by camel casing the # kind (which is given as an underscored symbol). diff --git a/lib/god/conditions/cpu_usage.rb b/lib/god/conditions/cpu_usage.rb index f87ae5d..3057502 100644 --- a/lib/god/conditions/cpu_usage.rb +++ b/lib/god/conditions/cpu_usage.rb @@ -29,7 +29,7 @@ module God # c.pid_file = "/var/run/mongrel.3000.pid" # end class CpuUsage < PollCondition - attr_accessor :above, :times + attr_accessor :above, :times, :pid_file def initialize super @@ -49,17 +49,19 @@ module God @timeline.clear end + def pid + self.watch.pid || File.read(self.pid_file).strip.to_i + end + def valid? valid = true - valid &= complain("Attribute 'pid_file' must be specified", self) if self.watch.pid_file.nil? + valid &= complain("Attribute 'pid_file' must be specified", self) if self.watch.pid_file.nil? && self.pid_file.nil? valid &= complain("Attribute 'above' must be specified", self) if self.above.nil? valid end def test - return false unless File.exist?(self.watch.pid_file) - - pid = File.read(self.watch.pid_file).strip + pid = self.watch.pid process = System::Process.new(pid) @timeline.push(process.percent_cpu) diff --git a/lib/god/conditions/memory_usage.rb b/lib/god/conditions/memory_usage.rb index 4facc75..b5f8075 100644 --- a/lib/god/conditions/memory_usage.rb +++ b/lib/god/conditions/memory_usage.rb @@ -31,7 +31,7 @@ module God # c.pid_file = "/var/run/mongrel.3000.pid" # end class MemoryUsage < PollCondition - attr_accessor :above, :times + attr_accessor :above, :times, :pid_file def initialize super @@ -51,17 +51,19 @@ module God @timeline.clear end + def pid + self.watch.pid || File.read(self.pid_file).strip.to_i + end + def valid? valid = true - valid &= complain("Attribute 'pid_file' must be specified", self) if self.watch.pid_file.nil? + valid &= complain("Attribute 'pid_file' must be specified", self) if self.watch.pid_file.nil? && self.pid_file.nil? valid &= complain("Attribute 'above' must be specified", self) if self.above.nil? valid end def test - return false unless File.exist?(self.watch.pid_file) - - pid = File.read(self.watch.pid_file).strip + pid = self.pid process = System::Process.new(pid) @timeline.push(process.memory) diff --git a/lib/god/conditions/process_exits.rb b/lib/god/conditions/process_exits.rb index be955e2..a48cb5e 100644 --- a/lib/god/conditions/process_exits.rb +++ b/lib/god/conditions/process_exits.rb @@ -28,13 +28,11 @@ module God end def valid? - valid = true - valid &= complain("Attribute 'pid_file' must be specified", self) if self.watch.pid_file.nil? - valid + true end - + def register - pid = File.read(self.watch.pid_file).strip.to_i + pid = self.watch.pid begin EventHandler.register(pid, :proc_exit) do |extra| @@ -51,14 +49,14 @@ module God end def deregister - if File.exist?(self.watch.pid_file) - pid = File.read(self.watch.pid_file).strip.to_i + pid = self.watch.pid + if pid EventHandler.deregister(pid, :proc_exit) msg = "#{self.watch.name} deregistered 'proc_exit' event for pid #{pid}" applog(self.watch, :info, msg) else - applog(self.watch, :error, "#{self.watch.name} could not deregister: no such PID file #{self.watch.pid_file} (#{self.base_name})") + applog(self.watch, :error, "#{self.watch.name} could not deregister: no cached PID or PID file #{self.watch.pid_file} (#{self.base_name})") end end end diff --git a/lib/god/conditions/process_running.rb b/lib/god/conditions/process_running.rb index d500df0..ecc6f69 100644 --- a/lib/god/conditions/process_running.rb +++ b/lib/god/conditions/process_running.rb @@ -34,25 +34,29 @@ module God # c.pid_file = "/var/run/mongrel.3000.pid" # end class ProcessRunning < PollCondition - attr_accessor :running + attr_accessor :running, :pid_file + + def pid + self.watch.pid || File.read(self.pid_file).strip.to_i + end def valid? valid = true - valid &= complain("Attribute 'pid_file' must be specified", self) if self.watch.pid_file.nil? + valid &= complain("Attribute 'pid_file' must be specified", self) if self.watch.pid_file.nil? && self.pid_file.nil? valid &= complain("Attribute 'running' must be specified", self) if self.running.nil? valid end - + def test self.info = [] - unless File.exist?(self.watch.pid_file) - self.info << "#{self.watch.name} #{self.class.name}: no such pid file: #{self.watch.pid_file}" - return !self.running - end + # unless File.exist?(self.watch.pid_file) + # self.info << "#{self.watch.name} #{self.class.name}: no such pid file: #{self.watch.pid_file}" + # return !self.running + # end - pid = File.read(self.watch.pid_file).strip - active = System::Process.new(pid).exists? + pid = self.watch.pid + active = pid && System::Process.new(pid).exists? if (self.running && active) self.info << "process is running" diff --git a/lib/god/hub.rb b/lib/god/hub.rb index 13ec325..382d40d 100644 --- a/lib/god/hub.rb +++ b/lib/god/hub.rb @@ -20,8 +20,9 @@ module God # Returns nothing def self.attach(condition, metric) self.mutex.synchronize do - self.directory[condition] = metric + condition.phase = condition.watch.phase condition.reset + self.directory[condition] = metric case condition when PollCondition @@ -51,13 +52,14 @@ module God # Trigger evaluation of the condition # +condition+ is the Condition to evaluate + # +phase+ is the phase of the Watch at the time the condition was scheduled # # Returns nothing - def self.trigger(condition) + def self.trigger(condition, phase = nil) self.mutex.synchronize do case condition when PollCondition - self.handle_poll(condition) + self.handle_poll(condition, phase) when EventCondition, TriggerCondition self.handle_event(condition) end @@ -69,9 +71,10 @@ module God # Asynchronously evaluate and handle the given poll condition. Handles logging # notifications, and moving to the new state if necessary # +condition+ is the Condition to handle + # +phase+ is the phase of the Watch that should be matched # # Returns nothing - def self.handle_poll(condition) + def self.handle_poll(condition, phase) metric = self.directory[condition] # it's possible that the timer will trigger an event before it can be cleared @@ -83,45 +86,48 @@ module God watch = metric.watch watch.mutex.synchronize do - # run the test - result = condition.test - - # log - messages = self.log(watch, metric, condition, result) - - # notify - if condition.notify && self.trigger?(metric, result) - self.notify(condition, messages.last) - end - - # after-condition - condition.after - - # get the destination - dest = - if result && condition.transition - # condition override - condition.transition - else - # regular - metric.destination && metric.destination[result] - end - - # transition or reschedule - if dest - # transition - begin - watch.move(dest) - rescue EventRegistrationFailedError - msg = watch.name + ' Event registration failed, moving back to previous state' - applog(watch, :info, msg) - - dest = watch.state - retry + # ensure this condition is still active when we finally get the mutex + if self.directory[condition] && phase == watch.phase + # run the test + result = condition.test + + # log + messages = self.log(watch, metric, condition, result) + + # notify + if condition.notify && self.trigger?(metric, result) + self.notify(condition, messages.last) + end + + # after-condition + condition.after + + # get the destination + dest = + if result && condition.transition + # condition override + condition.transition + else + # regular + metric.destination && metric.destination[result] + end + + # transition or reschedule + if dest + # transition + begin + watch.move(dest) + rescue EventRegistrationFailedError + msg = watch.name + ' Event registration failed, moving back to previous state' + applog(watch, :info, msg) + + dest = watch.state + retry + end + else + # reschedule + Timer.get.schedule(condition) end - else - # reschedule - Timer.get.schedule(condition) end end rescue Exception => e @@ -149,26 +155,29 @@ module God watch = metric.watch watch.mutex.synchronize do - # log - messages = self.log(watch, metric, condition, true) - - # notify - if condition.notify && self.trigger?(metric, true) - self.notify(condition, messages.last) - end - - # get the destination - dest = - if condition.transition - # condition override - condition.transition - else - # regular - metric.destination && metric.destination[true] - end - - if dest - watch.move(dest) + # ensure this condition is still active when we finally get the mutex + if self.directory[condition] + # log + messages = self.log(watch, metric, condition, true) + + # notify + if condition.notify && self.trigger?(metric, true) + self.notify(condition, messages.last) + end + + # get the destination + dest = + if condition.transition + # condition override + condition.transition + else + # regular + metric.destination && metric.destination[true] + end + + if dest + watch.move(dest) + end end end rescue Exception => e diff --git a/lib/god/process.rb b/lib/god/process.rb index 5da262a..45f2811 100644 --- a/lib/god/process.rb +++ b/lib/god/process.rb @@ -12,6 +12,7 @@ module God @pid_file = nil @tracking_pid = true @user_log = false + @pid = nil end def alive? @@ -124,6 +125,18 @@ module God @pid_file ||= default_pid_file end + def pid + contents = File.read(self.pid_file).strip rescue '' + real_pid = contents =~ /^\d+$/ ? contents.to_i : nil + + if real_pid + @pid = real_pid + real_pid + else + @pid + end + end + def start! call_action(:start) end diff --git a/lib/god/task.rb b/lib/god/task.rb index 678f13c..9eba751 100644 --- a/lib/god/task.rb +++ b/lib/god/task.rb @@ -1,7 +1,7 @@ module God class Task - attr_accessor :name, :interval, :group, :valid_states, :initial_state + attr_accessor :name, :interval, :group, :valid_states, :initial_state, :phase attr_writer :autostart def autostart?; @autostart; end @@ -128,6 +128,9 @@ module God # Move from one state to another def move(to_state) self.mutex.synchronize do + # set the phase for this move + self.phase = Time.now + orig_to_state = to_state from_state = self.state diff --git a/lib/god/timer.rb b/lib/god/timer.rb index a25b285..a46d4dd 100644 --- a/lib/god/timer.rb +++ b/lib/god/timer.rb @@ -1,7 +1,7 @@ module God class TimerEvent - attr_accessor :condition, :at + attr_accessor :condition, :at, :phase # Instantiate a new TimerEvent that will be triggered after the specified delay # +condition+ is the Condition @@ -10,7 +10,10 @@ module God # Returns TimerEvent def initialize(condition, delay) self.condition = condition - self.at = Time.now.to_i + delay + self.phase = condition.watch.phase + + now = (Time.now.to_f * 4).round / 4.0 + self.at = now + delay end end @@ -41,7 +44,8 @@ module God # Returns Timer def initialize @events = [] - @mutex = Mutex.new + @conditions = [] + @mutex = Monitor.new @timer = Thread.new do loop do @@ -66,6 +70,7 @@ module God # remove all triggered events triggered.each do |event| + @conditions.delete(event.condition) @events.delete(event) end end @@ -88,8 +93,11 @@ module God # Returns nothing def schedule(condition, delay = condition.interval) @mutex.synchronize do - @events << TimerEvent.new(condition, delay) - @events.sort! { |x, y| x.at <=> y.at } + unless @conditions.include?(condition) + @events << TimerEvent.new(condition, delay) + @conditions << condition + @events.sort! { |x, y| x.at <=> y.at } + end end end @@ -99,6 +107,7 @@ module God # Returns nothing def unschedule(condition) @mutex.synchronize do + @conditions.delete(condition) @events.reject! { |x| x.condition == condition } end end @@ -108,7 +117,7 @@ module God # # Returns nothing def trigger(event) - Hub.trigger(event.condition) + Hub.trigger(event.condition, event.phase) end # Join the timer thread diff --git a/lib/god/watch.rb b/lib/god/watch.rb index 547e22c..ce698d3 100644 --- a/lib/god/watch.rb +++ b/lib/god/watch.rb @@ -13,7 +13,7 @@ module God extend Forwardable def_delegators :@process, :name, :uid, :gid, :start, :stop, :restart, :name=, :uid=, :gid=, :start=, :stop=, :restart=, - :pid_file, :pid_file=, :log, :log=, :alive? + :pid_file, :pid_file=, :log, :log=, :alive?, :pid # def initialize super diff --git a/test/configs/child_polls/simple_server.rb b/test/configs/child_polls/simple_server.rb index 369d3dd..0c43286 100755 --- a/test/configs/child_polls/simple_server.rb +++ b/test/configs/child_polls/simple_server.rb @@ -8,5 +8,5 @@ loop do 100000.times { data << 'x' } - sleep 0.1 + sleep 10 end \ No newline at end of file diff --git a/test/test_conditions_process_running.rb b/test/test_conditions_process_running.rb index 59392ca..c0b709d 100644 --- a/test/test_conditions_process_running.rb +++ b/test/test_conditions_process_running.rb @@ -6,9 +6,11 @@ class TestConditionsProcessRunning < Test::Unit::TestCase c = Conditions::ProcessRunning.new c.running = r - c.stubs(:watch).returns(stub(:pid_file => '', :name => 'foo')) + c.stubs(:watch).returns(stub(:pid => 123, :name => 'foo')) - no_stdout { assert_equal !r, c.test } + # no_stdout do + assert_equal !r, c.test + # end end end @@ -18,7 +20,7 @@ class TestConditionsProcessRunning < Test::Unit::TestCase c.running = r File.stubs(:exist?).returns(true) - c.stubs(:watch).returns(stub(:pid_file => '')) + c.stubs(:watch).returns(stub(:pid => 123)) File.stubs(:read).returns('5') System::Process.any_instance.stubs(:exists?).returns(false) @@ -32,7 +34,7 @@ class TestConditionsProcessRunning < Test::Unit::TestCase c.running = r File.stubs(:exist?).returns(true) - c.stubs(:watch).returns(stub(:pid_file => '')) + c.stubs(:watch).returns(stub(:pid => 123)) File.stubs(:read).returns('5') System::Process.any_instance.stubs(:exists?).returns(true) diff --git a/test/test_hub.rb b/test/test_hub.rb index 41478ce..ee0526d 100644 --- a/test/test_hub.rb +++ b/test/test_hub.rb @@ -12,13 +12,15 @@ class TestHub < Test::Unit::TestCase end @watch = God.watches['foo'] + @watch.phase = Time.now end # attach def test_attach_should_store_condition_metric_association c = Conditions::FakePollCondition.new - m = Metric.new(@watch, :foo) + c.watch = @watch + m = Metric.new(@watch, {true => :up}) Hub.attach(c, m) assert_equal m, Hub.directory[c] @@ -26,7 +28,8 @@ class TestHub < Test::Unit::TestCase def test_attach_should_schedule_for_poll_condition c = Conditions::FakePollCondition.new - m = Metric.new(@watch, :foo) + c.watch = @watch + m = Metric.new(@watch, {true => :up}) Timer.any_instance.expects(:schedule).with(c, 0) @@ -35,7 +38,8 @@ class TestHub < Test::Unit::TestCase def test_attach_should_regsiter_for_event_condition c = Conditions::FakeEventCondition.new - m = Metric.new(@watch, :foo) + c.watch = @watch + m = Metric.new(@watch, {true => :up}) c.expects(:register) @@ -46,7 +50,8 @@ class TestHub < Test::Unit::TestCase def test_detach_should_remove_condition_metric_association c = Conditions::FakePollCondition.new - m = Metric.new(@watch, :foo) + c.watch = @watch + m = Metric.new(@watch, {true => :up}) Hub.attach(c, m) Hub.detach(c) @@ -56,7 +61,8 @@ class TestHub < Test::Unit::TestCase def test_detach_should_unschedule_poll_conditions c = Conditions::FakePollCondition.new - m = Metric.new(@watch, :foo) + c.watch = @watch + m = Metric.new(@watch, {true => :up}) Hub.attach(c, m) Timer.any_instance.expects(:unschedule).with(c) @@ -67,7 +73,8 @@ class TestHub < Test::Unit::TestCase def test_detach_should_deregister_event_conditions c = Conditions::FakeEventCondition.new - m = Metric.new(@watch, :foo) + c.watch = @watch + m = Metric.new(@watch, {true => :up}) Hub.attach(c, m) c.expects(:deregister).once @@ -79,22 +86,24 @@ class TestHub < Test::Unit::TestCase def test_trigger_should_handle_poll_for_poll_condition c = Conditions::FakePollCondition.new - Hub.expects(:handle_poll).with(c) + c.watch = @watch + Hub.expects(:handle_poll).with(c, @watch.phase) - Hub.trigger(c) + Hub.trigger(c, @watch.phase) end def test_trigger_should_handle_event_for_event_condition c = Conditions::FakeEventCondition.new Hub.expects(:handle_event).with(c) - Hub.trigger(c) + Hub.trigger(c, @watch.phase) end # handle_poll def test_handle_poll_no_change_should_reschedule c = Conditions::FakePollCondition.new + c.watch = @watch c.interval = 10 m = Metric.new(@watch, {true => :up}) @@ -104,13 +113,14 @@ class TestHub < Test::Unit::TestCase Timer.any_instance.expects(:schedule) no_stdout do - t = Hub.handle_poll(c) + t = Hub.handle_poll(c, @watch.phase) t.join end end def test_handle_poll_change_should_move c = Conditions::FakePollCondition.new + c.watch = @watch c.interval = 10 m = Metric.new(@watch, {true => :up}) @@ -120,13 +130,14 @@ class TestHub < Test::Unit::TestCase @watch.expects(:move).with(:up) no_stdout do - t = Hub.handle_poll(c) + t = Hub.handle_poll(c, @watch.phase) t.join end end def test_handle_poll_should_not_abort_on_exception c = Conditions::FakePollCondition.new + c.watch = @watch c.interval = 10 m = Metric.new(@watch, {true => :up}) @@ -136,7 +147,7 @@ class TestHub < Test::Unit::TestCase assert_nothing_raised do no_stdout do - t = Hub.handle_poll(c) + t = Hub.handle_poll(c, @watch.phase) t.join end end @@ -144,6 +155,7 @@ class TestHub < Test::Unit::TestCase def test_handle_poll_should_use_overridden_transition c = Conditions::Tries.new + c.watch = @watch c.times = 1 c.transition = :start c.prepare @@ -154,13 +166,14 @@ class TestHub < Test::Unit::TestCase @watch.expects(:move).with(:start) no_stdout do - t = Hub.handle_poll(c) + t = Hub.handle_poll(c, @watch.phase) t.join end end def test_handle_poll_should_notify_if_triggering c = Conditions::FakePollCondition.new + c.watch = @watch c.interval = 10 c.notify = 'tom' @@ -171,13 +184,14 @@ class TestHub < Test::Unit::TestCase Hub.expects(:notify) no_stdout do - t = Hub.handle_poll(c) + t = Hub.handle_poll(c, @watch.phase) t.join end end def test_handle_poll_should_not_notify_if_not_triggering c = Conditions::FakePollCondition.new + c.watch = @watch c.interval = 10 c.notify = 'tom' @@ -188,7 +202,7 @@ class TestHub < Test::Unit::TestCase Hub.expects(:notify).never no_stdout do - t = Hub.handle_poll(c) + t = Hub.handle_poll(c, @watch.phase) t.join end end @@ -197,6 +211,7 @@ class TestHub < Test::Unit::TestCase def test_handle_event_should_move c = Conditions::FakeEventCondition.new + c.watch = @watch m = Metric.new(@watch, {true => :up}) Hub.attach(c, m) @@ -211,6 +226,7 @@ class TestHub < Test::Unit::TestCase def test_handle_event_should_notify_if_triggering c = Conditions::FakeEventCondition.new + c.watch = @watch c.notify = 'tom' m = Metric.new(@watch, {true => :up}) @@ -226,6 +242,7 @@ class TestHub < Test::Unit::TestCase def test_handle_event_should_not_notify_if_no_notify_set c = Conditions::FakeEventCondition.new + c.watch = @watch m = Metric.new(@watch, {true => :up}) Hub.attach(c, m) diff --git a/test/test_process.rb b/test/test_process.rb index 0c7d77a..60f5ea1 100644 --- a/test/test_process.rb +++ b/test/test_process.rb @@ -144,6 +144,34 @@ class TestProcessDaemon < Test::Unit::TestCase end end + # pid + + def test_pid_should_return_integer_for_valid_pid_files + File.stubs(:read).returns("123") + assert_equal 123, @p.pid + end + + def test_pid_should_return_nil_for_missing_files + @p.pid_file = '' + assert_equal nil, @p.pid + end + + def test_pid_should_return_nil_for_invalid_pid_files + File.stubs(:read).returns("four score and seven years ago") + assert_equal nil, @p.pid + end + + def test_pid_should_retain_last_pid_value_if_pid_file_is_removed + File.stubs(:read).returns("123") + assert_equal 123, @p.pid + + File.stubs(:read).returns("") + assert_equal 123, @p.pid + + File.stubs(:read).returns("246") + assert_equal 246, @p.pid + end + # defaul_pid_file def test_default_pid_file diff --git a/test/test_timer.rb b/test/test_timer.rb index f6c8ae2..aaec522 100644 --- a/test/test_timer.rb +++ b/test/test_timer.rb @@ -14,36 +14,39 @@ class TestTimer < Test::Unit::TestCase Time.stubs(:now).returns(0) w = Watch.new - @t.schedule(stub(:interval => 20)) + @t.schedule(stub(:interval => 20, :watch => w)) assert_equal 1, @t.events.size end def test_timer_should_remove_expired_events - @t.schedule(stub(:interval => 0)) + @t.schedule(stub(:interval => -1, :watch => Watch.new)) sleep(0.3) assert_equal 0, @t.events.size end def test_timer_should_remove_only_expired_events - @t.schedule(stub(:interval => 0)) - @t.schedule(stub(:interval => 1000)) + w = Watch.new + @t.schedule(stub(:interval => -1, :watch => w)) + @t.schedule(stub(:interval => 1000, :watch => w)) sleep(0.3) assert_equal 1, @t.events.size end def test_timer_should_sort_timer_events - @t.schedule(stub(:interval => 1000)) - @t.schedule(stub(:interval => 800)) - @t.schedule(stub(:interval => 900)) - @t.schedule(stub(:interval => 100)) + w = Watch.new + @t.schedule(stub(:interval => 1000, :watch => w)) + @t.schedule(stub(:interval => 800, :watch => w)) + @t.schedule(stub(:interval => 900, :watch => w)) + @t.schedule(stub(:interval => 100, :watch => w)) sleep(0.3) assert_equal [100, 800, 900, 1000], @t.events.map { |x| x.condition.interval } end def test_unschedule_should_remove_conditions - a = stub() - b = stub() + w = Watch.new + a = stub(:watch => w) + b = stub(:watch => w) @t.schedule(a, 100) @t.schedule(b, 200) assert_equal 2, @t.events.size @@ -52,11 +55,12 @@ class TestTimer < Test::Unit::TestCase end def test_time_should_recover_from_exceptions + w = Watch.new @t.expects(:trigger).raises(Exception.new) no_stdout do - @t.schedule(stub(:interval => 0)) + @t.schedule(stub(:interval => -1, :watch => w)) sleep(0.3) - @t.schedule(stub(:interval => 0)) + @t.schedule(stub(:interval => 0, :watch => w)) end end -- 2.11.4.GIT