File Coverage

blib/lib/MogileFS/Worker/Monitor.pm
Criterion Covered Total %
statement 36 365 9.8
branch 0 118 0.0
condition 0 62 0.0
subroutine 12 50 24.0
pod 0 29 0.0
total 48 624 7.6


line stmt bran cond sub pod time code
1             package MogileFS::Worker::Monitor;
2 21     21   93 use strict;
  21         32  
  21         656  
3 21     21   93 use warnings;
  21         27  
  21         676  
4              
5 21     21   80 use base 'MogileFS::Worker';
  21         35  
  21         2717  
6             use fields (
7 21         170 'last_test_write', # devid -> time. time we last tried writing to a device.
8             'monitor_start', # main monitor start time
9             'skip_host', # hostid -> 1 if already noted dead (reset every loop)
10             'seen_hosts', # IP -> 1 (reset every loop)
11             'iow', # MogileFS::IOStatWatcher object
12             'prev_data', # DB data from previous run
13             'devutil', # Running tally of device utilization
14             'events', # Queue of state events
15             'refresh_state', # devid -> { used, total, callbacks }, temporary data in each refresh run
16             'have_masterdb', # Hint flag for if the master DB is available
17             'updateable_devices', # devid -> Device, avoids device table updates
18             'parent', # socketpair to parent process
19             'refresh_pending', # set if there was a manually-requested refresh
20             'db_monitor_ran', # We announce "monitor_just_ran" every time the
21             # device checks are run, but only if the DB has
22             # been checked inbetween.
23             'devs_to_update' # device table update queue
24 21     21   110 );
  21         33  
25              
26 21     21   2432 use Danga::Socket 1.56;
  21         530  
  21         533  
27 21     21   106 use MogileFS::Config;
  21         37  
  21         2849  
28 21     21   112 use MogileFS::Util qw(error debug encode_url_args apply_state_events_list);
  21         32  
  21         1200  
29 21     21   7649 use MogileFS::IOStatWatcher;
  21         52  
  21         583  
30 21     21   227 use MogileFS::Server;
  21         31  
  21         375  
31 21     21   7833 use MogileFS::Connection::Parent;
  21         45  
  21         496  
32 21     21   100 use Digest::MD5 qw(md5_base64);
  21         26  
  21         1162  
33              
34 21     21   94 use constant UPDATE_DB_EVERY => 15;
  21         24  
  21         68773  
35              
36             sub new {
37 0     0 0   my ($class, $psock) = @_;
38 0           my $self = fields::new($class);
39 0           $self->SUPER::new($psock);
40              
41 0           $self->{last_test_write} = {};
42 0           $self->{iow} = MogileFS::IOStatWatcher->new;
43 0           $self->{prev_data} = { domain => {}, class => {}, host => {},
44             device => {} };
45 0           $self->{devutil} = { cur => {}, prev => {}, tmp => {} };
46 0           $self->{events} = [];
47 0           $self->{have_masterdb} = 0;
48 0           return $self;
49             }
50              
51             sub watchdog_timeout {
52 0     0 0   30;
53             }
54              
55             # returns 1 if a DB update was attempted
56             # returns 0 immediately if the (device) monitor is already running
57             sub cache_refresh {
58 0     0 0   my $self = shift;
59              
60 0 0         if ($self->{refresh_state}) {
61 0           debug("Monitor run in progress, will not check for DB updates");
62 0           return 0;
63             }
64              
65 0           debug("Monitor running; checking DB for updates");
66             # "Fix" our local cache of this flag, so we always check the master DB.
67 0           MogileFS::Config->cache_server_setting('_master_db_alive', 1);
68 0           my $have_dbh = $self->validate_dbh;
69 0 0 0       if ($have_dbh && !$self->{have_masterdb}) {
    0          
70 0           $self->{have_masterdb} = 1;
71 0           $self->set_event('srvset', '_master_db_alive', { value => 1 });
72             } elsif (!$have_dbh) {
73 0           $self->{have_masterdb} = 0;
74 0           $self->set_event('srvset', '_master_db_alive', { value => 0 });
75 0           error("Cannot connect to master database!");
76             }
77              
78 0 0         if ($have_dbh) {
79 0           my $db_data = $self->grab_all_data;
80              
81             # Stack diffs to ship back later
82 0           $self->diff_data($db_data);
83             }
84              
85 0           $self->send_events_to_parent;
86 0           $self->{db_monitor_ran} = 1;
87              
88 0           return 1;
89             }
90              
91             sub usage_refresh {
92 0     0 0   my ($self) = @_;
93              
94             # prevent concurrent refresh
95 0 0         return if $self->{refresh_state};
96              
97 0           debug("Monitor running; scanning usage files");
98              
99 0           $self->{refresh_state} = {}; # devid -> ...
100 0           $self->{monitor_start} = Time::HiRes::time();
101              
102 0           my $have_dbh = $self->validate_dbh;
103              
104             # See if we should be allowed to update the device table rows.
105 0 0 0       if ($have_dbh && Mgd::get_store()->get_lock('mgfs:device_update', 0)) {
106             # Fetch the freshlist list of entries, to avoid excessive writes.
107 0           $self->{updateable_devices} = { map { $_->{devid} => $_ }
  0            
108             Mgd::get_store()->get_all_devices };
109 0           $self->{devs_to_update} = [];
110             } else {
111 0           $self->{updateable_devices} = undef;
112             }
113              
114 0           $self->{skip_host} = {}; # hostid -> 1 if already noted dead.
115 0           $self->{seen_hosts} = {}; # IP -> 1
116              
117 0           my $dev_factory = MogileFS::Factory::Device->get_factory();
118 0           my $devutil = $self->{devutil};
119              
120 0           $devutil->{tmp} = {};
121             # kick off check_device to test host/devs. diff against old values.
122 0           for my $dev ($dev_factory->get_all) {
123 0 0         if (my $state = $self->is_iow_diff($dev)) {
124 0           $self->state_event('device', $dev->id, {utilization => $state});
125             }
126 0           $devutil->{tmp}->{$dev->id} = $devutil->{cur}->{$dev->id};
127              
128 0 0         $dev->can_read_from or next;
129 0           $self->check_device_begin($dev);
130             }
131             # we're done if we didn't schedule any work
132 0 0         $self->usage_refresh_done unless keys %{$self->{refresh_state}};
  0            
133             }
134              
135             sub usage_refresh_done {
136 0     0 0   my ($self) = @_;
137              
138 0           $self->{devutil}->{prev} = $self->{devutil}->{tmp};
139             # Set the IOWatcher hosts (once old monitor code has been disabled)
140              
141 0           $self->send_events_to_parent;
142              
143 0           $self->{iow}->set_hosts(keys %{$self->{seen_hosts}});
  0            
144              
145 0           foreach my $devid (keys %{$self->{refresh_state}}) {
  0            
146 0           error("device check incomplete for dev$devid");
147             }
148              
149 0           my $start = delete $self->{monitor_start};
150 0           my $elapsed = Time::HiRes::time() - $start;
151 0           debug("device refresh finished after $elapsed");
152              
153 0           $self->{refresh_state} = undef;
154 0           my $pending_since = $self->{refresh_pending};
155              
156             # schedule another usage_refresh immediately if somebody requested it
157             # Don't announce :monitor_just_ran if somebody requested a refresh
158             # while we were running, we could've been refreshing on a stale DB
159 0 0 0       if ($pending_since && $pending_since > $start) {
160             # using AddTimer to schedule the refresh to avoid stack overflow
161             # since usage_refresh can call usage_refresh_done directly if
162             # there are no devices
163             Danga::Socket->AddTimer(0, sub {
164 0     0     $self->cache_refresh;
165 0           $self->usage_refresh;
166 0           });
167             }
168              
169             # announce we're done if we ran on schedule, or we had a
170             # forced refresh that was requested before we started.
171 0 0 0       if (!$pending_since || $pending_since <= $start) {
172             # totally done refreshing, accept manual refresh requests again
173 0           $self->{parent}->watch_read(1);
174 0           delete $self->{refresh_pending};
175 0 0 0       if (delete $self->{db_monitor_ran} || $pending_since) {
176 0           $self->send_to_parent(":monitor_just_ran");
177             }
178             }
179              
180 0 0         if ($self->{updateable_devices}) {
181 0           my $sto = Mgd::get_store();
182 0           my $updates = delete $self->{devs_to_update};
183 0     0     $sto->update_device_usages($updates, sub { $self->still_alive });
  0            
184 0           $sto->release_lock('mgfs:device_update');
185 0           $self->{updateable_devices} = undef;
186             }
187             }
188              
189             sub work {
190 0     0 0   my $self = shift;
191              
192             # It makes sense to have monitor use a shorter timeout
193             # (conn_timeout) across the board to skip slow hosts. Other workers
194             # are less tolerant, and may use a higher value in node_timeout.
195 0           MogileFS::Config->set_config_no_broadcast("node_timeout", MogileFS::Config->config("conn_timeout"));
196              
197 0           my $iow = $self->{iow};
198             $iow->on_stats(sub {
199 0     0     my ($hostname, $stats) = @_;
200              
201 0           while (my ($devid, $util) = each %$stats) {
202             # Lets not propagate devices that we accidentally find.
203 0           my $dev = Mgd::device_factory()->get_by_id($devid);
204 0 0         next unless $dev;
205 0           $self->{devutil}->{cur}->{$devid} = $util;
206             }
207 0           });
208              
209 0           my $db_monitor;
210             $db_monitor = sub {
211 0     0     $self->still_alive;
212              
213             # reschedule immediately if we were blocked by main_monitor.
214             # setting refresh_pending will call cache_refresh again
215 0 0         if (!$self->cache_refresh) {
216 0   0       $self->{refresh_pending} ||= Time::HiRes::time();
217             }
218              
219             # always reschedule in 4 seconds, regardless
220 0           Danga::Socket->AddTimer(4, $db_monitor);
221 0           };
222              
223 0           $db_monitor->();
224 0           $self->read_from_parent;
225              
226 0           my $main_monitor;
227             $main_monitor = sub {
228 0     0     $self->{parent}->ping;
229 0           $self->usage_refresh;
230 0           Danga::Socket->AddTimer(2.5, $main_monitor);
231 0           };
232              
233 0           $self->parent_ping; # ensure we get the initial DB state back
234 0           $self->{parent} = MogileFS::Connection::Parent->new($self);
235 0           Danga::Socket->AddTimer(0, $main_monitor);
236 0           Danga::Socket->EventLoop;
237             }
238              
239             sub process_line {
240 0     0 0   my MogileFS::Worker::Monitor $self = shift;
241 0           my $lineref = shift;
242 0 0         if ($$lineref =~ /^:refresh_monitor$/) {
243 0 0         if ($self->cache_refresh) {
244 0           $self->usage_refresh;
245             } else {
246 0   0       $self->{refresh_pending} ||= Time::HiRes::time();
247             }
248             # try to stop processing further refresh_monitor requests
249             # if we're acting on a manual refresh
250 0           $self->{parent}->watch_read(0);
251 0           return 1;
252             }
253 0           return 0;
254             }
255              
256             # --------------------------------------------------------------------------
257              
258             # Flattens and flips events up to the parent. Can be huge on startup!
259             # Events: set type foo=bar&baz=quux
260             # remove type id
261             # setstate type id foo=bar&baz=quux
262             # Combined: ev_mode=set&ev_type=device&foo=bar
263             # ev_mode=setstate&ev_type=device&ev_id=1&foo=bar
264             sub send_events_to_parent {
265 0     0 0   my $self = shift;
266 0           my @flat = ();
267 0           for my $ev (@{$self->{events}}) {
  0            
268 0           my ($mode, $type, $args) = @$ev;
269 0           $args->{ev_mode} = $mode;
270 0           $args->{ev_type} = $type;
271 0           push(@flat, encode_url_args($args));
272             }
273 0 0         return unless @flat;
274 0           $self->{events} = [];
275              
276             {
277             # $events can be several MB, so let it go out-of-scope soon:
278 0           my $events = join(' ', ':monitor_events', @flat);
  0            
279 0           debug("sending state changes $events", 2);
280 0           $self->send_to_parent($events);
281             }
282              
283 0           apply_state_events_list(@flat);
284             }
285              
286             sub add_event {
287 0     0 0   push(@{$_[0]->{events}}, $_[1]);
  0            
288             }
289              
290             sub set_event {
291             # Allow callers to use shorthand
292 0     0 0   $_[3]->{ev_id} = $_[2];
293 0           $_[0]->add_event(['set', $_[1], $_[3]]);
294             }
295 0     0 0   sub remove_event { $_[0]->add_event(['remove', $_[1], { ev_id => $_[2] }]); }
296             sub state_event {
297 0     0 0   $_[3]->{ev_id} = $_[2];
298 0           $_[0]->add_event(['setstate', $_[1], $_[3]]);
299             }
300              
301             sub is_iow_diff {
302 0     0 0   my ($self, $dev) = @_;
303 0           my $devid = $dev->id;
304 0           my $p = $self->{devutil}->{prev}->{$devid};
305 0           my $c = $self->{devutil}->{cur}->{$devid};
306 0 0 0       if ( ! defined $p || $p ne $c ) {
307 0           return $c;
308             }
309 0           return undef;
310             }
311              
312             sub diff_data {
313 0     0 0   my ($self, $db_data) = @_;
314              
315 0           my $new_data = {};
316 0           my $prev_data = $self->{prev_data};
317 0           for my $type (keys %{$db_data}) {
  0            
318 0           my $d_data = $db_data->{$type};
319 0           my $p_data = $prev_data->{$type};
320 0           my $n_data = {};
321              
322 0           for my $item (@{$d_data}) {
  0            
323 0 0         my $id = $type eq 'domain' ? $item->{dmid}
    0          
    0          
    0          
    0          
324             : $type eq 'class' ? $item->{dmid} . '-' . $item->{classid}
325             : $type eq 'host' ? $item->{hostid}
326             : $type eq 'device' ? $item->{devid}
327             : $type eq 'srvset' ? $item->{field}
328             : die "Unknown type";
329 0           my $old = delete $p_data->{$id};
330             # Special case: for devices, we don't care if mb_asof changes.
331             # FIXME: Change the grab routine (or filter there?).
332 0 0         delete $item->{mb_asof} if $type eq 'device';
333 0 0 0       if (!$old || $self->diff_hash($old, $item)) {
334 0           $self->set_event($type, $id, { %$item });
335             }
336 0           $n_data->{$id} = $item;
337             }
338 0           for my $id (keys %{$p_data}) {
  0            
339 0           $self->remove_event($type, $id);
340             }
341              
342 0           $new_data->{$type} = $n_data;
343             }
344 0           $self->{prev_data} = $new_data;
345             }
346              
347             # returns 1 if the hashes are different.
348             sub diff_hash {
349 0     0 0   my ($self, $old, $new) = @_;
350              
351 0           my %keys = ();
352 0           map { $keys{$_}++ } keys %$old, keys %$new;
  0            
353 0           for my $k (keys %keys) {
354 0 0 0       return 1 if (exists $old->{$k} && ! exists $new->{$k});
355 0 0 0       return 1 if (exists $new->{$k} && ! exists $old->{$k});
356 0 0 0       return 1 if (defined $old->{$k} && ! defined $new->{$k});
357 0 0 0       return 1 if (defined $new->{$k} && ! defined $old->{$k});
358 0 0 0       next if (! defined $new->{$k} && ! defined $old->{$k});
359 0 0         return 1 if ($old->{$k} ne $new->{$k});
360             }
361 0           return 0;
362             }
363              
364             sub grab_all_data {
365 0     0 0   my $self = shift;
366 0           my $sto = Mgd::get_store();
367              
368             # Normalize the domain data to the rest to simplify the differ.
369             # FIXME: Once new objects are swapped in, fix the original
370 0           my %dom = $sto->get_all_domains;
371 0           my @fixed_dom = ();
372 0           while (my ($name, $id) = each %dom) {
373 0           push(@fixed_dom, { namespace => $name, dmid => $id });
374             }
375              
376 0           my $set = $sto->server_settings;
377 0           my @fixed_set = ();
378 0           while (my ($field, $value) = each %$set) {
379 0           push(@fixed_set, { field => $field, value => $value });
380             }
381              
382 0           my %ret = ( domain => \@fixed_dom,
383             class => [$sto->get_all_classes],
384             host => [$sto->get_all_hosts],
385             device => [$sto->get_all_devices],
386             srvset => \@fixed_set, );
387 0           return \%ret;
388             }
389              
390             # returns true on success, false on failure
391             sub check_usage_response {
392 0     0 0   my ($self, $dev, $response) = @_;
393 0           my $devid = $dev->id;
394              
395 0           my %stats;
396 0           my $data = $response->content;
397 0           foreach (split(/\r?\n/, $data)) {
398 0 0         next unless /^(\w+)\s*:\s*(.+)$/;
399 0           $stats{$1} = $2;
400             }
401              
402 0           my ($used, $total) = ($stats{used}, $stats{total});
403 0 0 0       unless ($used && $total) {
404 0 0         $used = "" unless defined $used;
405 0 0         $total = "" unless defined $total;
406 0   0       my $clen = length($data || "");
407 0           error("dev$devid reports used = $used, total = $total, content-length: $clen, error?");
408 0           return 0;
409             }
410              
411 0           my $rstate = $self->{refresh_state}->{$devid};
412 0           ($rstate->{used}, $rstate->{total}) = ($used, $total);
413              
414             # only update database every ~15 seconds per device
415 0 0         if ($self->{updateable_devices}) {
416 0           my $devrow = $self->{updateable_devices}->{$devid};
417 0 0 0       my $last = ($devrow && $devrow->{mb_asof}) ? $devrow->{mb_asof} : 0;
418 0           my $now = time();
419 0 0         if ($last + UPDATE_DB_EVERY < $now) {
420 0           my %upd = (mb_total => int($total / 1024),
421             mb_used => int($used / 1024),
422             mb_asof => $now,
423             devid => $devid);
424 0           push @{$self->{devs_to_update}}, \%upd;
  0            
425             }
426             }
427 0           return 1;
428             }
429              
430             sub dev_debug {
431 0     0 0   my ($self, $dev, $writable) = @_;
432 0 0         return unless $Mgd::DEBUG >= 1;
433 0           my $devid = $dev->id;
434 0           my $rstate = $self->{refresh_state}->{$devid};
435 0           my ($used, $total) = ($rstate->{used}, $rstate->{total});
436              
437 0           debug("dev$devid: used = $used, total = $total, writeable = $writable");
438             }
439              
440             sub check_write {
441 0     0 0   my ($self, $dev) = @_;
442 0           my $rstate = $self->{refresh_state}->{$dev->id};
443 0           my $test_write = $rstate->{test_write};
444              
445 0 0 0       if (!$test_write || $test_write->{tries} > 0) {
446             # this was "$$-$now" before, but we don't yet have a cleaner in
447             # mogstored for these files
448 0           my $num = int(rand 100);
449 0   0       $test_write = $rstate->{test_write} ||= {};
450 0           $test_write->{path} = "/dev${\$dev->id}/test-write/test-write-$num";
  0            
451 0           $test_write->{content} = "time=" . time . " rand=$num";
452 0   0       $test_write->{tries} ||= 2;
453             }
454 0           $test_write->{tries}--;
455              
456 0           my $opts = { content => $test_write->{content} };
457             $dev->host->http("PUT", $test_write->{path}, $opts, sub {
458 0     0     my ($response) = @_;
459 0           $self->on_check_write_response($dev, $response);
460 0           });
461             }
462              
463             # starts the lengthy device check process
464             sub check_device_begin {
465 0     0 0   my ($self, $dev) = @_;
466 0           $self->{refresh_state}->{$dev->id} = {};
467              
468 0           $self->check_device($dev);
469             }
470              
471             # the lengthy device check process
472             sub check_device {
473 0     0 0   my ($self, $dev) = @_;
474 0 0         return $self->check_device_done($dev) if $self->{skip_host}{$dev->hostid};
475              
476 0           my $devid = $dev->id;
477 0           my $url = $dev->usage_url;
478 0           my $host = $dev->host;
479              
480 0           $self->{seen_hosts}{$host->ip} = 1;
481              
482             # now try to get the data with a short timeout
483 0           my $start_time = Time::HiRes::time();
484             $host->http_get("GET", $dev->usage_url, undef, sub {
485 0     0     my ($response) = @_;
486 0 0         if (!$self->on_usage_response($dev, $response, $start_time)) {
487 0           return $self->check_device_done($dev);
488             }
489             # next if we're not going to try this now
490 0           my $now = time();
491 0 0 0       if (($self->{last_test_write}{$devid} || 0) + UPDATE_DB_EVERY > $now) {
492 0           return $self->check_device_done($dev);
493             }
494 0           $self->{last_test_write}{$devid} = $now;
495              
496 0 0         unless ($dev->can_delete_from) {
497             # we should not try to write on readonly devices because it can be
498             # mounted as RO.
499 0           return $self->dev_observed_readonly($dev);
500             }
501             # now we want to check if this device is writeable
502              
503             # first, create the test-write directory. this will return
504             # immediately after the first time, as the 'create_directory'
505             # function caches what it's already created.
506             $dev->create_directory("/dev$devid/test-write", sub {
507 0           $self->check_write($dev);
508 0           });
509 0           });
510             }
511              
512             # called on a successful PUT, ensure the data we get back is what we uploaded
513             sub check_reread {
514 0     0 0   my ($self, $dev) = @_;
515             # now let's get it back to verify; note we use the get_port to
516             # verify that the distinction works (if we have one)
517 0           my $test_write = $self->{refresh_state}->{$dev->id}->{test_write};
518             $dev->host->http_get("GET", $test_write->{path}, undef, sub {
519 0     0     my ($response) = @_;
520 0           $self->on_check_reread_response($dev, $response);
521 0           });
522             }
523              
524             sub on_check_reread_response {
525 0     0 0   my ($self, $dev, $response) = @_;
526 0           my $test_write = $self->{refresh_state}->{$dev->id}->{test_write};
527              
528             # if success and the content matches, mark it writeable
529 0 0         if ($response->is_success) {
530 0 0         if ($response->content eq $test_write->{content}) {
531 0 0         if (!$dev->observed_writeable) {
532 0           my $event = { observed_state => 'writeable' };
533 0           $self->state_event('device', $dev->id, $event);
534             }
535 0           $self->dev_debug($dev, 1);
536              
537 0           return $self->check_bogus_md5($dev); # onto the final check...
538             }
539              
540             # content didn't match due to race, retry and hope we're lucky
541 0 0         return $self->check_write($dev) if ($test_write->{tries} > 0);
542             }
543              
544 0           return $self->dev_observed_readonly($dev); # it's read-only at least
545             }
546              
547             sub on_check_write_response {
548 0     0 0   my ($self, $dev, $response) = @_;
549 0 0         return $self->check_reread($dev) if $response->is_success;
550 0           return $self->dev_observed_readonly($dev);
551             }
552              
553             # returns true on success, false on failure
554             sub on_usage_response {
555 0     0 0   my ($self, $dev, $response, $start_time) = @_;
556 0           my $host = $dev->host;
557 0           my $hostip = $host->ip;
558              
559 0 0         if ($response->is_success) {
560             # at this point we can reach the host
561 0 0         if (!$host->observed_reachable) {
562 0           my $event = { observed_state => 'reachable' };
563 0           $self->state_event('host', $dev->hostid, $event);
564             }
565 0           $self->{iow}->restart_monitoring_if_needed($hostip);
566              
567 0           return $self->check_usage_response($dev, $response);
568             }
569              
570 0           my $url = $dev->usage_url;
571 0           my $failed_after = Time::HiRes::time() - $start_time;
572 0 0         if ($failed_after < 0.5) {
573 0 0         if (!$dev->observed_unreachable) {
574 0           my $event = { observed_state => 'unreachable' };
575 0           $self->state_event('device', $dev->id, $event);
576             }
577 0           my $get_port = $host->http_get_port;
578 0           error("Port $get_port not listening on $hostip ($url)? Error was: " . $response->status_line);
579             } else {
580 0           $failed_after = sprintf("%.02f", $failed_after);
581 0 0         if (!$host->observed_unreachable) {
582 0           my $event = { observed_state => 'unreachable' };
583 0           $self->state_event('host', $dev->hostid, $event);
584             }
585 0           $self->{skip_host}{$dev->hostid} = 1;
586             }
587 0           return 0; # failure
588             }
589              
590             sub check_bogus_md5 {
591 0     0 0   my ($self, $dev) = @_;
592 0           my $put_path = "/dev${\$dev->id}/test-write/test-md5";
  0            
593 0           my $opts = {
594             headers => { "Content-MD5" => md5_base64("!") . "==", },
595             content => '.',
596             };
597              
598             # success is bad here, it means the server doesn't understand how to
599             # verify and reject corrupt bodies from Content-MD5 headers.
600             # most servers /will/ succeed here :<
601             $dev->host->http("PUT", $put_path, $opts, sub {
602 0     0     my ($response) = @_;
603 0           $self->on_bogus_md5_response($dev, $response);
604 0           });
605             }
606              
607             sub on_bogus_md5_response {
608 0     0 0   my ($self, $dev, $response) = @_;
609 0 0         my $rej = $response->is_success ? 0 : 1;
610 0           my $prev = $dev->reject_bad_md5;
611              
612 0 0 0       if (!defined($prev) || $prev != $rej) {
613 0           debug("dev${\$dev->id}: reject_bad_md5 = $rej");
  0            
614 0           $self->state_event('device', $dev->id, { reject_bad_md5 => $rej });
615             }
616 0           return $self->check_device_done($dev);
617             }
618              
619             # if we fall through to here, then we know that something is not so
620             # good, so mark it readable which is guaranteed given we even tested
621             # writeability
622             sub dev_observed_readonly {
623 0     0 0   my ($self, $dev) = @_;
624              
625 0 0         if (!$dev->observed_readable) {
626 0           my $event = { observed_state => 'readable' };
627 0           $self->state_event('device', $dev->id, $event);
628             }
629 0           $self->dev_debug($dev, 0);
630 0           return $self->check_device_done($dev);
631             }
632              
633             # called when all checks are done for a particular device
634             sub check_device_done {
635 0     0 0   my ($self, $dev) = @_;
636              
637 0           $self->still_alive; # Ping parent if needed so we don't time out
638             # given lots of devices.
639 0           delete $self->{refresh_state}->{$dev->id};
640              
641             # if refresh_state is totally empty, we're done
642 0 0         if ((scalar keys %{$self->{refresh_state}}) == 0) {
  0            
643 0           $self->usage_refresh_done;
644             }
645             }
646              
647             1;
648              
649             # Local Variables:
650             # mode: perl
651             # c-basic-indent: 4
652             # indent-tabs-mode: nil
653             # End: