File Coverage

lib/Provision/Unix/VirtualOS.pm
Criterion Covered Total %
statement 56 385 14.5
branch 6 150 4.0
condition 2 27 7.4
subroutine 11 46 23.9
pod 17 34 50.0
total 92 642 14.3


line stmt bran cond sub pod time code
1             package Provision::Unix::VirtualOS;
2             # ABSTRACT: Provision virtual computers (VPS,VM,VE,Jail,etc)
3              
4 8     8   6220 use strict;
  8         15  
  8         273  
5 8     8   40 use warnings;
  8         16  
  8         300  
6              
7             our $VERSION = '0.59';
8              
9 8     8   42 use Data::Dumper;
  8         11  
  8         430  
10 8     8   39 use English qw( -no_match_vars );
  8         11  
  8         67  
11 8     8   10144 use LWP::Simple;
  8         640121  
  8         79  
12 8     8   4058 use LWP::UserAgent;
  8         16  
  8         223  
13 8     8   71 use Params::Validate qw(:all);
  8         13  
  8         1604  
14 8     8   47 use Time::Local;
  8         16  
  8         61794  
15              
16             our $AUTOLOAD;
17             my ($prov, $util);
18             my @std_opts = qw/ test_mode debug fatal /;
19             my %std_opts = (
20             test_mode => { type => BOOLEAN, optional => 1 },
21             debug => { type => BOOLEAN, optional => 1, default => 1 },
22             fatal => { type => BOOLEAN, optional => 1, default => 1 },
23             );
24              
25             sub new {
26              
27             # Usage : $vos->new( prov => $prov );
28             # Purpose : create a $vos object
29             # Returns : a Provision::Unix::VirtualOS object
30             # Parameters :
31             # Required : prov - a Provision::Unix object
32             # Optional : etc_dir - an etc directory used by some P:U:V classes
33              
34 7     7 0 251 my $class = shift;
35 7         498 my %p = validate(
36             @_,
37             { prov => { type => OBJECT },
38             etc_dir => { type => SCALAR, optional => 1 },
39             debug => { type => BOOLEAN, optional => 1, default => 1 },
40             fatal => { type => BOOLEAN, optional => 1, default => 1 },
41             }
42             );
43              
44 7         75 $prov = $p{prov};
45 7         180 my $debug = $p{debug};
46 7         21 my $fatal = $p{fatal};
47 7         53 $util = $prov->get_util;
48              
49 7         51 my $self = {
50             prov => $prov,
51             debug => $debug,
52             fatal => $fatal,
53             etc_dir => $p{etc_dir},
54             util => $util,
55             };
56 7         118 bless( $self, $class );
57              
58 7         64 $prov->audit( $class . sprintf( " loaded by %s, %s, %s", caller ) );
59              
60 7 50       104 $self->{vtype} = $self->_get_virt_type( fatal => $fatal, debug => $debug )
61             or die $prov->{errors}[-1]{errmsg};
62              
63 0         0 return $self;
64             }
65              
66             sub create {
67 0     0 1 0 my $self = shift;
68 0         0 my @opt_scalars = qw/ config cpu disk_root disk_size hostname
69             kernel_version mac_address nameservers password ram
70             searchdomain ssh_key template /;
71 0         0 my %opt_scalars = map { $_ => { type => SCALAR, optional => 1 } } @opt_scalars;
  0         0  
72 0         0 my @opt_bools = qw/ skip_start /;
73 0         0 my %opt_bools = map { $_ => { type => BOOLEAN, optional => 1 } } @opt_bools;
  0         0  
74              
75 0         0 my %p = validate(
76             @_,
77             { name => { type => SCALAR },
78             ip => { type => SCALAR },
79             %opt_scalars,
80             %opt_bools,
81             %std_opts,
82             }
83             );
84              
85 0         0 $prov->audit( "initializing request to create virtual os '$p{name}'");
86              
87 0         0 $self->{name} = $self->set_name( $p{name} );
88 0 0       0 $self->{ip} = $self->get_ips( $p{ip} ) or return;
89              
90 0         0 foreach ( @opt_scalars, @opt_bools, @std_opts ) {
91 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
92             };
93              
94 0 0       0 if ( $p{nameservers} ) {
95 0         0 $prov->audit( "getting nameserver IP list");
96 0         0 $self->{nameservers} = $self->get_ips( $p{nameservers} );
97             };
98              
99 0         0 my ($delegate) = $self->{vtype} =~ m/^(.*)=HASH/;
100 0         0 $prov->audit("\tdelegating create request to $delegate");
101 0         0 $self->{vtype}->create();
102             }
103              
104             sub destroy {
105              
106 0     0 1 0 my $self = shift;
107 0         0 my %p = validate(
108             @_,
109             { 'name' => { type => SCALAR },
110             'disk_root' => { type => SCALAR, optional => 1 },
111             %std_opts
112             }
113             );
114              
115 0         0 my $name = $self->set_name( $p{name} );
116 0         0 $prov->audit("initializing request to destroy virtual os '$name'");
117              
118 0         0 foreach ( @std_opts ) {
119 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
120             };
121              
122 0         0 $self->{vtype}->destroy();
123             }
124              
125             sub start {
126              
127 0     0 1 0 my $self = shift;
128 0         0 my %p = validate(
129             @_,
130             { 'name' => { type => SCALAR },
131             %std_opts
132             }
133             );
134              
135 0         0 foreach ( 'name', @std_opts ) {
136 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
137             };
138              
139 0         0 $self->{vtype}->start();
140             }
141              
142             sub stop {
143              
144 0     0 1 0 my $self = shift;
145              
146 0         0 my %p = validate(
147             @_,
148             { 'name' => { type => SCALAR },
149             %std_opts,
150             }
151             );
152              
153 0         0 foreach ( 'name', @std_opts ) {
154 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
155             };
156              
157 0         0 $self->{vtype}->stop();
158             }
159              
160             sub restart {
161              
162 0     0 1 0 my $self = shift;
163              
164 0         0 my %p = validate(
165             @_,
166             { 'name' => { type => SCALAR },
167             %std_opts
168             }
169             );
170              
171 0         0 foreach ( 'name', @std_opts ) {
172 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
173             };
174              
175 0         0 $self->{vtype}->restart();
176             }
177              
178             sub disable {
179              
180 0     0 1 0 my $self = shift;
181              
182 0         0 my %p = validate(
183             @_,
184             { name => { type => SCALAR },
185             disk_root => { type => SCALAR, optional => 1 },
186             %std_opts
187             }
188             );
189              
190 0         0 foreach ( qw/ name disk_root /, @std_opts ) {
191 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
192             };
193              
194 0         0 $self->{vtype}->disable();
195             }
196              
197             sub enable {
198              
199 0     0 1 0 my $self = shift;
200              
201 0         0 my %p = validate(
202             @_,
203             { name => { type => SCALAR },
204             disk_root => { type => SCALAR, optional => 1 },
205             %std_opts
206             }
207             );
208              
209 0         0 foreach ( qw/ name disk_root /, @std_opts ) {
210 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
211             };
212              
213 0         0 $self->{vtype}->enable();
214             }
215              
216             sub migrate {
217 0     0 1 0 my $self = shift;
218 0         0 my @req_scalars = qw/ name new_node /;
219 0         0 my %req_scalars = map { $_ => { type => SCALAR } } @req_scalars;
  0         0  
220 0         0 my @opt_scalars = qw/ /;
221 0         0 my %opt_scalars = map { $_ => { type => SCALAR, optional => 1 } } @opt_scalars;
  0         0  
222 0         0 my @opt_bools = qw/ connection_test /;
223 0         0 my %opt_bools = map { $_ => { type => BOOLEAN, optional => 1 } } @opt_bools;
  0         0  
224              
225 0         0 my %p = validate( @_, { %req_scalars, %opt_scalars, %opt_bools, %std_opts, } );
226              
227 0         0 my $name = $p{name};
228 0         0 my $new_node = $p{new_node};
229              
230 0         0 $prov->audit("initializing request to migrate VE '$name' to $new_node");
231              
232 0         0 foreach ( @req_scalars, @opt_scalars, @std_opts ) {
233 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
234             };
235 0         0 foreach ( @opt_bools ) {
236 0 0       0 $self->{$_} = defined $p{$_} ? $p{$_} : 0;
237             };
238              
239 0         0 $prov->audit("\tdelegating request to $self->{vtype}");
240              
241 0         0 $self->{vtype}->migrate();
242             };
243              
244             sub modify {
245              
246 0     0 1 0 my $self = shift;
247 0         0 my @req_scalars = qw/ name disk_size hostname ip ram /;
248 0         0 my %req_scalars = map { $_ => { type => SCALAR } } @req_scalars;
  0         0  
249 0         0 my @opt_scalars = qw/ config cpu disk_root mac_address nameservers
250             password searchdomain ssh_key template /;
251 0         0 my %opt_scalars = map { $_ => { type => SCALAR, optional => 1 } } @opt_scalars;
  0         0  
252              
253 0         0 my %p = validate( @_, { %req_scalars, %opt_scalars, %std_opts, } );
254              
255 0         0 $prov->audit("initializing request to modify VE '$p{name}'");
256              
257 0         0 foreach ( @req_scalars, @opt_scalars, @std_opts ) {
258 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
259             };
260              
261 0 0       0 $self->{ip} = $self->get_ips( $p{ip} ) if $p{ip};
262 0 0       0 $self->{nameservers} = $self->get_ips( $p{nameservers} ) if $p{nameservers};
263              
264 0         0 $prov->audit("\tdelegating request to $self->{vtype}");
265              
266 0         0 $self->{vtype}->modify();
267             }
268              
269             sub reinstall {
270              
271             # Usage : $vos->reinstall( name => '42', ip=>'127.0.0.2' );
272             # Purpose : reinstall the OS in virtual machine
273             # Returns : true or undef on failure
274             # Parameters :
275             # Required : name - name/ID of the virtual OS
276             # : template - a 'template' or tarball the OS is patterned after
277             # : ip - IP address(es), space delimited
278             # Optional : hostname - the FQDN of the virtual OS
279             # : disk_root - the root directory of the virt os
280             # : disk_size - disk space allotment (MB)
281             # : ram - (MB)
282             # : config - a config file with virtual specific settings
283             # : password - the root/admin password for the virtual
284             # : nameservers -
285             # : searchdomain -
286              
287 0     0 0 0 my $self = shift;
288 0         0 my @opt_scalars = qw/ config cpu disk_root disk_size hostname
289             kernel_version mac_address nameservers password ram
290             searchdomain ssh_key template /;
291 0         0 my %opt_scalars = map { $_ => { type => SCALAR, optional => 1 } } @opt_scalars;
  0         0  
292              
293 0         0 my %p = validate(
294             @_,
295             { name => { type => SCALAR },
296             ip => { type => SCALAR },
297             %opt_scalars,
298             %std_opts,
299             }
300             );
301              
302 0         0 $prov->audit( "initializing request to reinstall ve '$p{name}'");
303              
304 0         0 foreach ( 'name', @opt_scalars, @std_opts ) {
305 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
306             };
307              
308 0         0 $self->{name} = $self->set_name( $p{name} );
309 0 0       0 $self->{ip} = $self->get_ips( $p{ip} ) or return;
310 0 0       0 $self->{nameservers} = $self->get_ips( $p{nameservers} ) if $p{nameservers};
311              
312 0         0 $prov->audit("\tdelegating request to $self->{vtype}");
313 0         0 $self->{vtype}->reinstall();
314             }
315              
316             sub AUTOLOAD {
317 0     0   0 my $self = shift;
318              
319             # this AUTOLOAD method works for any methods in P:U:V:* whose only required
320             # argument is the VE name.
321 0         0 my %p = validate(
322             @_,
323             { name => { type => SCALAR },
324             %std_opts,
325             }
326             );
327 0         0 my $fatal = $p{fatal};
328 0         0 my $v_type = ref $self->{vtype};
329              
330 0         0 foreach ( 'name', @std_opts ) {
331 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
332             };
333              
334 0         0 my $sub = $AUTOLOAD;
335 0         0 $sub =~ s/.*://; # strip off everything before the last :
336              
337 0         0 $prov->audit( "initializing request to $sub for ve '$p{name}'");
338              
339 0 0       0 ref ($self) || return $prov->error( "invalid call to $sub", fatal => $fatal );
340 0 0       0 return $self->{vtype}->$sub() if $self->{vtype}->can($sub);
341              
342 0         0 $prov->error( "The VE platform $v_type does not have support for '$sub'", fatal => $fatal);
343 0         0 return;
344             };
345              
346 0     0   0 sub DESTROY {};
347             sub create_snapshot;
348             sub destroy_snapshot;
349             sub mount_snapshot;
350             sub unmount_snapshot;
351             sub get_disk_usage;
352             sub get_mac_address;
353             sub get_config;
354             sub transition;
355             sub untransition;
356              
357             sub mount {
358 0     0 1 0 my $self = shift;
359 0         0 my %p = validate(
360             @_,
361             { name => { type => SCALAR },
362             refresh => { type => BOOLEAN, optional => 1, default => 1 },
363             %std_opts,
364             }
365             );
366              
367 0         0 $prov->audit( "initializing request to mount ve '$p{name}'");
368              
369 0         0 foreach ( qw/ name refresh /, @std_opts ) {
370 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
371             };
372              
373 0         0 $self->{vtype}->mount();
374             };
375              
376             sub unmount {
377 0     0 1 0 my $self = shift;
378 0         0 my %p = validate(
379             @_,
380             { name => { type => SCALAR },
381             refresh => { type => BOOLEAN, optional => 1, default => 1 },
382             %std_opts,
383             }
384             );
385              
386 0         0 $prov->audit( "initializing request to unmount ve '$p{name}'");
387              
388 0         0 foreach ( qw/ name refresh /, @std_opts ) {
389 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
390             };
391              
392 0         0 $self->{vtype}->unmount();
393             };
394              
395             sub publish_arp {
396 0     0 1 0 my $self = shift;
397 0         0 my %p = validate(
398             @_,
399             { ip => { type => SCALAR|ARRAYREF },
400             %std_opts,
401             }
402             );
403              
404 0         0 my $ip = $p{ip};
405 0 0       0 my @ips = ref $ip ? @$ip : $ip;
406              
407 0         0 my $arpsend = $util->find_bin( 'arpsend', fatal => 0 );
408              
409 0 0       0 if ( -x $arpsend ) {
410 0         0 foreach ( @ips ) {
411 0         0 $prov->audit( "$arpsend -U -c2 -i $_ eth0" );
412 0         0 system "$arpsend -U -c2 -i $_ eth0";
413             };
414 0         0 return 1;
415             }
416              
417             # TODO: try using arping, but I haven't been able to get it to work
418 0         0 return;
419             };
420              
421             sub do_connectivity_test {
422 0     0 0 0 my $self = shift;
423              
424 0 0       0 return 1 if ! $self->{connection_test};
425              
426 0         0 my $new_node = $self->{new_node};
427 0         0 my $debug = $self->{debug};
428 0         0 my $ssh = $util->find_bin( 'ssh', debug => $debug );
429 0 0       0 my $r = $util->syscmd( "$ssh $new_node /bin/uname -a", debug => $debug, fatal => 0)
430             or return $prov->error("could not validate connectivity to $new_node", fatal => 0);
431 0         0 $prov->audit("connectivity to $new_node is good");
432 0         0 return 1;
433             };
434              
435             sub gen_config {
436 0     0 0 0 my $self = shift;
437 0         0 my %p = validate(
438             @_,
439             { name => { type => SCALAR },
440             ram => { type => SCALAR },
441             disk_root => { type => SCALAR },
442             disk_size => { type => SCALAR },
443             template => { type => SCALAR },
444             config => { type => SCALAR },
445             hostname => { type => SCALAR },
446             ip => { type => SCALAR },
447             %std_opts,
448             }
449             );
450              
451 0         0 foreach ( qw/ name ram disk_size disk_root template config hostname /,
452             @std_opts )
453             {
454 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
455             };
456              
457 0         0 $self->{ip} = $self->get_ips( $p{ip} );
458              
459 0         0 $self->{vtype}->gen_config();
460             };
461              
462             sub get_fs_root {
463 0     0 0 0 my $self = shift;
464 0   0     0 my $name = shift || $self->{name};
465 0         0 my $fs_root;
466 0 0       0 if ( $self->{vtype}->can('get_fs_root') ) {
467 0         0 return $self->{vtype}->get_fs_root( $name );
468             }
469 0         0 return $self->{vtype}->get_ve_home( $name );
470             };
471              
472             sub get_ve_home {
473 0     0 0 0 my $self = shift;
474 0   0     0 my $name = shift || $self->{name};
475 0         0 my $fs_root;
476 0 0       0 if ( $self->{vtype}->can('get_ve_home') ) {
477 0         0 return $self->{vtype}->get_ve_home( $name );
478             }
479 0         0 return;
480             };
481              
482             sub get_ips {
483 0     0 0 0 my $self = shift;
484 0         0 my $ip_string = shift;
485              
486 0         0 $prov->audit("\textracting IPs from string: $ip_string");
487              
488 0         0 my @r;
489 0         0 my @ips = split / /, $ip_string;
490 0         0 foreach my $ip (@ips) {
491 0         0 my $ip = $self->is_valid_ip($ip);
492 0 0       0 push @r, $ip if $ip;
493             }
494              
495 0         0 my $ips = @r;
496 0 0       0 if ( $ips == 0 ) {
497 0         0 return $prov->error( "no valid IPs in request!",
498             debug => $self->{debug},
499             fatal => $self->{fatal},
500             );
501             }
502 0 0       0 my $ip_plural = $ips > 1 ? 'ips' : 'ip';
503 0         0 $prov->audit("\tfound $ips valid $ip_plural");
504 0         0 return \@r;
505             }
506              
507             sub get_status {
508 0     0 1 0 my $self = shift;
509 0         0 my %p = validate(
510             @_,
511             { name => { type => SCALAR | UNDEF, optional => 1 },
512             %std_opts,
513             }
514             );
515              
516 0         0 foreach ( 'name', @std_opts ) {
517 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
518             };
519              
520 0         0 $self->{vtype}->get_status();
521             }
522              
523             sub get_template_dir {
524              
525 0     0 0 0 my $self = shift;
526 0         0 my %p = validate(
527             @_,
528             { v_type => { type => SCALAR },
529             %std_opts,
530             }
531             );
532              
533 0         0 my $v_type = $p{v_type};
534              
535 0         0 my $dir = $prov->{config}{VirtualOS}{"${v_type}_template_dir"};
536 0 0       0 return $dir if $dir; # they defined it in provision.conf, use it
537              
538             # try to autodetect
539 0 0       0 $dir = -d "/templates" ? '/templates'
    0          
    0          
540             : -d "/vz/template/cache" ? '/vz/template/cache'
541             : -d "/vz/template" ? '/vz/template'
542             : undef;
543              
544 0 0       0 $dir and return $dir;
545              
546 0         0 return $prov->error( 'unable to determine template directory',
547             fatal => $p{fatal},
548             debug => $p{debug},
549             );
550             };
551              
552             sub get_template_list {
553 0     0 0 0 my $self = shift;
554 0         0 my %p = validate(
555             @_,
556             { v_type => { type => SCALAR },
557             url => { type => SCALAR, optional => 1 },
558             %std_opts,
559             }
560             );
561              
562 0         0 my $url = $p{url};
563 0         0 my $v_type = $p{v_type};
564 0         0 my @templates;
565              
566 0 0       0 if ( ! $url ) {
567 0 0       0 my $template_dir = $self->get_template_dir( v_type=> $v_type )
568             or return $prov->error( 'unable to determine template directory',
569             fatal => $p{fatal},
570             debug => $p{debug},
571             );
572              
573 0         0 my @template_names = glob("$template_dir/*.tar.gz");
574 0         0 foreach my $template ( @template_names ) {
575 0         0 ($template) = $template =~ /\/([\w\.\-]+)\.tar\.gz$/;
576 0         0 push @templates, { name => $template };
577             };
578              
579 0 0       0 return \@templates if scalar @templates;
580 0         0 return;
581             };
582              
583 0         0 my $ua = LWP::UserAgent->new( timeout => 10);
584 0         0 my $response = $ua->get($url);
585              
586 0 0       0 die $response->status_line if ! $response->is_success;
587              
588 0         0 my $content = $response->content;
589             #warn Dumper($content);
590              
591             # >centos-5-i386-plesk-8.6.tar.gz<
592 0         0 my @fields = grep { /\-.*?\-/ } split /<.*?>/, $content;
  0         0  
593 0         0 while ( scalar @fields ) {
594 0 0       0 my $file = shift @fields or last;
595 0 0       0 next if $file !~ /tar.gz/;
596 0         0 my $date = shift @fields;
597 0         0 my $timestamp = $self->get_template_timestamp($date);
598 0         0 push @templates, { name => $file, date => $date, timestamp => $timestamp };
599             };
600              
601 0         0 return \@templates;
602             };
603              
604             sub get_template_timestamp {
605 0     0 0 0 my ( $self, $time ) = @_;
606            
607 0         0 my %months = (
608             'jan' => 1, 'feb' => 2, 'mar' => 3, 'apr' => 4,
609             'may' => 5, 'jun' => 6, 'jul' => 7, 'aug' => 8,
610             'sep' => 9, 'oct' => 10, 'nov' => 11, 'dec' => 12,
611             );
612              
613 0 0       0 my ( $Y, $M, $D, $h, $m, $s )
614             = ( $time =~ /^(\d{4})-(\w{3})-(\d{2})\s+(\d{2})?:?(\d{2})?:?(\d{2})?/ )
615             or die "invalid timestamp format: $time\n";
616              
617 0         0 my $txt_m = lc($M);
618 0         0 $M = $months{$txt_m}; # convert to an integer
619 0         0 $M -= 1;
620 0         0 $Y -= 1900;
621 0         0 return timelocal( $s, $m, $h, $D, $M, $Y );
622             };
623              
624             sub get_version {
625 0     0 0 0 return $prov->get_version();
626             };
627              
628             sub probe {
629 0     0 0 0 my $self = shift;
630 0         0 return $self->get_status(@_);
631             };
632              
633             sub set_hostname {
634 0     0 1 0 my $self = shift;
635 0         0 my %p = validate(
636             @_,
637             { 'name' => { type => SCALAR },
638             'hostname' => { type => SCALAR },
639             %std_opts,
640             }
641             );
642              
643 0         0 foreach ( qw/ name hostname /, @std_opts ) {
644 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
645             };
646              
647 0         0 $self->{vtype}->set_hostname();
648             }
649              
650             sub set_name {
651 0     0 0 0 my $self = shift;
652 0   0     0 my $name = shift || $self->{name} || die "unable to set VE name\n";
653 0         0 $self->{name} = $name;
654 0         0 return $name;
655             };
656              
657             sub set_nameservers {
658 0     0 1 0 my $self = shift;
659 0         0 my %p = validate(
660             @_,
661             { name => { type => SCALAR, optional => 1 },
662             nameservers => { type => SCALAR, optional => 1 },
663             searchdomain => { type => SCALAR, optional => 1 },
664             %std_opts,
665             }
666             );
667              
668 0         0 my $name;
669 0 0       0 $name = $self->set_name( $p{name} ) if $p{name};
670 0         0 my $searchdomain = $p{searchdomain};
671 0 0       0 $self->{nameservers} = $self->get_ips( $p{nameservers} ) if $p{nameservers};
672 0 0       0 $self->{nameservers} or die 'missing nameservers';
673 0         0 $self->{searchdomain} = $searchdomain;
674 0         0 $self->{test_mode} = $p{test_mode};
675 0         0 my $debug = $self->{debug} = $p{debug};
676 0         0 my $fatal = $self->{fatal} = $p{fatal};
677              
678             # if the virtualzation package has the method, call it.
679 0 0       0 if ( $self->{vtype}->can( 'set_nameservers' ) ) {
680 0         0 return $self->{vtype}->set_nameservers();
681             };
682              
683             # otherwise, use this default method
684 0         0 my $fs_root = $self->get_fs_root();
685 0         0 my $nameservers = $self->{nameservers};
686 0         0 my $resolv = "$fs_root/etc/resolv.conf";
687              
688 0         0 my @new;
689 0 0       0 push @new, "searchdomain $searchdomain" if $searchdomain;
690 0 0       0 if ( -f $resolv ) {
691 0         0 my @lines = $util->file_read( $resolv, fatal => $fatal );
692              
693 0         0 foreach my $line ( @lines ) {
694 0 0       0 next if $line =~ /^nameserver\s/i;
695 0 0 0     0 next if $searchdomain && $line =~ /^searchdomain\s/i;
696 0         0 push @new, $line;
697             };
698             };
699              
700 0         0 foreach ( @$nameservers ) {
701 0         0 push @new, "nameserver $_";
702             };
703              
704 0         0 return $util->file_write( $resolv, lines => \@new, fatal => $fatal );
705             }
706              
707             sub set_password {
708 0     0 1 0 my $self = shift;
709 0         0 my %p = validate(
710             @_,
711             { name => { type => SCALAR },
712             password => { type => SCALAR },
713             user => { type => SCALAR | UNDEF, optional => 1 },
714             disk_root => { type => SCALAR, optional => 1 },
715             ssh_key => { type => SCALAR, optional => 1 },
716             %std_opts,
717             }
718             );
719              
720 0   0     0 $self->{user} = $p{user} || 'root';
721              
722 0         0 foreach ( qw/ name password ssh_key disk_root /, @std_opts ) {
723 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
724             };
725              
726 0         0 return $self->{vtype}->set_password();
727             }
728              
729             sub set_ssh_key {
730 0     0 1 0 my $self = shift;
731 0         0 my %p = validate(
732             @_,
733             { name => { type => SCALAR },
734             ssh_key => { type => SCALAR },
735             user => { type => SCALAR | UNDEF, optional => 1 },
736             disk_root => { type => SCALAR, optional => 1 },
737             %std_opts,
738             }
739             );
740              
741 0   0     0 $self->{user} = $p{user} || 'root';
742              
743 0         0 foreach ( qw/ name ssh_key disk_root /, @std_opts ) {
744 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
745             };
746              
747 0         0 return $self->{vtype}->set_ssh_key();
748             }
749              
750             sub setup_log_files {
751 0     0 0 0 my $self = shift;
752              
753 0         0 my %p = validate( @_, { fs_root => { type => SCALAR } } );
754              
755 0         0 my $fs_root = $p{fs_root};
756              
757 0         0 my @logfiles = `find $fs_root/var/log/ -maxdepth 1 -type f -print`;
758 0         0 foreach ( @logfiles ) {
759 0         0 chomp $_;
760 0         0 $util->file_write( $_, lines => [ '' ], fatal => 0, debug => 0 );
761             };
762             };
763              
764             sub setup_ssh_host_keys {
765 0     0 0 0 my $self = shift;
766              
767 0         0 my %p = validate( @_, { fs_root => { type => SCALAR } } );
768              
769 0         0 my $fs_root = $p{fs_root};
770              
771 0         0 foreach my $type ( qw/ dsa rsa / ) {
772 0         0 my $file_path = "$fs_root/etc/ssh/ssh_host_${type}_key";
773              
774 0 0       0 unlink "$file_path" if -e "$file_path";
775 0 0       0 unlink "$file_path.pub" if -e "$file_path.pub";
776              
777 0         0 my $cmd = "/usr/bin/ssh-keygen -q -t $type -f $file_path -N ''";
778 0         0 $util->syscmd( $cmd, debug => 0 );
779             };
780             };
781              
782             sub is_mounted {
783 0     0 0 0 my $self = shift;
784 0         0 my %p = validate(
785             @_,
786             { name => { type => SCALAR },
787             refresh => { type => BOOLEAN, optional => 1, default => 1 },
788             %std_opts,
789             }
790             );
791              
792 0         0 foreach ( qw/ name refresh /, @std_opts ) {
793 0 0       0 $self->{$_} = $p{$_} if defined $p{$_};
794             };
795              
796 0         0 $self->{vtype}->is_mounted();
797             }
798              
799             sub is_valid_ip {
800 0     0 0 0 my $self = shift;
801 0         0 my $ip = shift;
802 0         0 my $error = "'$ip' is not a valid IPv4 address";
803              
804 0         0 my $r = grep /\./, split( //, $ip ); # need 3 dots
805 0 0       0 return $prov->error( $error, fatal => 0, debug => 0 )
806             if $r != 3;
807              
808 0         0 my @octets = split /\./, $ip;
809 0 0       0 return $prov->error( $error, fatal => 0, debug => 0 )
810             if @octets != 4;
811              
812 0         0 foreach (@octets) {
813 0 0 0     0 return unless /^\d{1,3}$/ and $_ >= 0 and $_ <= 255;
      0        
814 0         0 $_ = 0 + $_;
815             }
816              
817 0 0       0 return $prov->error( $error, fatal => 0, debug => 0 )
818             if $octets[0] == 0; # 0. is invalid
819              
820 0 0       0 return $prov->error( $error, fatal => 0, debug => 0 )
821             if 0 + $octets[0] + $octets[1] + $octets[2] + $octets[3]
822             == 0; # 0.0.0.0 is invalid
823              
824 0 0       0 return $prov->error( $error, fatal => 0, debug => 0 )
825             if grep( $_ eq '255', @octets ) == 4; # 255.255.255.255 is invalid
826              
827 0         0 return join( '.', @octets );
828             }
829              
830             sub _get_virt_type {
831 7     7   22 my $self = shift;
832 7         257 my %p = validate( @_, { %std_opts });
833              
834 7         48 my $debug = $p{debug};
835 7         16 my $fatal = $p{fatal};
836 7         65 my $prov = $self->{prov};
837              
838 7 50       98 return $self->_get_virt_type_linux( %p ) if lc($OSNAME) eq 'linux';
839              
840 0 0       0 if ( lc( $OSNAME) eq 'freebsd' ) {
841 0         0 my $ezjail = $util->find_bin( 'ezjail-admin', fatal => 0, debug => 0 );
842 0 0       0 if ( $ezjail ) {
843 0         0 require Provision::Unix::VirtualOS::FreeBSD::Ezjail;
844 0         0 return Provision::Unix::VirtualOS::FreeBSD::Ezjail->new( vos => $self );
845             };
846              
847 0         0 require Provision::Unix::VirtualOS::FreeBSD::Jail;
848 0         0 return Provision::Unix::VirtualOS::FreeBSD::Jail->new( vos => $self );
849             };
850 0         0 $prov->error(
851             "No virtualization methods for $OSNAME yet",
852             fatal => $fatal,
853             debug => $debug,
854             );
855 0         0 return;
856             }
857              
858             sub _get_virt_type_linux {
859 7     7   14 my $self = shift;
860 7         149 my %p = validate( @_, { %std_opts });
861              
862 7         40 my $err_before = scalar @{ $prov->{errors} };
  7         21  
863 7         58 my $xm = $util->find_bin( 'xm', fatal => 0, debug => 0);
864 7         85 my $vzctl = $util->find_bin( 'vzctl', fatal => 0, debug => 0);
865 7 50       101 if ( scalar @{$prov->{errors}} > $err_before ) {
  7         116  
866 7         82 delete $prov->{errors}[-1]; # clear the last error
867 7 50       54 delete $prov->{errors}[-1] if scalar @{$prov->{errors}} > $err_before;
  7         122  
868             };
869              
870 7         15759 require Provision::Unix::VirtualOS::Linux;
871 7         131 $self->{linux} = Provision::Unix::VirtualOS::Linux->new( vos => $self );
872              
873 7 50 33     61 if ( $xm && ! $vzctl ) {
874 0         0 require Provision::Unix::VirtualOS::Linux::Xen;
875 0         0 return Provision::Unix::VirtualOS::Linux::Xen->new( vos => $self );
876             };
877 7 50 33     63 if ( $vzctl && ! $xm ) {
878             # this could be Virtuozzo or OpenVZ. The way to tell is by
879             # checking for the presence of /vz/template/cache (OpenVZ only)
880             # also, a Virtuozzo VE will have a cow directory inside the
881             # VE home directory.
882 0 0       0 if ( -d "/vz/template" ) {
883 0 0       0 if ( -d "/vz/template/cache" ) {
884 0         0 require Provision::Unix::VirtualOS::Linux::OpenVZ;
885 0         0 return Provision::Unix::VirtualOS::Linux::OpenVZ->new( vos => $self );
886             }
887             else {
888 0         0 require Provision::Unix::VirtualOS::Linux::Virtuozzo;
889 0         0 return Provision::Unix::VirtualOS::Linux::Virtuozzo->new( vos => $self );
890             }
891             }
892             else {
893             # has someone moved the template cache directory from the default location?
894 0         0 require Provision::Unix::VirtualOS::Linux::OpenVZ;
895 0         0 return Provision::Unix::VirtualOS::Linux::OpenVZ->new( vos => $self );
896             };
897             };
898              
899 7         86 $prov->error(
900             "Unable to determine your virtualization method. You need one supported hypervisor (xen, openvz) installed.",
901             fatal => $p{fatal},
902             debug => $p{debug},
903             );
904             };
905              
906             1;
907              
908              
909              
910             =pod
911              
912             =head1 NAME
913              
914             Provision::Unix::VirtualOS - Provision virtual computers (VPS,VM,VE,Jail,etc)
915              
916             =head1 VERSION
917              
918             version 1.06
919              
920             =head1 SYNOPSIS
921              
922             use Provision::Unix;
923             use Provision::Unix::VirtualOS;
924              
925             my $prov = Provision::Unix->new();
926             my $vos = Provision::Unix::VirtualOS->new( prov => $prov );
927              
928             $vos->create(
929             name => 42,
930             password => 't0ps3kretWerD',
931             ip => '10.1.1.43',
932             hostname => 'test_debian_5.example.com',
933             disk_size => 1000,
934             ram => 512,
935             template => 'debian-5-i386-default.tar.gz',
936             nameservers => '10.1.1.2 10.1.1.3',
937             searchdomain => 'example.com',
938             )
939             or $prov->error( "unable to create VE" );
940              
941             =head1 DESCRIPTION
942              
943             Provision::Unix::VirtualOS aims to provide a clean, consistent way to manage virtual machines on a variety of virtualization platforms including Xen, OpenVZ, Vmware, FreeBSD jails, and others. P:U:V provides a command line interface (prov_virtual) and a stable programming interface (API) for provisioning virtual machines on supported platforms. To start a VE on any supported virtualization platform, you run this command:
944              
945             prov_virtual --name=42 --action=start
946              
947             Versus this:
948              
949             xen: xm create /home/xen/42/42.cfg
950             ovz: vzctl start 42
951             ezj: ezjail-admin start 42
952              
953             P:U:V tries very hard to insure that every valid command that can succeed will. There is abundant code for handling common errors, such as unmounting xen volumes before starting a VE, making sure a disk volume is not in use before mounting it, and making sure connectivity to the new HW node exists before attempting to migrate.
954              
955             In addition to the pre-flight checks, there are also post-action checks to determine if the action succeeded. When actions fail, they provide reasonably good error messages geared towards comprehension by sysadmins. Where feasible, actions that fail are rolled back so that when the problem(s) is corrected, the action can be safely retried.
956              
957             =head1 USAGE
958              
959             If you are looking for a command line utility, have a look at the docs for prov_virtual. If you are looking to mate an existing Customer Relationship Manager (CRM) or billing system (like Ubersmith, WHMCS, Modernbill, etc..) with a rack full of hardware nodes, this class is it. There are two existing implementations, the prov_virtual CLI, and an RPC agent. The CLI and remote portion of the RPC agent is included in the distribution as bin/remoteagent.
960              
961             =head2 CLI
962              
963             The best way to interface with P:U:V is using a RPC agent to drive this class directly. However, doing so requires a programmer to write an application that accepts/processes requests from your CRMS system and formats them into P:U:V requests.
964              
965             If you don't have the resources to write your own RPC agent, and your CRM/billing software supports it, you may be able to dispatch the requests to the HW nodes via a terminal connection. If you do this, your CRM software will need to inspect the result code of the script to determine success or failure.
966              
967             P:U calls are quiet by default. If you want to see all the logging, append each CLI request with --verbose. Doing so will dump the audit and error reports to stdout.
968              
969             =head2 API
970              
971             The implementation I use implements RPC over SSH. The billing system we use is rather 'limited' so I wrote a separate request brokering application (maitred). It uses the billing systems API as a trigger to perform basic account management actions (create, destroy, enable, disable). We also provide a control panel so our clients can manage their VEs. The control panel also generates requests (start, stop, install, reinstall, reboot, upgrade, etc). Administrators also have their own control panel which also generates requests.
972              
973             When a request is initiated, the broker allocates any necessary resources (IPs, licences, a slot on a HW node, etc) and then dispatches the request. The dispatcher builds an appropriate SSH invocation that connects to the remote HW node and runs the remoteagent. Once connected to the remoteagent, the P:U:V class is loaded and its methods are invoked directly. The RPC agent checks the result code of each call, as well as the audit and error logs, feeding those request events back. The RPC local agent logs the request events into the request brokers database so there's a complete audit trail.
974              
975             =head2 RPC methods
976              
977             RPC is often implemented over HTTP, using SOAP or XML::RPC. However, our VEs are deployed with local storage. We needed the ability to move a VE from one node to another. In addition to the broker to node relationship, we would have also need temporary trust relationships between the nodes, in order to move files between them with root permissions.
978              
979             The trust relationships are much easier to manage with SSH keys. In our environment, only the request brokers are trusted. In addition to being able to connect to any node, they can also connect from node to node using ssh-agent and key forwarding.
980              
981             =head2 RPC gotcha
982              
983             The $vos->migrate() function expects to be running as a user that has the ability to initiate a SSH connection from the node on which it's running, to the node on which you are moving the VE. Our RPC agent connects to the HW nodes as the maitred user and then invokes the remoteagent using sudo. Our sudo config on the HW nodes looks like this:
984              
985             Cmnd_Alias MAITRED_CMND=/usr/bin/remoteagent, /usr/bin/prov_virtual
986             maitred ALL=NOPASSWD: SETENV: MAITRED_CMND
987              
988             Since the RPC remoteagent is running as root, the request broker has access to a wide variety of tools (tar over ssh pipe, rsync, etc) to move files from one node to another, without the nodes having any sort of trust relationship between them.
989              
990             =head1 FUNCTIONS
991              
992             $vos->create(
993             name => 42,
994             ip => '10.0.0.42',
995             hostname => 'vps.example.com',
996             disk_size => 4096, # 4GB
997             ram => 512,
998             template => 'debian-5-i386-default',
999             password => 't0ps3kretWerD',
1000             nameservers => '10.0.0.2 10.0.0.3',
1001             searchdomain => 'example.com',
1002             );
1003             $vos->start( name => 42 );
1004             $vos->stop( name => 42 );
1005             $vos->restart( name => 42 );
1006              
1007             $vos->enable( name => 42 );
1008             $vos->disable( name => 42 );
1009              
1010             $vos->set_hostname( name => 42, hostname => 'new-host.example.com' );
1011             $vos->set_nameservers( name => 42, nameservers => '10.1.1.3 10.1.1.4' );
1012             $vos->set_password( name => 42, password => 't0ps3kretWerD' );
1013             $vos->set_ssh_key( name => 42, ssh_key => 'ssh-rsa AAAAB3N..' );
1014              
1015             $vos->modify(
1016             name => 42,
1017             disk_size => 4000,
1018             hostname => 'new-host.example.com',
1019             ip => '10.1.1.43 10.1.1.44',
1020             ram => 768,
1021             );
1022              
1023             $vos->get_status( name => 42 );
1024             $vos->migrate( name => 42, new_node => '10.1.1.13' );
1025             $vos->destroy( name => 42 );
1026              
1027             =head2 create
1028              
1029             ##############
1030             # Usage : $vos->create( name => '42', ip=>'127.0.0.2' );
1031             # Purpose : create a virtual OS instance
1032             # Returns : true or undef on failure
1033             # Parameters :
1034             # Required : name - name/ID of the virtual OS
1035             # : ip - IP address(es), space delimited
1036             # Optional : hostname - the FQDN of the virtual OS
1037             # : disk_root - the root directory of the virt os
1038             # : disk_size - disk space allotment in MB
1039             # : ram - in MB
1040             # : cpu - how many CPU cores the VE can use/see
1041             # : template - a 'template' or tarball the OS is patterned after
1042             # : config - a config file with virtual specific settings
1043             # : password - the root/admin password for the virtual
1044             # : ssh_key - ssh public key for root user
1045             # : mac_address - the MAC adress to assign to the vif
1046             # : nameservers -
1047             # : searchdomain -
1048             # : kernel_version -
1049             # : skip_start - do not start the VE after creation
1050              
1051             =head2 start
1052              
1053             # Usage : $vos->start( name => '42' );
1054             # Purpose : start a virtual OS instance
1055             # Returns : true or undef on failure
1056             # Parameters :
1057             # Required : name
1058              
1059             =head2 stop
1060              
1061             # Usage : $vos->stop( name => '42' );
1062             # Purpose : stop a virtual OS instance
1063             # Returns : true or undef on failure
1064             # Parameters :
1065             # Required : name
1066              
1067             =head2 restart
1068              
1069             # Usage : $vos->restart( name => '42' );
1070             # Purpose : restart a virtual OS instance
1071             # Returns : true or undef on failure
1072             # Parameters :
1073             # Required : name
1074              
1075             =head2 enable
1076              
1077             # Usage : $vos->enable( name => '42' );
1078             # Purpose : enable/reactivate/unsuspend a virtual OS instance
1079             # Returns : true or undef on failure
1080             # Parameters :
1081             # Required : name
1082              
1083             =head2 disable
1084              
1085             # Usage : $vos->disable( name => '42' );
1086             # Purpose : disable a virtual OS instance
1087             # Returns : true or undef on failure
1088             # Parameters :
1089             # Required : name
1090              
1091             =head2 set_hostname
1092              
1093             # Usage : $vos->set_hostname(
1094             # name => '42',
1095             # hostname => '42.example.com',
1096             # );
1097             # Purpose : update the hostname of a VE
1098             # Returns : true or undef on failure
1099             # Parameters :
1100             # Required : name
1101             # : hostname - the new FQDN for the virtual OS
1102              
1103             =head2 set_nameservers
1104              
1105             # Usage : $vos->set_nameservers(
1106             # name => '42',
1107             # nameservers => '10.0.1.4 10.0.1.5',
1108             # searchdomain => 'example.com',
1109             # );
1110             # Purpose : update the nameservers in /etc/resolv.conf
1111             # Returns : true or undef on failure
1112             # Parameters :
1113             # Required : name
1114             # : nameservers - space delimited list of IPs
1115             # Optional : searchdomain - space delimited list of domain names
1116              
1117             =head2 set_password
1118              
1119             # Usage : $vos->set_password(
1120             # name => '42',
1121             # password => 't0ps3kretWerD',
1122             # );
1123             # Purpose : update the password of a user inside a VE
1124             # Returns : true or undef on failure
1125             # Parameters :
1126             # Required : name
1127             # : password - the plaintext password to store in /etc/shadow|passwd
1128             # Optional : user - /etc/password user name, defaults to 'root'
1129             # : ssh_key - an ssh public key, to install in ~/.ssh/authorized_keys
1130             # : disk_root- the full to the VE root (ie, / within the VE)
1131              
1132             =head2 set_ssh_key
1133              
1134             # Usage : $vos->set_ssh_key(
1135             # name => '42',
1136             # ssh_key => 'ssh-rsa AAAA.....',
1137             # );
1138             # Purpose : install an SSH key for a user inside a VE
1139             # Returns : true or undef on failure
1140             # Parameters :
1141             # Required : name
1142             # : ssh_key - an ssh public key, to install in ~/.ssh/authorized_keys
1143             # Optional : user - /etc/password user name, defaults to 'root'
1144             # : disk_root- the full to the VE root (ie, / within the VE)
1145              
1146             =head2 modify
1147              
1148             # Usage : $vos->modify( name => '42' );
1149             # Purpose : modify a VE
1150             # Returns : true or undef on failure
1151             # Parameters :
1152             # Required : name
1153             # : disk_size
1154             # : hostname
1155             # : ip
1156             # : ram
1157             # Optional : config
1158             # : cpu
1159             # : disk_root
1160             # : mac_address
1161             # : nameservers
1162             # : password
1163             # : searchdomain
1164             # : ssh_key
1165             # : template
1166              
1167             =head2 get_status
1168              
1169             # Usage : $vos->get_status( name => '42' );
1170             # Purpose : get information about a VE
1171             # Returns : a hashref with state info about a VE
1172             # Parameters :
1173             # Required : name
1174             #
1175             # Example result object:
1176             # {
1177             # 'dom_id' => '42',
1178             # 'disk_use' => 560444,
1179             # 'disks' => [
1180             # 'phy:/dev/vol00/42_rootimg,sda1,w',
1181             # 'phy:/dev/vol00/42_vmswap,sda2,w'
1182             # ],
1183             # 'ips' => '10.0.1.42',
1184             # 'cpu_time' => '2699.9',
1185             # 'mem' => 256,
1186             # 'cpus' => '2',
1187             # 'state' => 'running'
1188             # }
1189              
1190             =head2 migrate
1191              
1192             # Usage : $vos->migrate( name => '42', new_node => 'xen5' );
1193             # Purpose : move a VE from one HW node to another
1194             # Returns : true or undef on failure
1195             # Parameters :
1196             # Required : name
1197             # : new_node - hostname of the new node
1198             # Optional : connection_test - don't migrate, just test SSH connectivity
1199             # between the existing and new HW node
1200              
1201             =head2 destroy
1202              
1203             # Usage : $vos->destroy( name => 42 );
1204             # Purpose : destroy a virtual OS instance
1205             # Returns : true or undef on failure
1206             # Parameters :
1207             # Required : name
1208              
1209             =head2 mount
1210              
1211             =head2 unmount
1212              
1213             =head2 publish_arp
1214              
1215             # Usage : $vos->publish_arp( ip => '10.1.0.42' );
1216             # Purpose : update our neighbors with an ARP request for the provided IP(s)
1217             # Parameters :
1218             # Required : ip, can be a string with one IP, or an arrayref
1219              
1220             =head2 create_snapshot
1221              
1222             Create a snapshot of the VE. Only applies to VEs with logical volumes (LVM)
1223              
1224             =head2 destroy_snapshot
1225              
1226             Create disk snapshots. Opposite of create_snapshot.
1227              
1228             =head2 mount_snapshot
1229              
1230             After a snapshot is created, it can be mounted with this method. For xen VEs, the volume is mounted in ~/mnt, which usually looks like this: /home/xen/42/snap
1231              
1232             =head2 unmount_snapshot
1233              
1234             unmounts a snapshot.
1235              
1236             =head2 get_config
1237              
1238             returns an array representing with each line in the VE config file being an element in the array.
1239              
1240             =head1 BUGS
1241              
1242             Please report any bugs or feature requests to C<bug-unix-provision-virtualos at rt.cpan.org>, or through the web interface at L<http://rt.cpan.org/NoAuth/ReportBug.html?Queue=Provision-Unix>. I will be notified, and then you'll automatically be notified of progress on your bug as I make changes.
1243              
1244             =head1 SUPPORT
1245              
1246             You can find documentation for this module with the perldoc command.
1247              
1248             perldoc Provision::Unix::VirtualOS
1249              
1250             You can also look for information at:
1251              
1252             =over 4
1253              
1254             =item * RT: CPAN's request tracker
1255              
1256             L<http://rt.cpan.org/NoAuth/Bugs.html?Dist=Provision-Unix-VirtualOS>
1257              
1258             =item * AnnoCPAN: Annotated CPAN documentation
1259              
1260             L<http://annocpan.org/dist/Provision-Unix-VirtualOS>
1261              
1262             =item * CPAN Ratings
1263              
1264             L<http://cpanratings.perl.org/d/Provision-Unix-VirtualOS>
1265              
1266             =item * Search CPAN
1267              
1268             L<http://search.cpan.org/dist/Provision-Unix-VirtualOS>
1269              
1270             =back
1271              
1272             =head1 AUTHOR
1273              
1274             Matt Simerson <msimerson@cpan.org>
1275              
1276             =head1 COPYRIGHT AND LICENSE
1277              
1278             This software is copyright (c) 2013 by The Network People, Inc..
1279              
1280             This is free software; you can redistribute it and/or modify it under
1281             the same terms as the Perl 5 programming language system itself.
1282              
1283             =cut
1284              
1285              
1286             __END__
1287              
1288              
1289