File Coverage

blib/lib/Pg/Reindex.pm
Criterion Covered Total %
statement 12 14 85.7
branch n/a
condition n/a
subroutine 5 5 100.0
pod n/a
total 17 19 89.4


line stmt bran cond sub pod time code
1             package Pg::Reindex;
2 1     1   21299 use Exporter qw/import/;
  1         3  
  1         64  
3             @ISA = qw(Exporter);
4             @EXPORT_OK = qw(prepare rebuild);
5              
6 1     1   19 use 5.010001;
  1         3  
7 1     1   5 use strict;
  1         6  
  1         26  
8 1     1   5 use warnings;
  1         1  
  1         26  
9              
10 1     1   576 use DBI;
  0            
  0            
11             use DBD::Pg qw/:async/;
12             use AnyEvent;
13             use POSIX qw/SIGTERM SIGINT SIG_BLOCK SIG_UNBLOCK/;
14             use Getopt::Long;
15             use Pod::Usage;
16             use Time::HiRes ();
17              
18             our ( $dbh, $opt_dryrun, $opt_throttle_on, $opt_throttle_off );
19             our $VERSION = '0.01';
20              
21             sub lg {
22             return print localtime() . ": ", @_;
23             }
24              
25             sub query {
26             my ( $descr, @param ) = @_;
27             my $sql = pop @param;
28              
29             # warn $sql;
30              
31             my $tm = Time::HiRes::time;
32              
33             my $stmt
34             = ref $sql ? $sql : $dbh->prepare( $sql, { pg_async => PG_ASYNC } );
35              
36             my $done = AE::cv;
37             my $cancel = sub {
38             $dbh->pg_cancel if $dbh->{pg_async_status} == 1;
39             $done->send;
40             };
41             my $pg_w = AE::io $dbh->{pg_socket}, 0, sub {
42             $dbh->pg_ready and $done->send;
43             };
44              
45             my $sigblock = POSIX::SigSet->new( SIGTERM, SIGINT );
46             POSIX::sigprocmask SIG_BLOCK, $sigblock;
47             my @sig_w = map { AE::signal $_, $cancel } qw/TERM INT/;
48             $stmt->execute(@param);
49             POSIX::sigprocmask SIG_UNBLOCK, $sigblock;
50              
51             $done->wait;
52              
53             die "query cancelled\n" unless $dbh->{pg_async_status} == 1;
54              
55             my $rc = $dbh->pg_result;
56             my $result = $stmt->{Active} ? $stmt->fetchall_arrayref : undef;
57              
58             lg sprintf "$descr took %.3f s\n", ( Time::HiRes::time- $tm ) if $descr;
59              
60             return wantarray ? ( $rc, $result ) : $result;
61             }
62              
63             sub wquery { ## no critic
64             goto \&query unless $opt_dryrun;
65              
66             my ( $descr, @param ) = @_;
67             my $sql = pop @param;
68              
69             my $n = 1;
70             $sql =~ s/\?/'$'.$n++/ge;
71             $sql =~ s/\$(\d+)/$dbh->quote($param[$1-1])/ge;
72              
73             print "$sql;\n";
74              
75             return 1;
76             }
77              
78             sub throttle {
79             return if $opt_dryrun;
80             state $q = $dbh->prepare( <<'SQL', { pg_async => PG_ASYNC } );
81             SELECT coalesce(max(pg_xlog_location_diff(pg_current_xlog_location(), r.flush_location)), 0)
82             FROM pg_stat_replication r
83             SQL
84              
85             my ($xlog_diff) = @{ query( '', $q )->[0] };
86              
87             if ( $xlog_diff > $opt_throttle_on ) {
88             lg "streaming lag = $xlog_diff ==> pausing\n";
89             LOOP: {
90             do {
91             select undef, undef, undef, 1; ## no critic
92             ($xlog_diff) = @{ query( '', $q )->[0] };
93             } while ( $xlog_diff > $opt_throttle_off );
94              
95             # sleep for another 30 sec and check every second the lag.
96             # sometimes the wal sender process disconnects and reconnects
97             # a moment later. In that case we may have fallen below the
98             # throttle limit simply because we checked at the wrong time.
99             for ( my $i = 0; $i < 30; $i++ ) {
100             select undef, undef, undef, 1; ## no critic
101             ($xlog_diff) = @{ query( '', $q )->[0] };
102             redo LOOP if $xlog_diff > $opt_throttle_off;
103             }
104             }
105             lg "streaming lag = $xlog_diff -- continuing\n";
106             }
107             return;
108             }
109              
110             sub next_index {
111             my @list = query '', <<'SQL';
112             WITH wl AS (
113             UPDATE reindex.worklist
114             SET status='in progress'
115             WHERE idx=(SELECT idx
116             FROM reindex.worklist
117             WHERE status<>'done'
118             ORDER BY ord
119             LIMIT 1)
120             RETURNING idx
121             )
122             SELECT c.oid, n.nspname, quote_ident(n.nspname), c.relname, quote_ident(c.relname),
123             pg_catalog.pg_get_indexdef(c.oid) indexdef,
124             pg_catalog.pg_relation_size(c.oid::regclass)
125             FROM pg_catalog.pg_class c
126             JOIN pg_catalog.pg_namespace n ON c.relnamespace=n.oid
127             JOIN wl ON wl.idx=c.oid
128             SQL
129              
130             return @{ $list[1]->[0] || [] };
131             }
132              
133             sub do_transaction {
134             my ( $stmt_pointer, $limit, $sub ) = @_;
135              
136             LOOP: {
137             eval {
138             if ($opt_dryrun) {
139             wquery '', 'BEGIN ISOLATION LEVEL REPEATABLE READ';
140             } else {
141             $dbh->begin_work;
142             wquery '', 'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ';
143             }
144              
145             $sub->();
146              
147             if ($opt_dryrun) {
148             wquery '', 'COMMIT';
149             } else {
150             $dbh->commit;
151             }
152             1;
153             } or do {
154             my $sqlstate = $dbh->state;
155             my $err = $@;
156             eval { $dbh->rollback }; ## no critic
157             if ( $limit-- ) {
158             for my $state (qw/40P01 40001/)
159             { # deadlock detected; serialization failure
160             if ( $sqlstate eq $state ) {
161             $err = ">>$$stmt_pointer<<\n$err" if $stmt_pointer;
162             $err =~ s/\s+$//;
163             $err =~ s/\n/\n /g;
164             lg
165             " SQL state $state ==> retry transaction\n $err\n";
166             redo LOOP;
167             }
168             }
169             }
170             die $err;
171             };
172             }
173             return;
174             }
175              
176             sub wait_for_concurrent_tx {
177             return if $opt_dryrun;
178             eval {
179             $dbh->begin_work;
180              
181             while (
182             !query( '',
183             'SELECT txid_current()=txid_snapshot_xmin(txid_current_snapshot())'
184             )->[0]->[0]
185             )
186             {
187             select undef, undef, undef, .5; ## no critic
188             }
189              
190             $dbh->rollback;
191             1;
192             } or do {
193             my $err = $@;
194             eval { $dbh->rollback }; ## no critic
195             die $err;
196             };
197             return;
198             }
199              
200             sub reindex {
201             my ( $oid, $nspname, $quoted_nspname, $idxname, $quoted_idxname, $idxdef,
202             $size, $opt_validate )
203             = @_;
204              
205             throttle; # wait for streaming replicas to catch up
206              
207             lg "Rebuilding Index $quoted_nspname.$quoted_idxname\n";
208              
209             my @log_id;
210             @log_id = query '', $oid, <<'SQL' unless $opt_dryrun;
211             INSERT INTO reindex.log(tstmp, nspname, tblname, idxname, sz_before, status)
212             SELECT now(), n.nspname, tc.relname, ic.relname, pg_catalog.pg_relation_size(i.indexrelid::regclass), 'started'
213             FROM pg_catalog.pg_index i
214             JOIN pg_catalog.pg_class ic ON i.indexrelid=ic.oid
215             JOIN pg_catalog.pg_class tc ON i.indrelid=tc.oid
216             JOIN pg_catalog.pg_namespace n ON ic.relnamespace=n.oid
217             WHERE i.indexrelid=$1
218             RETURNING id
219             SQL
220              
221             my $tmp = '__temp_reidx';
222             $idxdef =~ s/^(CREATE (?:UNIQUE )?INDEX) (\S+)/$1 CONCURRENTLY $tmp/
223             or die "Cannot replace index name in $idxdef\n";
224              
225             my $retry = 5;
226             my ( @rc, $err );
227             while ( --$retry > 0 ) {
228             @rc = eval { wquery " CREATE CONCURRENTLY", $idxdef } and last;
229             $err = $@;
230             eval {
231             query "$quoted_nspname.$quoted_idxname creation failed. Dropping",
232             qq{DROP INDEX $quoted_nspname.$tmp};
233             1;
234             }
235             or warn "While dropping the index: $@";
236             }
237              
238             unless ( $rc[0] ) {
239             chomp $err;
240              
241             query '', $log_id[1]->[0]->[0], $err, <<'SQL' unless $opt_dryrun;
242             UPDATE reindex.log
243             SET status='failed to create temp index: ' || $2,
244             tm_taken=now()-tstmp
245             WHERE id=$1
246             SQL
247              
248             die "Cannot create index: $err";
249             }
250              
251             my @revalidate;
252             my $current_cmd;
253              
254             eval {
255             do_transaction \$current_cmd, 100, sub {
256              
257             # check if the index still exists
258             @rc = query '', $oid, $nspname, $idxname, $current_cmd = <<'SQL';
259             SELECT 1
260             FROM pg_catalog.pg_class c
261             JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace
262             WHERE c.oid=$1
263             AND c.relkind='i'
264             AND n.nspname=$2
265             AND c.relname=$3
266             SQL
267              
268             if ( $rc[1]->[0]->[0] == 1 ) {
269             unless ($opt_dryrun) {
270             @rc = query '', $log_id[1]->[0]->[0],
271             "$quoted_nspname.$tmp", $current_cmd = <<'SQL';
272             UPDATE reindex.log
273             SET sz_after=pg_catalog.pg_relation_size($2::regclass)
274             WHERE id=$1
275             RETURNING sz_after
276             SQL
277             lg sprintf(
278             " size: %d ==> %d (%.2f%%)\n",
279             $size,
280             $rc[1]->[0]->[0],
281             $rc[1]->[0]->[0] * 100 / $size - 100
282             );
283             }
284              
285             my @trans
286             = (
287             "ALTER INDEX $quoted_nspname.$tmp RENAME TO $quoted_idxname"
288             );
289             @rc = query '', $oid, <<'SQL';
290             SELECT con.conname, quote_ident(con.conname),
291             n.nspname, quote_ident(n.nspname),
292             con.contype,
293             pg_get_constraintdef(con.oid),
294             con.conrelid::regclass::text,
295             con.confrelid::regclass::text,
296             con.confmatchtype,
297             key.key, fkey.fkey
298             FROM pg_catalog.pg_constraint con
299             JOIN pg_catalog.pg_namespace n ON n.oid=con.connamespace
300             CROSS JOIN LATERAL (SELECT array_agg(quote_ident(a.attname))
301             FROM unnest(con.conkey) x(k) JOIN pg_catalog.pg_attribute a
302             ON x.k=a.attnum AND a.attrelid=con.conrelid) key(key)
303             CROSS JOIN LATERAL (SELECT array_agg(quote_ident(a.attname))
304             FROM unnest(con.confkey) x(k) JOIN pg_catalog.pg_attribute a
305             ON x.k=a.attnum AND a.attrelid=con.confrelid) fkey(fkey)
306             JOIN (VALUES ('p'::TEXT, 1::INT),
307             ('u'::TEXT, 2::INT),
308             ('f'::TEXT, 3::INT)) ord(type, ord) ON ord.type=con.contype
309             WHERE con.conindid=$1
310             AND con.contype<>'x' -- exclusion constraints are not yet implemented
311             ORDER BY ord.ord ASC
312             SQL
313              
314             if ( @{ $rc[1] } ) {
315             for my $con ( @{ $rc[1] } ) {
316             my ($conname, $quoted_conname, $nspname,
317             $quoted_nspname, $contype, $condef,
318             $rel, $frel, $matchtype,
319             $key, $fkey
320             ) = @$con;
321             if ( $contype eq 'u' ) {
322             unshift @trans,
323             "ALTER TABLE $rel DROP CONSTRAINT $quoted_conname";
324             push @trans,
325             (
326             "ALTER TABLE $rel ADD CONSTRAINT $quoted_conname "
327             . "UNIQUE USING INDEX $quoted_idxname" );
328             } elsif ( $contype eq 'p' ) {
329             unshift @trans,
330             "ALTER TABLE $rel DROP CONSTRAINT $quoted_conname";
331             push @trans,
332             (
333             "ALTER TABLE $rel ADD CONSTRAINT $quoted_conname "
334             . "PRIMARY KEY USING INDEX $quoted_idxname"
335             );
336             } elsif ( $contype eq 'f' ) {
337             unshift @trans,
338             "ALTER TABLE $rel DROP CONSTRAINT $quoted_conname";
339             push @trans,
340             "ALTER TABLE $rel ADD CONSTRAINT $quoted_conname $condef NOT VALID";
341             push @revalidate,
342             $con; # needs to be revalidated after commit
343             } elsif ( $contype eq 'x' ) {
344             ...; # exclusion constraints are not yet implemented
345             } else {
346             }
347             }
348             } else {
349             unshift @trans,
350             "DROP INDEX $quoted_nspname.$quoted_idxname";
351             }
352              
353             wquery '', $current_cmd = $_ for (@trans);
354             } else {
355             eval {
356             query
357             "Index $quoted_nspname.$quoted_idxname has vanished. Dropping temporary",
358             qq{DROP INDEX $quoted_nspname.$tmp};
359             1;
360             }
361             or warn "While dropping the index: $@";
362             }
363             };
364              
365             1;
366             } or do {
367             my $err = $@;
368             chomp $err;
369             $err = ">>$current_cmd<<\n$err";
370              
371             eval {
372             query
373             "Transaction for $quoted_nspname.$quoted_idxname failed. Dropping",
374             qq{DROP INDEX $quoted_nspname.$tmp};
375             1;
376             }
377             or warn "While dropping the index: $@";
378              
379             query '', $log_id[1]->[0]->[0], $err, <<'SQL' unless $opt_dryrun;
380             UPDATE reindex.log
381             SET status='failed to rename index or recreate constraints: ' || $2,
382             tm_taken=now()-tstmp
383             WHERE id=$1
384             SQL
385              
386             die "$err";
387             };
388              
389             unless ($opt_validate) {
390             query '', $log_id[1]->[0]->[0], $err, <<'SQL' unless $opt_dryrun;
391             UPDATE reindex.log
392             SET status='done: constraints not validated',
393             tm_taken=now()-tstmp
394             WHERE id=$1
395             SQL
396              
397             return 1;
398             }
399              
400             wait_for_concurrent_tx;
401              
402             my @not_validated;
403             for my $con (@revalidate) {
404             my ($conname, $quoted_conname, $nspname, $quoted_nspname,
405             $contype, $condef, $rel, $frel,
406             $matchtype, $key, $fkey
407             ) = @$con;
408             my $sql;
409              
410             my $join_cond
411             = '('
412             . join( ', ', map {"b.$_"} @$fkey ) . ')=('
413             . join( ', ', map {"a.$_"} @$key ) . ')';
414             my $match = (
415             $matchtype eq 's' # MATCH SIMPLE
416             ? 'AND '
417             : $matchtype eq 'f' # MATCH FULL
418             ? ' OR '
419             : do {...}
420             ); # MATCH PARTIAL not yet implemented by PG
421             $match = join( $match, map {"a.$_ IS NOT NULL"} @$key );
422              
423             @rc = wquery " Validate FK constraint $quoted_conname on $rel",
424             $rel, $conname, <<"SQL";
425             UPDATE pg_catalog.pg_constraint
426             SET convalidated = NOT EXISTS(SELECT 1
427             FROM ONLY $rel a
428             LEFT JOIN ONLY $frel b ON $join_cond
429             WHERE b.$fkey->[0] IS NULL -- inner join failed
430             AND ($match))
431             WHERE conrelid=\$1::regclass::oid
432             AND conname=\$2
433             RETURNING convalidated
434             SQL
435             unless ($opt_dryrun) {
436             lg ' ' . ( $rc[1]->[0]->[0] ? '' : 'NOT ' ) . "VALID\n";
437             push @not_validated, $quoted_conname unless $rc[1]->[0]->[0];
438             }
439             }
440              
441             unless ($opt_dryrun) {
442             if (@not_validated) {
443             query '', $log_id[1]->[0]->[0],
444             '[' . join( '], [', @not_validated ) . ']', <<'SQL';
445             UPDATE reindex.log
446             SET status='failed: some constraints could not be validated: ',
447             tm_taken=now()-tstmp
448             WHERE id=$1
449             SQL
450             } else {
451             query '', $log_id[1]->[0]->[0], <<'SQL';
452             UPDATE reindex.log
453             SET status='done',
454             tm_taken=now()-tstmp
455             WHERE id=$1
456             SQL
457             }
458             }
459              
460             return 1;
461             }
462              
463             sub init {
464              
465             }
466              
467             sub prepare {
468              
469             $dbh = shift;
470             my @opt_namespaces = @{ shift @_ };
471             my @opt_tables = @{ shift @_ };
472             my @opt_indexes = @{ shift @_ };
473              
474             my $qual;
475             my @param;
476              
477             query '', 'SET client_min_messages TO WARNING';
478              
479             query '', 'CREATE SCHEMA IF NOT EXISTS reindex';
480              
481             query '', <<'SQL';
482             CREATE TABLE IF NOT EXISTS reindex.log(
483             id BIGSERIAL PRIMARY KEY,
484             tstmp TIMESTAMP,
485             nspname NAME,
486             tblname NAME,
487             idxname NAME,
488             sz_before BIGINT,
489             sz_after BIGINT,
490             status TEXT,
491             tm_taken INTERVAL
492             )
493             SQL
494              
495             query '', <<'SQL';
496             CREATE UNLOGGED TABLE IF NOT EXISTS reindex.worklist(
497             ord SERIAL,
498             idx OID,
499             status TEXT
500             )
501             SQL
502              
503             if (@opt_namespaces) {
504             $qual .= " AND n.nspname IN ("
505             . join( ', ', ('?') x ( 0 + @opt_namespaces ) ) . ")\n";
506             push @param, @opt_namespaces;
507             } else {
508             $qual .= <<'SQL';
509             AND n.nspname !~ '^pg_'
510             AND n.nspname <> 'information_schema'
511             AND n.nspname <> 'reindex'
512             SQL
513             }
514              
515             if (@opt_tables) {
516             $qual .= " AND EXISTS(SELECT 1
517             FROM pg_catalog.pg_class xc
518             JOIN pg_catalog.pg_index xi ON xc.oid=xi.indrelid
519             WHERE xi.indexrelid=c.oid
520             AND xc.relname IN ("
521             . join( ', ', ('?') x ( 0 + @opt_tables ) ) . "))\n";
522             push @param, @opt_tables;
523             }
524              
525             if (@opt_indexes) {
526             $qual .= " AND c.relname IN ("
527             . join( ', ', ('?') x ( 0 + @opt_indexes ) ) . ")\n";
528             push @param, @opt_indexes;
529             }
530              
531             query '', 'TRUNCATE reindex.worklist';
532             query '',
533             q{SELECT pg_catalog.setval('reindex.worklist_ord_seq', 1, false)};
534              
535             query '', @param, <<'SQL'. $qual . <<'SQL';
536             INSERT INTO reindex.worklist(idx, status)
537             SELECT c.oid, 'planned'
538             FROM pg_catalog.pg_class c
539             JOIN pg_catalog.pg_namespace n ON c.relnamespace=n.oid
540             WHERE c.relkind = 'i'
541             SQL
542             ORDER BY n.nspname, c.relname
543             SQL
544             return;
545             }
546              
547             sub rebuild {
548              
549             my ( $options, $opt_validate );
550              
551             ( $dbh, $options, $opt_dryrun ) = @_;
552              
553             $opt_validate = $options->{Validate};
554             $opt_throttle_on = $options->{ThrottleOn} || 10000000;
555             $opt_throttle_off = $options->{ThrottleOff} || 100000;
556              
557             while ( my @idx = next_index() ) {
558             reindex( @idx, $opt_validate );
559             query '', $idx[0],
560             q{UPDATE reindex.worklist SET status='done' WHERE idx=$1};
561             }
562             }
563              
564             sub run {
565             my ($opt_help, $opt_dbname, $opt_server,
566             $opt_user, $opt_pwd, $opt_port,
567             @opt_namespaces, @opt_tables, @opt_indexes,
568             $opt_validate, $opt_dryrun, $opt_throttle_on,
569             $opt_throttle_off
570             );
571             ( $opt_user, $opt_port, $opt_server, $opt_validate, $opt_throttle_on,
572             $opt_throttle_off
573             ) = (qw/postgres 5432 localhost 1 10000000 100000/);
574              
575             GetOptions(
576             'help' => \$opt_help,
577             'database|dbname=s' => \$opt_dbname,
578             'server=s' => \$opt_server,
579             'user=s' => \$opt_user,
580             'password=s' => \$opt_pwd,
581             'port=s' => \$opt_port,
582             'table=s' => \@opt_tables,
583             'namespace=s' => \@opt_namespaces,
584             'index=s' => \@opt_indexes,
585             'validate!' => \$opt_validate,
586             'dryrun!' => \$opt_dryrun,
587             'high-txn-lag=i' => \$opt_throttle_on,
588             'low-txn-lag=i' => \$opt_throttle_off,
589             )
590             || pod2usage(
591             -exitval => 1,
592             -verbose => 2
593             );
594              
595             my $action = $ARGV[0];
596             if ($action) {
597             if ( $action =~ /^p(?:r(?:e(?:p(?:a(?:re?)?)?)?)?)?$/i ) {
598             $action = 'prepare';
599             } elsif ( $action =~ /^c(?:o(?:n(?:t(?:i(?:n(?:ue?)?)?)?)?)?)?$/i ) {
600             $action = 'continue';
601             } else {
602             pod2usage(
603             -exitval => 1,
604             -verbose => 2
605             );
606             }
607             }
608              
609             pod2usage(
610             -exitval => 0,
611             -verbose => 2
612             ) if $opt_help;
613              
614             {
615             my $fh;
616             if ( !defined $opt_pwd ) {
617              
618             # noop
619             } elsif ( $opt_pwd =~ /^\d+$/ ) {
620             open $fh, '<&=' . $opt_pwd ## no critic
621             or die "Cannot open file descriptor $opt_pwd: $!\n";
622             $opt_pwd = readline $fh;
623             chomp $opt_pwd;
624             } else {
625             open $fh, '<', $opt_pwd or die "Cannot open $opt_pwd: $!\n";
626             $opt_pwd = readline $fh;
627             chomp $opt_pwd;
628             }
629             close $fh;
630             }
631              
632             my $dbh = DBI->connect(
633             "dbi:Pg:database=$opt_dbname;host=$opt_server;port=$opt_port;sslmode=prefer",
634             $opt_user,
635             $opt_pwd,
636             { pg_server_prepare => 0,
637             PrintError => 0,
638             RaiseError => 1,
639             }
640             );
641              
642             prepare( $dbh, \@opt_namespaces, \@opt_tables, \@opt_indexes )
643             unless defined $action and $action eq 'continue';
644              
645             rebuild(
646             $dbh,
647             { ThrottleOn => $opt_throttle_on,
648             ThrottleOff => $opt_throttle_off,
649             Validate => $opt_validate
650             },
651             $opt_dryrun
652             ) if !defined $action or $action eq 'continue';
653              
654             $dbh->disconnect;
655             exit 0;
656             }
657              
658             __PACKAGE__->run unless caller;
659              
660             =head1 NAME
661              
662             Pg::Reindex - rebuild postgresql indexes concurrently without locking.
663              
664             =head1 VERSION
665              
666             Version 0.01
667              
668             =head1 SYNOPSIS
669              
670             =head3 Use as a module
671              
672             use Pg::Reindex qw(prepare rebuild);
673              
674             prepare($dbh, \@namespaces, \@tables, \@indexes);
675             rebuild($dbh, \%options, $dryrun);
676              
677             =head3 Run as a perl script
678              
679             perl `perldoc -l Pg::Reindex` \
680             [--help] \
681             [--server=localhost] \
682             [--port=5432] \
683             [--user=postgres] \
684             [--password=PASSWORD] \
685             [--table=TABLE] ... \
686             [--namespace=NAMESPACE] ... \
687             [--index=INDEX] ... \
688             [--[no]validate] \
689             [--high_txn_lag=BYTES] \
690             [--log_txn_lag=BYTES] \
691             [--[no]dryrun] \
692             [prepare|continue]
693              
694             =head1 DESCRIPTION
695              
696             Postgresql indexes should be rebuilt on a regular basis for good performance.
697             This can be done with the C command, however, building indexes this way
698             requires an exclusive lock on the table. On the other hand, using
699             C avoids this lock.
700              
701             C builds new indexes using C. Then it
702             starts a transaction for each index in which it drops the old index and
703             renames the new one.
704              
705             It handles normal indexes and C, C and C
706             constraints.
707              
708             =head2 Streaming replication and throttling
709              
710             Before creating the next index, the streaming replication lag is checked to
711             be below a certain limit. If so, nothing special happens and the index is
712             built.
713              
714             Otherwise, C waits for the replicas to catch up. When the lag
715             drops under a second limit, the C does not immediately continue.
716             Instead it waits for another 30 seconds and checks the lag every second
717             within that period. Only if the lag stays below the limit for the whole
718             time, execution is continued. This grace period is to deal with the fact
719             that a wal sender process may suddenly disappear and reappear after a
720             few seconds. Without the grace period the program may encounter a false
721             drop below the limit and hence continue. For large indexes this adds a
722             lot of lag.
723              
724             =head1 USING AS A MODULE
725              
726             To use Pg::Reindex as a module, first you need to load the
727             Pg::Reindex module:
728              
729             use Pg::Reindex qw(prepare rebuild);
730             use strict;
731              
732             (The C isn't required but is strongly recommended.)
733              
734             Then you need to L the indexes that you want rebuilt.
735             You can filter by combinations of namespace, tables, and indexes.
736              
737             prepare($dbh, \@opt_namespaces,\@opt_tables, \@opt_indexes);
738              
739             After "preparing" the set of indexes to be rebuilt, then you rebuild them:
740              
741             rebuild( $dbh, { ThrottleOn => 10000000,
742             ThrottleOff => 100000, Validate => 1 }, $opt_dryrun);
743              
744             =head2 SUBROUTINES/METHODS
745              
746             =head3 prepare
747              
748             C determines the list of indexes that would be re-indexed, and
749             sets up the data structures used by C. C must be called
750             before C is called.
751              
752             C creates a new schema named C with 2 tables,
753             C and C. C is created as C
754             table. C saves information on all indexes that need to be rebuilt
755             to C. The information in C is used by C.
756              
757             =over
758              
759             =item $dbh
760              
761             DBI database handle to the database whose indexes are to be reindexed.
762              
763             =item \@namespaces
764              
765             Rebuild only indexes in the C. If C is empty,
766             indexes in all namespaces except the following are considered:
767             those beginning with C, in Ci, or are
768             C namespaces.
769              
770             =item \@tables
771              
772             Rebuild only indexes that belong to the specified tables.
773              
774              
775             =item \@indexes
776              
777             List of indexes to reindex.
778              
779             =back
780              
781             If C, C and C are given simultaneously,
782             only indexes satisfying all conditions are considered.
783              
784             =head3 rebuild
785              
786             =over
787              
788             =item $dbh
789              
790             DBI database handle to the database whose indexes are to be reindexed.
791              
792             =item \%options
793              
794             ThrottleOn
795             ThrottleOff
796             Validate
797            
798             =item $dryrun
799              
800             =back
801              
802             =head1 USING AS A PERL SCRIPT
803              
804              
805             To use Pg::Reindex as a perl script you need to have perl run it. The command
806             below would do that by using C to determine C's location.
807              
808             perl `perldoc -l Pg::Reindex` \
809             [--help] \
810             [--server=localhost] \
811             [--port=5432] \
812             [--user=postgres] \
813             [--password=PASSWORD] \
814             [--table=TABLE] ... \
815             [--namespace=NAMESPACE] ... \
816             [--index=INDEX] ... \
817             [--[no]validate] \
818             [--high_txn_lag=BYTES] \
819             [--log_txn_lag=BYTES] \
820             [--[no]dryrun] \
821             [prepare|continue]
822              
823             =head2 OPTIONS
824              
825             Options can be abbreviated.
826              
827             =over 4
828              
829             =item --server
830              
831             Hostname / IP address or directory path to use to connect to the Postgres
832             server. If you want to use a local UNIX domain socket, specify the socket
833             directory path.
834              
835             Default: localhost
836              
837             =item --port
838              
839             The port to connect to.
840              
841             Default: 5432
842              
843             =item --user
844              
845             The user.
846              
847             Default: postgres
848              
849             =item --password
850              
851             a file name or open file descriptor where to read the password from.
852             If the parameter value consists of only digits, it's evaluated as file
853             descriptor.
854              
855             There is no default.
856              
857             A convenient way to specify the password on the BASH command line is
858              
859             reindex.pl --password=3 3<<
860              
861             That way the password appears in F<.bash_history>. But that file is
862             usually only readable to the owner.
863              
864             =item --table
865              
866             Reindex only indexes that belong to the specified table.
867              
868             This option can be given multiple times.
869              
870             If C<--table>, C<--namespace> and C<--index> are given simultaneously,
871             only indexes satisfying all conditions are considered.
872              
873             =item --namespace
874              
875             Without this option only namespaces are considered that are not in
876             beginning with C. Also C or C
877             namespaces are omitted.
878              
879             If C<--table>, C<--namespace> and C<--index> are given simultaneously,
880             only indexes satisfying all conditions are considered.
881              
882             =item --index
883              
884             If C<--table>, C<--namespace> and C<--index> are given simultaneously,
885             only indexes satisfying all conditions are considered.
886              
887             =item --[no]validate
888              
889             validate C constraints or leave them C. Default
890             it to validate.
891              
892             =item --[no]dryrun
893              
894             don't modify the database but print the essential SQL statements.
895              
896             =item --high-txn-lag
897              
898             the upper limit streaming replicas may lag behind in bytes.
899              
900             Default is 10,000,000.
901              
902             =item --low-txn-lag
903              
904             the lower limit in bytes when execution may be continued after it has been
905             interrupted due to exceeding C.
906              
907             Default is 100,000
908              
909             =item --help
910              
911             print this help
912              
913             =back
914              
915             =head1 AUTHOR
916              
917             BINARY, C<< >>
918              
919             =head1 BUGS
920              
921             Please report any bugs or feature requests to C, or through
922             the web interface at L. I will be notified, and then you'll
923             automatically be notified of progress on your bug as I make changes.
924              
925             =head1 LICENSE AND COPYRIGHT
926              
927             Copyright 2015 Binary Ltd.
928              
929             This program is free software; you can redistribute it and/or modify it
930             under the terms of the the Artistic License (2.0). You may obtain a
931             copy of the full license at:
932              
933             L
934              
935             Any use, modification, and distribution of the Standard or Modified
936             Versions is governed by this Artistic License. By using, modifying or
937             distributing the Package, you accept this license. Do not use, modify,
938             or distribute the Package, if you do not accept this license.
939              
940             If your Modified Version has been derived from a Modified Version made
941             by someone other than you, you are nevertheless required to ensure that
942             your Modified Version complies with the requirements of this license.
943              
944             This license does not grant you the right to use any trademark, service
945             mark, tradename, or logo of the Copyright Holder.
946              
947             This license includes the non-exclusive, worldwide, free-of-charge
948             patent license to make, have made, use, offer to sell, sell, import and
949             otherwise transfer the Package with respect to any patent claims
950             licensable by the Copyright Holder that are necessarily infringed by the
951             Package. If you institute patent litigation (including a cross-claim or
952             counterclaim) against any party alleging that the Package constitutes
953             direct or contributory patent infringement, then this Artistic License
954             to you shall terminate on the date that such litigation is filed.
955              
956             Disclaimer of Warranty: THE PACKAGE IS PROVIDED BY THE COPYRIGHT HOLDER
957             AND CONTRIBUTORS "AS IS' AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES.
958             THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
959             PURPOSE, OR NON-INFRINGEMENT ARE DISCLAIMED TO THE EXTENT PERMITTED BY
960             YOUR LOCAL LAW. UNLESS REQUIRED BY LAW, NO COPYRIGHT HOLDER OR
961             CONTRIBUTOR WILL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OR
962             CONSEQUENTIAL DAMAGES ARISING IN ANY WAY OUT OF THE USE OF THE PACKAGE,
963             EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
964              
965              
966             =cut
967              
968             1; # End of Pg::Reindex