line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
# $File: //depot/ebx/Sync.pm $ $Author: clkao $ |
2
|
|
|
|
|
|
|
# $Revision: #83 $ $Change: 2072 $ $DateTime: 2001/10/15 09:43:21 $ |
3
|
|
|
|
|
|
|
|
4
|
|
|
|
|
|
|
package OurNet::BBSApp::Sync; |
5
|
|
|
|
|
|
|
require 5.006; |
6
|
|
|
|
|
|
|
|
7
|
|
|
|
|
|
|
$VERSION = '0.87'; |
8
|
|
|
|
|
|
|
|
9
|
1
|
|
|
1
|
|
9874
|
use strict; |
|
1
|
|
|
|
|
3
|
|
|
1
|
|
|
|
|
50
|
|
10
|
1
|
|
|
1
|
|
977
|
use integer; |
|
1
|
|
|
|
|
12
|
|
|
1
|
|
|
|
|
7
|
|
11
|
|
|
|
|
|
|
|
12
|
1
|
|
|
1
|
|
1303
|
use IO::Handle; |
|
1
|
|
|
|
|
8087
|
|
|
1
|
|
|
|
|
57
|
|
13
|
1
|
|
|
1
|
|
1639
|
use Mail::Address; |
|
0
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
|
14
|
|
|
|
|
|
|
use OurNet::BBS; |
15
|
|
|
|
|
|
|
|
16
|
|
|
|
|
|
|
=head1 NAME |
17
|
|
|
|
|
|
|
|
18
|
|
|
|
|
|
|
OurNet::BBSApp::Sync - Sync between BBS article groups |
19
|
|
|
|
|
|
|
|
20
|
|
|
|
|
|
|
=head1 SYNOPSIS |
21
|
|
|
|
|
|
|
|
22
|
|
|
|
|
|
|
my $sync = OurNet::BBSApp::Sync->new({ |
23
|
|
|
|
|
|
|
artgrp => $local->{boards}{board1}{articles}, |
24
|
|
|
|
|
|
|
rartgrp => $remote->{boards}{board2}{articles}, |
25
|
|
|
|
|
|
|
param => { |
26
|
|
|
|
|
|
|
lseen => 0, |
27
|
|
|
|
|
|
|
rseen => 0, |
28
|
|
|
|
|
|
|
remote => 'bbs.remote.org', |
29
|
|
|
|
|
|
|
backend => 'BBSAgent', |
30
|
|
|
|
|
|
|
board => 'board2', |
31
|
|
|
|
|
|
|
lmsgid => '', |
32
|
|
|
|
|
|
|
msgids => { |
33
|
|
|
|
|
|
|
articles => [ |
34
|
|
|
|
|
|
|
'<20010610005743.6c+7nbaJ5I63v5Uq3cZxZw@geb.elixus.org>', |
35
|
|
|
|
|
|
|
'<20010608213307.suqAZQosHH7LxHCXVi1c9A@geb.elixus.org>', |
36
|
|
|
|
|
|
|
], |
37
|
|
|
|
|
|
|
archives => [ |
38
|
|
|
|
|
|
|
'<20010608213307.suqAZQosHH7LxHCXVi1c9A@geb.elixus.org>', |
39
|
|
|
|
|
|
|
'<20010608213307.suqAZQosHH7LxHCXVi1c9A@geb.elixus.org>', |
40
|
|
|
|
|
|
|
], |
41
|
|
|
|
|
|
|
}, |
42
|
|
|
|
|
|
|
}, |
43
|
|
|
|
|
|
|
force_fetch => 0, |
44
|
|
|
|
|
|
|
force_send => 0, |
45
|
|
|
|
|
|
|
force_none => 0, |
46
|
|
|
|
|
|
|
msgidkeep => 128, |
47
|
|
|
|
|
|
|
recursive => 0, |
48
|
|
|
|
|
|
|
clobber => 1, |
49
|
|
|
|
|
|
|
backend => 'BBSAgent', |
50
|
|
|
|
|
|
|
logfh => \*STDOUT, |
51
|
|
|
|
|
|
|
callback => sub { }, |
52
|
|
|
|
|
|
|
}); |
53
|
|
|
|
|
|
|
|
54
|
|
|
|
|
|
|
$sync->do_fetch('archives'); |
55
|
|
|
|
|
|
|
$sync->do_send; |
56
|
|
|
|
|
|
|
|
57
|
|
|
|
|
|
|
=head1 DESCRIPTION |
58
|
|
|
|
|
|
|
|
59
|
|
|
|
|
|
|
B performs a sophisticated synchronization algorithm |
60
|
|
|
|
|
|
|
on two L ArticleGroup objects. It operates on the first one |
61
|
|
|
|
|
|
|
(C)'s behalf, updates what's being done in the C field, |
62
|
|
|
|
|
|
|
and attempts to determine the minimally needed transactions to run. |
63
|
|
|
|
|
|
|
|
64
|
|
|
|
|
|
|
The two methods, L and L could be used independently. |
65
|
|
|
|
|
|
|
Beyond that, note that the interface might change in the future, and |
66
|
|
|
|
|
|
|
currently it's only a complement to the L toolkit. |
67
|
|
|
|
|
|
|
|
68
|
|
|
|
|
|
|
=head1 BUGS |
69
|
|
|
|
|
|
|
|
70
|
|
|
|
|
|
|
Lots. Please report bugs as much as possible. |
71
|
|
|
|
|
|
|
|
72
|
|
|
|
|
|
|
=cut |
73
|
|
|
|
|
|
|
|
74
|
|
|
|
|
|
|
use fields qw/artgrp rartgrp param backend logfh msgidkeep hostname |
75
|
|
|
|
|
|
|
force_send force_fetch force_none clobber recursive callback/; |
76
|
|
|
|
|
|
|
|
77
|
|
|
|
|
|
|
use constant SKIPPED_HEADERS => |
78
|
|
|
|
|
|
|
' name header xid id xmode idxfile time mtime btime basepath'. |
79
|
|
|
|
|
|
|
' dir hdrfile recno '; |
80
|
|
|
|
|
|
|
use constant SKIPPED_SIGILS => ' ¡» ¡· ¡º '; |
81
|
|
|
|
|
|
|
|
82
|
|
|
|
|
|
|
sub new { |
83
|
|
|
|
|
|
|
my $class = shift; |
84
|
|
|
|
|
|
|
my OurNet::BBSApp::Sync $self = fields::new($class); |
85
|
|
|
|
|
|
|
|
86
|
|
|
|
|
|
|
%{$self} = %{$_[0]}; |
87
|
|
|
|
|
|
|
|
88
|
|
|
|
|
|
|
$self->{msgidkeep} ||= 128; |
89
|
|
|
|
|
|
|
$self->{hostname} ||= $OurNet::BBS::Utils::hostname || 'localhost'; |
90
|
|
|
|
|
|
|
$self->{logfh} ||= IO::Handle->new->fdopen(fileno(STDOUT), 'w'); |
91
|
|
|
|
|
|
|
$self->{logfh}->autoflush(1); |
92
|
|
|
|
|
|
|
|
93
|
|
|
|
|
|
|
return $self; |
94
|
|
|
|
|
|
|
} |
95
|
|
|
|
|
|
|
|
96
|
|
|
|
|
|
|
# FIXME: use sorted array and bsearch here. |
97
|
|
|
|
|
|
|
sub nth { |
98
|
|
|
|
|
|
|
my ($ary, $ent) = @_; |
99
|
|
|
|
|
|
|
|
100
|
|
|
|
|
|
|
no warnings 'uninitialized'; |
101
|
|
|
|
|
|
|
|
102
|
|
|
|
|
|
|
foreach my $i (0 .. $#{$ary}) { |
103
|
|
|
|
|
|
|
return $i if $ary->[$i] eq $ent; |
104
|
|
|
|
|
|
|
} |
105
|
|
|
|
|
|
|
|
106
|
|
|
|
|
|
|
return -1; |
107
|
|
|
|
|
|
|
} |
108
|
|
|
|
|
|
|
|
109
|
|
|
|
|
|
|
sub do_retrack { |
110
|
|
|
|
|
|
|
my ($self, $rid, $myid, $low, $high) = @_; |
111
|
|
|
|
|
|
|
my $logfh = $self->{logfh}; |
112
|
|
|
|
|
|
|
|
113
|
|
|
|
|
|
|
return $low - 1 if $low > $high; |
114
|
|
|
|
|
|
|
|
115
|
|
|
|
|
|
|
my $try = ($low + $high) / 2; |
116
|
|
|
|
|
|
|
my $msgid = eval { |
117
|
|
|
|
|
|
|
my $art = $rid->[$try]; |
118
|
|
|
|
|
|
|
UNIVERSAL::isa($art, 'UNIVERSAL') |
119
|
|
|
|
|
|
|
? $art->{header}{'Message-ID'} : undef; |
120
|
|
|
|
|
|
|
}; |
121
|
|
|
|
|
|
|
|
122
|
|
|
|
|
|
|
return (($msgid && nth($myid, $msgid) == -1) |
123
|
|
|
|
|
|
|
? $low - 1 : $low) if $low == $high; |
124
|
|
|
|
|
|
|
|
125
|
|
|
|
|
|
|
$logfh->print(" [retrack] #$try: try in [$low - $high]\n"); |
126
|
|
|
|
|
|
|
|
127
|
|
|
|
|
|
|
if ($msgid and nth($myid, $msgid) != -1) { |
128
|
|
|
|
|
|
|
return $self->do_retrack($rid, $myid, $try + 1, $high); |
129
|
|
|
|
|
|
|
} |
130
|
|
|
|
|
|
|
else { |
131
|
|
|
|
|
|
|
return $self->do_retrack($rid, $myid, $low, $try - 1) |
132
|
|
|
|
|
|
|
} |
133
|
|
|
|
|
|
|
} |
134
|
|
|
|
|
|
|
|
135
|
|
|
|
|
|
|
sub retrack { |
136
|
|
|
|
|
|
|
my ($self, $rid, $myid, $rseen) = @_; |
137
|
|
|
|
|
|
|
my $logfh = $self->{logfh}; |
138
|
|
|
|
|
|
|
|
139
|
|
|
|
|
|
|
$logfh->print(" [retrack] #$rseen: checking\n"); |
140
|
|
|
|
|
|
|
|
141
|
|
|
|
|
|
|
return $rseen if (eval { |
142
|
|
|
|
|
|
|
$rid->[$rseen]{header}{'Message-ID'} |
143
|
|
|
|
|
|
|
} || '') eq $myid->[-1]; |
144
|
|
|
|
|
|
|
|
145
|
|
|
|
|
|
|
$self->do_retrack( |
146
|
|
|
|
|
|
|
$rid, |
147
|
|
|
|
|
|
|
$myid, |
148
|
|
|
|
|
|
|
($rseen > $self->{msgidkeep}) |
149
|
|
|
|
|
|
|
? $rseen - $self->{msgidkeep} : 0, |
150
|
|
|
|
|
|
|
$rseen - 1 |
151
|
|
|
|
|
|
|
); |
152
|
|
|
|
|
|
|
} |
153
|
|
|
|
|
|
|
|
154
|
|
|
|
|
|
|
sub do_send { |
155
|
|
|
|
|
|
|
my $self = $_[0]; |
156
|
|
|
|
|
|
|
my $artgrp = $self->{artgrp}; |
157
|
|
|
|
|
|
|
my $rartgrp = $self->{rartgrp}; |
158
|
|
|
|
|
|
|
my $param = $self->{param}; |
159
|
|
|
|
|
|
|
my $backend = $self->{backend}; |
160
|
|
|
|
|
|
|
my $logfh = $self->{logfh}; |
161
|
|
|
|
|
|
|
my $rbrdname = $param->{board}; |
162
|
|
|
|
|
|
|
my ($lseen, $lseen_last) = split(',', $param->{lseen}, 2); |
163
|
|
|
|
|
|
|
my ($lmsgid, $lmsgid_last) = split(',', $param->{lmsgid}, 2); |
164
|
|
|
|
|
|
|
|
165
|
|
|
|
|
|
|
return unless $lseen eq int($lseen || 0); # must be int |
166
|
|
|
|
|
|
|
$lseen = $#{$artgrp} + 1 if $#{$artgrp} < $lseen; |
167
|
|
|
|
|
|
|
|
168
|
|
|
|
|
|
|
$logfh->print(" [send] checking...\n"); |
169
|
|
|
|
|
|
|
|
170
|
|
|
|
|
|
|
$param->{lseen} = $lseen; |
171
|
|
|
|
|
|
|
$param->{lmsgid} = $lmsgid; |
172
|
|
|
|
|
|
|
|
173
|
|
|
|
|
|
|
if ($lmsgid || $lmsgid_last) { |
174
|
|
|
|
|
|
|
my $art; |
175
|
|
|
|
|
|
|
if ($lseen_last and ($lseen == 0 or |
176
|
|
|
|
|
|
|
($art = eval { $artgrp->[$lseen - 1] } and |
177
|
|
|
|
|
|
|
$art->{header}{'Message-ID'} eq $lmsgid)) and |
178
|
|
|
|
|
|
|
$art = eval { $artgrp->[$lseen_last - 1] } and |
179
|
|
|
|
|
|
|
$art->{header}{'Message-ID'} eq $lmsgid_last) { |
180
|
|
|
|
|
|
|
$lseen = $lseen_last; |
181
|
|
|
|
|
|
|
print " [send] (cached) checking from $lseen_last\n"; |
182
|
|
|
|
|
|
|
} |
183
|
|
|
|
|
|
|
else { |
184
|
|
|
|
|
|
|
++$lseen; |
185
|
|
|
|
|
|
|
|
186
|
|
|
|
|
|
|
while (--$lseen > 0) { |
187
|
|
|
|
|
|
|
my $art = eval { $artgrp->[$lseen - 1] } or next; |
188
|
|
|
|
|
|
|
|
189
|
|
|
|
|
|
|
$logfh->print(" [send] #$lseen: looking back\n"); |
190
|
|
|
|
|
|
|
last unless $lmsgid lt $art->{header}{'Message-ID'}; |
191
|
|
|
|
|
|
|
} |
192
|
|
|
|
|
|
|
|
193
|
|
|
|
|
|
|
$param->{lseen} = $lseen; |
194
|
|
|
|
|
|
|
} |
195
|
|
|
|
|
|
|
} |
196
|
|
|
|
|
|
|
|
197
|
|
|
|
|
|
|
while ($lseen++ <= $#{$artgrp}) { |
198
|
|
|
|
|
|
|
my $art = eval { $artgrp->[$lseen - 1] } or next; |
199
|
|
|
|
|
|
|
next unless defined $art->{title}; # sanity check |
200
|
|
|
|
|
|
|
|
201
|
|
|
|
|
|
|
$lseen_last = $lseen; |
202
|
|
|
|
|
|
|
$lmsgid_last = $art->{header}{'Message-ID'}; |
203
|
|
|
|
|
|
|
|
204
|
|
|
|
|
|
|
next unless ( |
205
|
|
|
|
|
|
|
$self->{force_send} or ( |
206
|
|
|
|
|
|
|
index(($art->{header}{'X-Originator'} || ''), |
207
|
|
|
|
|
|
|
"$rbrdname.board\@$param->{remote}") == -1 and |
208
|
|
|
|
|
|
|
($backend ne 'NNTP' or !$art->{header}{Path}) |
209
|
|
|
|
|
|
|
) |
210
|
|
|
|
|
|
|
); |
211
|
|
|
|
|
|
|
|
212
|
|
|
|
|
|
|
$logfh->print(" [send] #$lseen: posting $art->{title}\n"); |
213
|
|
|
|
|
|
|
|
214
|
|
|
|
|
|
|
my %xart = ( header => { %{$art->{header}} } ); |
215
|
|
|
|
|
|
|
safe_copy($art, \%xart); |
216
|
|
|
|
|
|
|
$xart{body} = $art->{body}; |
217
|
|
|
|
|
|
|
|
218
|
|
|
|
|
|
|
if ($self->{clobber}) { |
219
|
|
|
|
|
|
|
my $adr = (Mail::Address->parse($xart{header}{From}))[0]; |
220
|
|
|
|
|
|
|
|
221
|
|
|
|
|
|
|
$xart{header}{From} = ( |
222
|
|
|
|
|
|
|
$adr->address.'.bbs@'.$self->{hostname}.' '.$adr->comment |
223
|
|
|
|
|
|
|
) if $adr and index($adr->address, '@') == -1; |
224
|
|
|
|
|
|
|
} |
225
|
|
|
|
|
|
|
|
226
|
|
|
|
|
|
|
my $xorig = $artgrp->board.'.board@'.$self->{hostname}; |
227
|
|
|
|
|
|
|
|
228
|
|
|
|
|
|
|
if (index(' External NNTP MELIX DBI ', $backend) > -1 |
229
|
|
|
|
|
|
|
or ($backend eq 'OurNet' |
230
|
|
|
|
|
|
|
and index(' NNTP MELIX DBI ', $rartgrp->backend) > -1)) |
231
|
|
|
|
|
|
|
{ |
232
|
|
|
|
|
|
|
$xart{header}{'X-Originator'} = $xorig; |
233
|
|
|
|
|
|
|
} |
234
|
|
|
|
|
|
|
elsif (rindex($xart{body}, "--\n¡°") > -1) { |
235
|
|
|
|
|
|
|
chomp($xart{body}); |
236
|
|
|
|
|
|
|
chomp($xart{body}); |
237
|
|
|
|
|
|
|
$xart{body} .= "\n¡° X-Originator: $xorig"; |
238
|
|
|
|
|
|
|
} |
239
|
|
|
|
|
|
|
else { |
240
|
|
|
|
|
|
|
$xart{body} .= "--\n¡° X-Originator: $xorig"; |
241
|
|
|
|
|
|
|
} |
242
|
|
|
|
|
|
|
|
243
|
|
|
|
|
|
|
eval { $rartgrp->{''} = \%xart } unless $self->{force_none}; |
244
|
|
|
|
|
|
|
|
245
|
|
|
|
|
|
|
if ($@) { |
246
|
|
|
|
|
|
|
chomp(my $error = $@); |
247
|
|
|
|
|
|
|
$logfh->print(" [send] #$lseen: can't post ($error)\n"); |
248
|
|
|
|
|
|
|
} |
249
|
|
|
|
|
|
|
else { |
250
|
|
|
|
|
|
|
$param->{lseen} = $lseen; |
251
|
|
|
|
|
|
|
$param->{lmsgid} = $art->{header}{'Message-ID'}; |
252
|
|
|
|
|
|
|
|
253
|
|
|
|
|
|
|
$self->{callback}->($self, 'post') |
254
|
|
|
|
|
|
|
if UNIVERSAL::isa($self->{callback}, 'CODE'); # callback |
255
|
|
|
|
|
|
|
} |
256
|
|
|
|
|
|
|
} |
257
|
|
|
|
|
|
|
|
258
|
|
|
|
|
|
|
$param->{lseen} .= ",$lseen_last"; |
259
|
|
|
|
|
|
|
$param->{lmsgid} .= ",$lmsgid_last"; |
260
|
|
|
|
|
|
|
|
261
|
|
|
|
|
|
|
return 1; |
262
|
|
|
|
|
|
|
} |
263
|
|
|
|
|
|
|
|
264
|
|
|
|
|
|
|
sub do_fetch { |
265
|
|
|
|
|
|
|
my ($self, $dir, $depth) = @_; |
266
|
|
|
|
|
|
|
|
267
|
|
|
|
|
|
|
my $artgrp = $self->{artgrp}; |
268
|
|
|
|
|
|
|
my $rartgrp = $self->{rartgrp}; |
269
|
|
|
|
|
|
|
my $param = $self->{param}; |
270
|
|
|
|
|
|
|
my $backend = $self->{backend}; |
271
|
|
|
|
|
|
|
my $logfh = $self->{logfh}; |
272
|
|
|
|
|
|
|
my $msgids = $param->{msgids}{$dir} ||= []; |
273
|
|
|
|
|
|
|
my $btimes = $param->{msgids}{'__BTIME__'} ||= {}; |
274
|
|
|
|
|
|
|
my $rbrdname = $param->{board}; # remote board name |
275
|
|
|
|
|
|
|
my $padding = ' ' x (++$depth); |
276
|
|
|
|
|
|
|
|
277
|
|
|
|
|
|
|
my ($first, $last, $rseen); |
278
|
|
|
|
|
|
|
|
279
|
|
|
|
|
|
|
if ($backend eq 'NNTP') { |
280
|
|
|
|
|
|
|
$first = $rartgrp->first; |
281
|
|
|
|
|
|
|
$last = $rartgrp->last; |
282
|
|
|
|
|
|
|
$rseen = defined($param->{rseen}) |
283
|
|
|
|
|
|
|
? $param->{rseen} : ($last - $self->{msgidkeep}); |
284
|
|
|
|
|
|
|
} |
285
|
|
|
|
|
|
|
else { |
286
|
|
|
|
|
|
|
$first = 0; # for normal sequential backends |
287
|
|
|
|
|
|
|
$last = $#{$rartgrp}; |
288
|
|
|
|
|
|
|
$rseen = $param->{rseen}; |
289
|
|
|
|
|
|
|
} |
290
|
|
|
|
|
|
|
|
291
|
|
|
|
|
|
|
return unless defined($rseen) and length($rseen); # requires rseen |
292
|
|
|
|
|
|
|
|
293
|
|
|
|
|
|
|
$rseen += $last + 1 if $rseen < 0; # negative subscripts |
294
|
|
|
|
|
|
|
$rseen = $last + 1 if $rseen > $last; # upper bound |
295
|
|
|
|
|
|
|
|
296
|
|
|
|
|
|
|
$logfh->print($padding, "[fetch] #$param->{rseen}: checking\n"); |
297
|
|
|
|
|
|
|
|
298
|
|
|
|
|
|
|
if ($msgids and @{$msgids}) { |
299
|
|
|
|
|
|
|
if ($rseen and my $msgid = eval { |
300
|
|
|
|
|
|
|
$rartgrp->[$rseen - 1]{header}{'Message-ID'} |
301
|
|
|
|
|
|
|
}) { |
302
|
|
|
|
|
|
|
$msgid = "<$msgid>" if substr($msgid, 0, 1) ne '<'; |
303
|
|
|
|
|
|
|
$rseen = $self->retrack($rartgrp, $msgids, $rseen - 1) |
304
|
|
|
|
|
|
|
if $msgid ne $msgids->[-1]; |
305
|
|
|
|
|
|
|
} |
306
|
|
|
|
|
|
|
} |
307
|
|
|
|
|
|
|
else { # init |
308
|
|
|
|
|
|
|
my $rfirst = (($rseen - $first) > $self->{msgidkeep}) |
309
|
|
|
|
|
|
|
? $rseen - $self->{msgidkeep} : $first; |
310
|
|
|
|
|
|
|
|
311
|
|
|
|
|
|
|
my $i = $rfirst; |
312
|
|
|
|
|
|
|
|
313
|
|
|
|
|
|
|
while($i < $rseen) { |
314
|
|
|
|
|
|
|
$logfh->print($padding, "[fetch] #$i: init"); |
315
|
|
|
|
|
|
|
|
316
|
|
|
|
|
|
|
eval { |
317
|
|
|
|
|
|
|
my $art = $rartgrp->[$i++]; |
318
|
|
|
|
|
|
|
$art->refresh; |
319
|
|
|
|
|
|
|
$self->update_msgid( |
320
|
|
|
|
|
|
|
$dir, $art->{header}{'Message-ID'}, 'init' |
321
|
|
|
|
|
|
|
); |
322
|
|
|
|
|
|
|
}; |
323
|
|
|
|
|
|
|
|
324
|
|
|
|
|
|
|
$logfh->print($@ ? " failed: $@\n" : " ok\n"); |
325
|
|
|
|
|
|
|
} |
326
|
|
|
|
|
|
|
|
327
|
|
|
|
|
|
|
$rseen = $i; |
328
|
|
|
|
|
|
|
} |
329
|
|
|
|
|
|
|
|
330
|
|
|
|
|
|
|
$rseen = 0 if $rseen < 0; |
331
|
|
|
|
|
|
|
|
332
|
|
|
|
|
|
|
$logfh->print($padding, |
333
|
|
|
|
|
|
|
($rseen <= $last) |
334
|
|
|
|
|
|
|
? "[fetch] range: $rseen..$last\n" |
335
|
|
|
|
|
|
|
: "[fetch] nothing to fetch ($rseen > $last)\n" |
336
|
|
|
|
|
|
|
); |
337
|
|
|
|
|
|
|
|
338
|
|
|
|
|
|
|
return if $rseen > $last; |
339
|
|
|
|
|
|
|
|
340
|
|
|
|
|
|
|
my $xorig = $artgrp->board.".board\@$self->{hostname}"; |
341
|
|
|
|
|
|
|
|
342
|
|
|
|
|
|
|
while ($rseen <= $last) { |
343
|
|
|
|
|
|
|
my ($art, $btime); |
344
|
|
|
|
|
|
|
|
345
|
|
|
|
|
|
|
$logfh->print($padding, "[fetch] #$rseen: reading"); |
346
|
|
|
|
|
|
|
|
347
|
|
|
|
|
|
|
eval { |
348
|
|
|
|
|
|
|
$art = $rartgrp->[$rseen]; |
349
|
|
|
|
|
|
|
$art->refresh; |
350
|
|
|
|
|
|
|
}; |
351
|
|
|
|
|
|
|
|
352
|
|
|
|
|
|
|
if ($@) { |
353
|
|
|
|
|
|
|
$logfh->print("... nonexistent, failed\n"); |
354
|
|
|
|
|
|
|
++$rseen; next; |
355
|
|
|
|
|
|
|
} |
356
|
|
|
|
|
|
|
|
357
|
|
|
|
|
|
|
my ($msgid, $rhead); |
358
|
|
|
|
|
|
|
|
359
|
|
|
|
|
|
|
my $is_group = ($art->REF =~ m|ArticleGroup|); |
360
|
|
|
|
|
|
|
|
361
|
|
|
|
|
|
|
if ($is_group) { |
362
|
|
|
|
|
|
|
$btime = $art->btime; # saves its modification time |
363
|
|
|
|
|
|
|
|
364
|
|
|
|
|
|
|
$art = { |
365
|
|
|
|
|
|
|
date => $art->{date}, |
366
|
|
|
|
|
|
|
author => $art->{author}, |
367
|
|
|
|
|
|
|
title => $art->{title}, |
368
|
|
|
|
|
|
|
}; |
369
|
|
|
|
|
|
|
|
370
|
|
|
|
|
|
|
# not really a message so won't have MSGID; let's fake one here. |
371
|
|
|
|
|
|
|
$msgid = OurNet::BBS::Utils::get_msgid( |
372
|
|
|
|
|
|
|
@{$art}{qw/date author title/}, |
373
|
|
|
|
|
|
|
$rbrdname, |
374
|
|
|
|
|
|
|
$param->{remote}, |
375
|
|
|
|
|
|
|
); |
376
|
|
|
|
|
|
|
} |
377
|
|
|
|
|
|
|
else { |
378
|
|
|
|
|
|
|
$msgid = $art->{header}{'Message-ID'}; # XXX voodoo refresh |
379
|
|
|
|
|
|
|
|
380
|
|
|
|
|
|
|
$art = $art->SPAWN; |
381
|
|
|
|
|
|
|
$rhead = $art->{header}; |
382
|
|
|
|
|
|
|
|
383
|
|
|
|
|
|
|
if ($rhead->{'Message-ID'} ne $msgid) { |
384
|
|
|
|
|
|
|
# something's very, very wrong |
385
|
|
|
|
|
|
|
print "... lacks Message-ID, skipped\n"; |
386
|
|
|
|
|
|
|
++$rseen; next; |
387
|
|
|
|
|
|
|
} |
388
|
|
|
|
|
|
|
|
389
|
|
|
|
|
|
|
$msgid = "<$msgid>" if substr($msgid, 0, 1) ne '<'; # legacy |
390
|
|
|
|
|
|
|
} |
391
|
|
|
|
|
|
|
|
392
|
|
|
|
|
|
|
if ($self->{force_fetch} or |
393
|
|
|
|
|
|
|
rindex($art->{body}, "X-Originator: $xorig") == -1 and |
394
|
|
|
|
|
|
|
nth($msgids, $msgid) == -1 and |
395
|
|
|
|
|
|
|
($rhead->{'X-Originator'} || '') ne $xorig |
396
|
|
|
|
|
|
|
) { |
397
|
|
|
|
|
|
|
my (%xart, $xartref); |
398
|
|
|
|
|
|
|
|
399
|
|
|
|
|
|
|
$self->update_msgid($dir, $msgid, 'fetch'); |
400
|
|
|
|
|
|
|
|
401
|
|
|
|
|
|
|
if (!$is_group) { |
402
|
|
|
|
|
|
|
%xart = (header => $rhead); # maximal cache |
403
|
|
|
|
|
|
|
safe_copy($art, $xartref = \%xart); |
404
|
|
|
|
|
|
|
|
405
|
|
|
|
|
|
|
# the code below makes us *really* want a ??= operator. |
406
|
|
|
|
|
|
|
unless (defined $xart{body} or |
407
|
|
|
|
|
|
|
defined $xart{header}{Subject}) { |
408
|
|
|
|
|
|
|
print "... article empty, skipped\n"; |
409
|
|
|
|
|
|
|
++$rseen; next; |
410
|
|
|
|
|
|
|
} |
411
|
|
|
|
|
|
|
|
412
|
|
|
|
|
|
|
if ($dir eq 'archives' and $xart{header}{Subject} eq '#') { |
413
|
|
|
|
|
|
|
print "... '#' metadata, skipped\n"; |
414
|
|
|
|
|
|
|
++$rseen; next; |
415
|
|
|
|
|
|
|
} |
416
|
|
|
|
|
|
|
|
417
|
|
|
|
|
|
|
$xart{header}{'X-Originator'} = |
418
|
|
|
|
|
|
|
"$rbrdname.board\@$param->{remote}" if $backend ne 'NNTP'; |
419
|
|
|
|
|
|
|
|
420
|
|
|
|
|
|
|
$xart{body} =~ s|^((?:: )+)|'> ' x (length($1)/2)|gem; |
421
|
|
|
|
|
|
|
$xart{nick} = $1 if $xart{nick} =~ m/^\s*\((.*)\)$/; |
422
|
|
|
|
|
|
|
|
423
|
|
|
|
|
|
|
if ($self->{clobber} and $backend ne 'NNTP') { |
424
|
|
|
|
|
|
|
$xart{author} .= "." unless !$xart{author} |
425
|
|
|
|
|
|
|
or substr($xart{author}, -1) eq '.'; |
426
|
|
|
|
|
|
|
$xart{header}{From} = |
427
|
|
|
|
|
|
|
"$xart{author}bbs\@$param->{remote}" . |
428
|
|
|
|
|
|
|
($xart{nick} ? " ($xart{nick})" : '') |
429
|
|
|
|
|
|
|
unless $xart{header}{From} =~ /^[^\(]+\@/; |
430
|
|
|
|
|
|
|
} |
431
|
|
|
|
|
|
|
elsif (0) { # XXX: not yet supported |
432
|
|
|
|
|
|
|
$xart{header}{'Reply-To'} = |
433
|
|
|
|
|
|
|
"$xart{author}.bbs\@$param->{remote}" . |
434
|
|
|
|
|
|
|
(defined $xart{nick} ? " ($xart{nick})" : '') |
435
|
|
|
|
|
|
|
unless $xart{header}{From} =~ /^[^\(]+\@/; |
436
|
|
|
|
|
|
|
} |
437
|
|
|
|
|
|
|
|
438
|
|
|
|
|
|
|
$artgrp->{''} = $xartref unless $self->{force_none}; |
439
|
|
|
|
|
|
|
$logfh->print(" $xart{title}\n"); |
440
|
|
|
|
|
|
|
} |
441
|
|
|
|
|
|
|
else { # ArticleGroup code |
442
|
|
|
|
|
|
|
%xart = %{$art}; |
443
|
|
|
|
|
|
|
|
444
|
|
|
|
|
|
|
# strip down unnecessary sigils |
445
|
|
|
|
|
|
|
$xart{title} = substr($xart{title}, 3) |
446
|
|
|
|
|
|
|
if index(SKIPPED_SIGILS, substr($xart{title}, 0, 3)) > -1; |
447
|
|
|
|
|
|
|
|
448
|
|
|
|
|
|
|
$xartref = bless(\%xart, $artgrp->module('ArticleGroup')); |
449
|
|
|
|
|
|
|
|
450
|
|
|
|
|
|
|
$artgrp->{''} = $xartref unless $self->{force_none}; |
451
|
|
|
|
|
|
|
$logfh->print(" $xart{title}\n"); |
452
|
|
|
|
|
|
|
|
453
|
|
|
|
|
|
|
$self->fetch_archive( |
454
|
|
|
|
|
|
|
$artgrp->[-1], |
455
|
|
|
|
|
|
|
$rartgrp->[$rseen], |
456
|
|
|
|
|
|
|
0, # start anew |
457
|
|
|
|
|
|
|
$msgid, $depth, $btime, $btimes, |
458
|
|
|
|
|
|
|
); |
459
|
|
|
|
|
|
|
} |
460
|
|
|
|
|
|
|
} |
461
|
|
|
|
|
|
|
elsif ($is_group and $self->{recursive} |
462
|
|
|
|
|
|
|
and $btimes->{$msgid}[0] != $btime |
463
|
|
|
|
|
|
|
) { |
464
|
|
|
|
|
|
|
$logfh->print(" $art->{title} (updating)\n"); |
465
|
|
|
|
|
|
|
|
466
|
|
|
|
|
|
|
$self->fetch_archive( |
467
|
|
|
|
|
|
|
$artgrp->{$btimes->{$msgid}[1]}, # name |
468
|
|
|
|
|
|
|
$rartgrp->[$rseen], |
469
|
|
|
|
|
|
|
-$self->{msgidkeep}, # update cached only |
470
|
|
|
|
|
|
|
$msgid, $depth, $btime, $btimes, |
471
|
|
|
|
|
|
|
); |
472
|
|
|
|
|
|
|
} |
473
|
|
|
|
|
|
|
else { |
474
|
|
|
|
|
|
|
$logfh->print("... duplicate, skipped\n"); |
475
|
|
|
|
|
|
|
$self->update_msgid($dir, $msgid, 'duplicate'); |
476
|
|
|
|
|
|
|
} |
477
|
|
|
|
|
|
|
|
478
|
|
|
|
|
|
|
$param->{rseen} = ++$rseen; |
479
|
|
|
|
|
|
|
} |
480
|
|
|
|
|
|
|
|
481
|
|
|
|
|
|
|
return $artgrp->[-1] || 1; # must be here to re-initialize this board |
482
|
|
|
|
|
|
|
} |
483
|
|
|
|
|
|
|
|
484
|
|
|
|
|
|
|
sub update_msgid { |
485
|
|
|
|
|
|
|
my ($self, $dir, $msgid, $reason) = @_; |
486
|
|
|
|
|
|
|
|
487
|
|
|
|
|
|
|
push @{$self->{param}{msgids}{$dir}}, $msgid; |
488
|
|
|
|
|
|
|
|
489
|
|
|
|
|
|
|
$self->{callback}->($self, $reason) |
490
|
|
|
|
|
|
|
if UNIVERSAL::isa($self->{callback}, 'CODE'); # callback |
491
|
|
|
|
|
|
|
} |
492
|
|
|
|
|
|
|
|
493
|
|
|
|
|
|
|
sub fetch_archive { |
494
|
|
|
|
|
|
|
my $self = shift; |
495
|
|
|
|
|
|
|
return unless $self->{recursive}; |
496
|
|
|
|
|
|
|
|
497
|
|
|
|
|
|
|
my ($artgrp, $rartgrp) = @{$self}{qw/artgrp rartgrp/}; |
498
|
|
|
|
|
|
|
|
499
|
|
|
|
|
|
|
$self->{artgrp} = shift; |
500
|
|
|
|
|
|
|
$self->{rartgrp} = shift; |
501
|
|
|
|
|
|
|
$self->{param}{rseen} = shift; |
502
|
|
|
|
|
|
|
|
503
|
|
|
|
|
|
|
my ($msgid, $depth, $btime, $btimes) = @_; |
504
|
|
|
|
|
|
|
|
505
|
|
|
|
|
|
|
$self->do_fetch($msgid, $depth); |
506
|
|
|
|
|
|
|
$btimes->{$msgid} = [ |
507
|
|
|
|
|
|
|
$btime, $self->{artgrp}->name, |
508
|
|
|
|
|
|
|
]; |
509
|
|
|
|
|
|
|
|
510
|
|
|
|
|
|
|
@{$self}{qw/artgrp rartgrp/} = ($artgrp, $rartgrp); |
511
|
|
|
|
|
|
|
} |
512
|
|
|
|
|
|
|
|
513
|
|
|
|
|
|
|
sub safe_copy { |
514
|
|
|
|
|
|
|
my ($from, $to) = @_; |
515
|
|
|
|
|
|
|
|
516
|
|
|
|
|
|
|
while (my ($k, $v) = each (%{$from})) { |
517
|
|
|
|
|
|
|
$to->{$k} = $v if index( |
518
|
|
|
|
|
|
|
SKIPPED_HEADERS, " $k " |
519
|
|
|
|
|
|
|
) == -1; |
520
|
|
|
|
|
|
|
} |
521
|
|
|
|
|
|
|
} |
522
|
|
|
|
|
|
|
|
523
|
|
|
|
|
|
|
1; |
524
|
|
|
|
|
|
|
|
525
|
|
|
|
|
|
|
__END__ |