line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package Dezi::Test::Indexer; |
2
|
7
|
|
|
7
|
|
11057
|
use Moose; |
|
7
|
|
|
|
|
1366207
|
|
|
7
|
|
|
|
|
51
|
|
3
|
|
|
|
|
|
|
extends 'Dezi::Indexer'; |
4
|
7
|
|
|
7
|
|
51423
|
use Dezi::Test::Doc; |
|
0
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
|
5
|
|
|
|
|
|
|
use SWISH::3 qw( :constants ); |
6
|
|
|
|
|
|
|
use Search::Tools::UTF8; |
7
|
|
|
|
|
|
|
use Data::Dump qw( dump ); |
8
|
|
|
|
|
|
|
|
9
|
|
|
|
|
|
|
our $VERSION = '0.014'; |
10
|
|
|
|
|
|
|
|
11
|
|
|
|
|
|
|
sub invindex_class {'Dezi::Test::InvIndex'} |
12
|
|
|
|
|
|
|
sub use_swish3_tokenizer {1} |
13
|
|
|
|
|
|
|
|
14
|
|
|
|
|
|
|
my $doc_prop_map = SWISH_DOC_PROP_MAP(); |
15
|
|
|
|
|
|
|
|
16
|
|
|
|
|
|
|
sub swish3_handler { |
17
|
|
|
|
|
|
|
my ( $self, $payload ) = @_; |
18
|
|
|
|
|
|
|
my $s3config = $payload->config; |
19
|
|
|
|
|
|
|
my $s3props = $s3config->get_properties; |
20
|
|
|
|
|
|
|
my $s3metas = $s3config->get_metanames; |
21
|
|
|
|
|
|
|
|
22
|
|
|
|
|
|
|
# will hold all the parsed text, keyed by field name |
23
|
|
|
|
|
|
|
my %doc; |
24
|
|
|
|
|
|
|
|
25
|
|
|
|
|
|
|
# Swish built-in fields first |
26
|
|
|
|
|
|
|
for my $propname ( keys %$doc_prop_map ) { |
27
|
|
|
|
|
|
|
my $attr = $doc_prop_map->{$propname}; |
28
|
|
|
|
|
|
|
$doc{$propname} = [ $payload->doc->$attr ]; |
29
|
|
|
|
|
|
|
} |
30
|
|
|
|
|
|
|
|
31
|
|
|
|
|
|
|
# fields parsed from document |
32
|
|
|
|
|
|
|
my $props = $payload->properties; |
33
|
|
|
|
|
|
|
my $metas = $payload->metanames; |
34
|
|
|
|
|
|
|
|
35
|
|
|
|
|
|
|
#dump $props; |
36
|
|
|
|
|
|
|
#dump $metas; |
37
|
|
|
|
|
|
|
|
38
|
|
|
|
|
|
|
# flesh out %doc |
39
|
|
|
|
|
|
|
for my $field ( keys %$props ) { |
40
|
|
|
|
|
|
|
push @{ $doc{$field} }, @{ $props->{$field} }; |
41
|
|
|
|
|
|
|
} |
42
|
|
|
|
|
|
|
for my $field ( keys %$metas ) { |
43
|
|
|
|
|
|
|
next if exists $doc{$field}; # prefer property over metaname |
44
|
|
|
|
|
|
|
push @{ $doc{$field} }, @{ $metas->{$field} }; |
45
|
|
|
|
|
|
|
} |
46
|
|
|
|
|
|
|
for my $k ( keys %doc ) { |
47
|
|
|
|
|
|
|
$doc{$k} = to_utf8( join( SWISH_TOKENPOS_BUMPER(), @{ $doc{$k} } ) ); |
48
|
|
|
|
|
|
|
} |
49
|
|
|
|
|
|
|
|
50
|
|
|
|
|
|
|
$self->invindex->put_doc( Dezi::Test::Doc->new(%doc) ); |
51
|
|
|
|
|
|
|
|
52
|
|
|
|
|
|
|
# add tokens to our invindex |
53
|
|
|
|
|
|
|
my $term_cache = $self->invindex->term_cache; |
54
|
|
|
|
|
|
|
my $tokens = $payload->tokens; |
55
|
|
|
|
|
|
|
my $uri = $doc{swishdocpath}; |
56
|
|
|
|
|
|
|
while ( my $token = $tokens->next ) { |
57
|
|
|
|
|
|
|
my $str = $token->value; |
58
|
|
|
|
|
|
|
if ( !$term_cache->has($str) ) { |
59
|
|
|
|
|
|
|
$term_cache->add( $str => { $uri => 1 } ); |
60
|
|
|
|
|
|
|
} |
61
|
|
|
|
|
|
|
else { |
62
|
|
|
|
|
|
|
$term_cache->get($str)->{$uri}++; |
63
|
|
|
|
|
|
|
} |
64
|
|
|
|
|
|
|
} |
65
|
|
|
|
|
|
|
|
66
|
|
|
|
|
|
|
} |
67
|
|
|
|
|
|
|
|
68
|
|
|
|
|
|
|
__PACKAGE__->meta->make_immutable; |
69
|
|
|
|
|
|
|
|
70
|
|
|
|
|
|
|
1; |
71
|
|
|
|
|
|
|
|
72
|
|
|
|
|
|
|
__END__ |
73
|
|
|
|
|
|
|
|
74
|
|
|
|
|
|
|
=pod |
75
|
|
|
|
|
|
|
|
76
|
|
|
|
|
|
|
=head1 NAME |
77
|
|
|
|
|
|
|
|
78
|
|
|
|
|
|
|
Dezi::Test::Indexer - test indexer class |
79
|
|
|
|
|
|
|
|
80
|
|
|
|
|
|
|
=head1 SYNOPSIS |
81
|
|
|
|
|
|
|
|
82
|
|
|
|
|
|
|
use Dezi::Test::Indexer; |
83
|
|
|
|
|
|
|
|
84
|
|
|
|
|
|
|
my $spider = Dezi::Aggregator::Spider->new( |
85
|
|
|
|
|
|
|
indexer => Dezi::Test::Indexer->new() |
86
|
|
|
|
|
|
|
); |
87
|
|
|
|
|
|
|
$spider->crawl('http://localhost/foo'); |
88
|
|
|
|
|
|
|
|
89
|
|
|
|
|
|
|
=head1 DESCRIPTION |
90
|
|
|
|
|
|
|
|
91
|
|
|
|
|
|
|
Dezi::Test::Indexer uses Dezi::Test::InvIndex for |
92
|
|
|
|
|
|
|
running tests on the API, particularly Aggregator classes. |
93
|
|
|
|
|
|
|
|
94
|
|
|
|
|
|
|
=head1 CONSTANTS |
95
|
|
|
|
|
|
|
|
96
|
|
|
|
|
|
|
All the L<SWISH::3> constants are imported into this namespace, |
97
|
|
|
|
|
|
|
including: |
98
|
|
|
|
|
|
|
|
99
|
|
|
|
|
|
|
=head2 SWISH_DOC_PROP_MAP |
100
|
|
|
|
|
|
|
|
101
|
|
|
|
|
|
|
=head1 METHODS |
102
|
|
|
|
|
|
|
|
103
|
|
|
|
|
|
|
=head2 process( I<doc> ) |
104
|
|
|
|
|
|
|
|
105
|
|
|
|
|
|
|
Tokenizes content in I<doc> and adds each term |
106
|
|
|
|
|
|
|
to the InvIndex. |
107
|
|
|
|
|
|
|
|
108
|
|
|
|
|
|
|
=head2 swish3_handler |
109
|
|
|
|
|
|
|
|
110
|
|
|
|
|
|
|
Called by L<SWISH::3> handler() function. |
111
|
|
|
|
|
|
|
|
112
|
|
|
|
|
|
|
=head2 use_swish3_tokenizer |
113
|
|
|
|
|
|
|
|
114
|
|
|
|
|
|
|
Returns true. |
115
|
|
|
|
|
|
|
|
116
|
|
|
|
|
|
|
=head2 invindex_class |
117
|
|
|
|
|
|
|
|
118
|
|
|
|
|
|
|
Returns L<Dezi::Test::InvIndex>. |
119
|
|
|
|
|
|
|
|
120
|
|
|
|
|
|
|
=head1 AUTHOR |
121
|
|
|
|
|
|
|
|
122
|
|
|
|
|
|
|
Peter Karman, E<lt>perl@peknet.comE<gt> |
123
|
|
|
|
|
|
|
|
124
|
|
|
|
|
|
|
=head1 BUGS |
125
|
|
|
|
|
|
|
|
126
|
|
|
|
|
|
|
Please report any bugs or feature requests to C<bug-swish-prog at rt.cpan.org>, or through |
127
|
|
|
|
|
|
|
the web interface at L<http://rt.cpan.org/NoAuth/ReportBug.html?Queue=Dezi-App>. |
128
|
|
|
|
|
|
|
I will be notified, and then you'll |
129
|
|
|
|
|
|
|
automatically be notified of progress on your bug as I make changes. |
130
|
|
|
|
|
|
|
|
131
|
|
|
|
|
|
|
=head1 SUPPORT |
132
|
|
|
|
|
|
|
|
133
|
|
|
|
|
|
|
You can find documentation for this module with the perldoc command. |
134
|
|
|
|
|
|
|
|
135
|
|
|
|
|
|
|
perldoc Dezi |
136
|
|
|
|
|
|
|
|
137
|
|
|
|
|
|
|
|
138
|
|
|
|
|
|
|
You can also look for information at: |
139
|
|
|
|
|
|
|
|
140
|
|
|
|
|
|
|
=over 4 |
141
|
|
|
|
|
|
|
|
142
|
|
|
|
|
|
|
=item * Mailing list |
143
|
|
|
|
|
|
|
|
144
|
|
|
|
|
|
|
L<http://lists.swish-e.org/listinfo/users> |
145
|
|
|
|
|
|
|
|
146
|
|
|
|
|
|
|
=item * RT: CPAN's request tracker |
147
|
|
|
|
|
|
|
|
148
|
|
|
|
|
|
|
L<http://rt.cpan.org/NoAuth/Bugs.html?Dist=Dezi-App> |
149
|
|
|
|
|
|
|
|
150
|
|
|
|
|
|
|
=item * AnnoCPAN: Annotated CPAN documentation |
151
|
|
|
|
|
|
|
|
152
|
|
|
|
|
|
|
L<http://annocpan.org/dist/Dezi-App> |
153
|
|
|
|
|
|
|
|
154
|
|
|
|
|
|
|
=item * CPAN Ratings |
155
|
|
|
|
|
|
|
|
156
|
|
|
|
|
|
|
L<http://cpanratings.perl.org/d/Dezi-App> |
157
|
|
|
|
|
|
|
|
158
|
|
|
|
|
|
|
=item * Search CPAN |
159
|
|
|
|
|
|
|
|
160
|
|
|
|
|
|
|
L<http://search.cpan.org/dist/Dezi-App/> |
161
|
|
|
|
|
|
|
|
162
|
|
|
|
|
|
|
=back |
163
|
|
|
|
|
|
|
|
164
|
|
|
|
|
|
|
=head1 COPYRIGHT AND LICENSE |
165
|
|
|
|
|
|
|
|
166
|
|
|
|
|
|
|
Copyright 2008-2009 by Peter Karman |
167
|
|
|
|
|
|
|
|
168
|
|
|
|
|
|
|
This library is free software; you can redistribute it and/or modify |
169
|
|
|
|
|
|
|
it under the same terms as Perl itself. |
170
|
|
|
|
|
|
|
|
171
|
|
|
|
|
|
|
=head1 SEE ALSO |
172
|
|
|
|
|
|
|
|
173
|
|
|
|
|
|
|
L<http://swish-e.org/> |