line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package WWW::Wappalyzer; |
2
|
|
|
|
|
|
|
|
3
|
2
|
|
|
2
|
|
27382
|
use 5.006; |
|
2
|
|
|
|
|
4
|
|
4
|
2
|
|
|
2
|
|
6
|
use strict; |
|
2
|
|
|
|
|
2
|
|
|
2
|
|
|
|
|
33
|
|
5
|
2
|
|
|
2
|
|
5
|
use warnings; |
|
2
|
|
|
|
|
4
|
|
|
2
|
|
|
|
|
47
|
|
6
|
|
|
|
|
|
|
|
7
|
2
|
|
|
2
|
|
6
|
use base qw ( Exporter ); |
|
2
|
|
|
|
|
2
|
|
|
2
|
|
|
|
|
162
|
|
8
|
|
|
|
|
|
|
our @EXPORT_OK = qw( detect get_categories add_clues_file ); |
9
|
|
|
|
|
|
|
|
10
|
2
|
|
|
2
|
|
833
|
use lib::abs; |
|
2
|
|
|
|
|
3488
|
|
|
2
|
|
|
|
|
8
|
|
11
|
2
|
|
|
2
|
|
1237
|
use JSON qw(); |
|
2
|
|
|
|
|
19585
|
|
|
2
|
|
|
|
|
40
|
|
12
|
2
|
|
|
2
|
|
948
|
use Regexp::Parser; |
|
2
|
|
|
|
|
76564
|
|
|
2
|
|
|
|
|
2301
|
|
13
|
|
|
|
|
|
|
|
14
|
|
|
|
|
|
|
my %_categories; |
15
|
|
|
|
|
|
|
my @_clues_file_list = ( lib::abs::path( './apps.json' ) ); |
16
|
|
|
|
|
|
|
|
17
|
|
|
|
|
|
|
# List of multi per web page application categories |
18
|
|
|
|
|
|
|
my %MULTIPLE_APP_CATS = map { $_ => 1 } qw( |
19
|
|
|
|
|
|
|
widgets analytics javascript-frameworks video-players |
20
|
|
|
|
|
|
|
font-scripts miscellaneous advertising-networks payment-processors |
21
|
|
|
|
|
|
|
); |
22
|
|
|
|
|
|
|
|
23
|
|
|
|
|
|
|
=head1 NAME |
24
|
|
|
|
|
|
|
|
25
|
|
|
|
|
|
|
WWW::Wappalyzer - Perl port of Wappalyzer (http://wappalyzer.com) |
26
|
|
|
|
|
|
|
|
27
|
|
|
|
|
|
|
=head1 DESCRIPTION |
28
|
|
|
|
|
|
|
|
29
|
|
|
|
|
|
|
Uncovers the technologies used on websites: detects content management systems, web shops, |
30
|
|
|
|
|
|
|
web servers, JavaScript frameworks, analytics tools and many more. |
31
|
|
|
|
|
|
|
|
32
|
|
|
|
|
|
|
Lacks 'version', 'implies', 'excludes' support of original Wappalyzer in favour of speed. |
33
|
|
|
|
|
|
|
|
34
|
|
|
|
|
|
|
Clues: L |
35
|
|
|
|
|
|
|
|
36
|
|
|
|
|
|
|
More info on Wappalyzer: L |
37
|
|
|
|
|
|
|
|
38
|
|
|
|
|
|
|
=cut |
39
|
|
|
|
|
|
|
|
40
|
|
|
|
|
|
|
our $VERSION = '0.20'; |
41
|
|
|
|
|
|
|
|
42
|
|
|
|
|
|
|
=head1 SYNOPSIS |
43
|
|
|
|
|
|
|
|
44
|
|
|
|
|
|
|
use WWW::Wappalyzer; |
45
|
|
|
|
|
|
|
use LWP::UserAgent; |
46
|
|
|
|
|
|
|
|
47
|
|
|
|
|
|
|
my $response = LWP::UserAgent->new->get( 'http://www.drupal.org' ); |
48
|
|
|
|
|
|
|
my %detected = WWW::Wappalyzer::detect( |
49
|
|
|
|
|
|
|
html => $response->decoded_content, |
50
|
|
|
|
|
|
|
headers => $response->headers, |
51
|
|
|
|
|
|
|
); |
52
|
|
|
|
|
|
|
|
53
|
|
|
|
|
|
|
# %detected = ( |
54
|
|
|
|
|
|
|
# 'web-servers' => [ 'Apache' ], |
55
|
|
|
|
|
|
|
# 'cms' => [ 'Drupal' ], |
56
|
|
|
|
|
|
|
# 'cache-tools' => [ 'Varnish' ], |
57
|
|
|
|
|
|
|
# 'analytics' => [ 'Google Analytics' ], |
58
|
|
|
|
|
|
|
# 'operating-systems' => [ 'CentOS' ] |
59
|
|
|
|
|
|
|
# ); |
60
|
|
|
|
|
|
|
|
61
|
|
|
|
|
|
|
=head1 EXPORT |
62
|
|
|
|
|
|
|
|
63
|
|
|
|
|
|
|
None by default. |
64
|
|
|
|
|
|
|
|
65
|
|
|
|
|
|
|
=head1 SUBROUTINES/METHODS |
66
|
|
|
|
|
|
|
|
67
|
|
|
|
|
|
|
=head2 detect |
68
|
|
|
|
|
|
|
|
69
|
|
|
|
|
|
|
my %detected = detect( %params ) |
70
|
|
|
|
|
|
|
|
71
|
|
|
|
|
|
|
Tries to detect CMS, framework, etc for given html code, http headers, url. |
72
|
|
|
|
|
|
|
|
73
|
|
|
|
|
|
|
Available parameters: |
74
|
|
|
|
|
|
|
|
75
|
|
|
|
|
|
|
html - html code of web page |
76
|
|
|
|
|
|
|
headers - hash ref to http headers list |
77
|
|
|
|
|
|
|
url - url of web page |
78
|
|
|
|
|
|
|
cats - array ref to a list of trying categories, defaults to all categories; |
79
|
|
|
|
|
|
|
less cats => less cpu usage |
80
|
|
|
|
|
|
|
|
81
|
|
|
|
|
|
|
Returns the hash of detected applications by categorie: |
82
|
|
|
|
|
|
|
|
83
|
|
|
|
|
|
|
( |
84
|
|
|
|
|
|
|
cms => [ 'Joomla' ], |
85
|
|
|
|
|
|
|
'javascript-frameworks' => [ 'jQuery', 'jQuery UI' ], |
86
|
|
|
|
|
|
|
) |
87
|
|
|
|
|
|
|
|
88
|
|
|
|
|
|
|
=cut |
89
|
|
|
|
|
|
|
|
90
|
|
|
|
|
|
|
sub detect { |
91
|
9
|
|
|
9
|
1
|
5572
|
my %params = @_; |
92
|
|
|
|
|
|
|
|
93
|
9
|
50
|
66
|
|
|
33
|
return () unless $params{html} || $params{headers} || $params{url}; |
|
|
|
66
|
|
|
|
|
94
|
|
|
|
|
|
|
|
95
|
|
|
|
|
|
|
# search will be case insensitive |
96
|
9
|
100
|
|
|
|
19
|
if ( $params{html} ) { |
97
|
7
|
|
|
|
|
259
|
$params{html} = lc $params{html}; |
98
|
|
|
|
|
|
|
} |
99
|
9
|
100
|
|
|
|
3513
|
if ( $params{url} ) { |
100
|
1
|
|
|
|
|
10
|
$params{url} = lc $params{url}; |
101
|
|
|
|
|
|
|
} |
102
|
9
|
100
|
|
|
|
23
|
if ( $params{headers} ) { |
103
|
|
|
|
|
|
|
# do not modify orig headers |
104
|
5
|
|
|
|
|
6
|
my %tmp; |
105
|
5
|
|
|
|
|
8
|
$tmp{$_} = lc $params{headers}{$_} for keys %{ $params{headers} }; |
|
5
|
|
|
|
|
29
|
|
106
|
5
|
|
|
|
|
11
|
$params{headers} = \%tmp; |
107
|
|
|
|
|
|
|
} |
108
|
|
|
|
|
|
|
|
109
|
|
|
|
|
|
|
# Lazy load and process clues from JSON file |
110
|
9
|
100
|
|
|
|
25
|
_load_categories() unless scalar keys %_categories; |
111
|
|
|
|
|
|
|
|
112
|
|
|
|
|
|
|
my @cats = $params{cats} && ( ref( $params{cats} ) || '' ) eq 'ARRAY' |
113
|
9
|
100
|
66
|
|
|
47
|
? @{ $params{cats} } : get_categories(); |
|
2
|
|
|
|
|
5
|
|
114
|
|
|
|
|
|
|
|
115
|
9
|
|
|
|
|
20
|
my $headers_ref; |
116
|
9
|
100
|
|
|
|
19
|
if ( $params{headers} ) { |
117
|
|
|
|
|
|
|
# make all headers name lowercase |
118
|
5
|
|
|
|
|
7
|
while ( my ( $name, $value ) = each %{ $params{headers} } ) { |
|
11
|
|
|
|
|
33
|
|
119
|
6
|
|
|
|
|
17
|
$headers_ref->{ lc $name } = $value; |
120
|
|
|
|
|
|
|
} |
121
|
|
|
|
|
|
|
} |
122
|
|
|
|
|
|
|
|
123
|
9
|
|
|
|
|
9
|
my %detected; |
124
|
|
|
|
|
|
|
my %tried_multi_cat_apps; |
125
|
9
|
|
|
|
|
15
|
for my $cat ( @cats ) { |
126
|
363
|
50
|
|
|
|
614
|
my $apps_ref = $_categories{ $cat } or die "Unknown categorie $cat"; |
127
|
|
|
|
|
|
|
|
128
|
|
|
|
|
|
|
APP: |
129
|
363
|
|
|
|
|
362
|
for my $app_ref ( @$apps_ref ) { |
130
|
|
|
|
|
|
|
|
131
|
6113
|
|
|
|
|
3441
|
my $detected; |
132
|
|
|
|
|
|
|
|
133
|
|
|
|
|
|
|
# Some speed optimizations |
134
|
6113
|
100
|
66
|
|
|
15365
|
if ( @cats > 1 && $app_ref->{multi_cat} |
|
|
|
100
|
|
|
|
|
135
|
|
|
|
|
|
|
&& exists $tried_multi_cat_apps{ $app_ref->{name} } |
136
|
|
|
|
|
|
|
) { |
137
|
506
|
|
|
|
|
439
|
$detected = $tried_multi_cat_apps{ $app_ref->{name} }; |
138
|
|
|
|
|
|
|
} |
139
|
|
|
|
|
|
|
else { |
140
|
|
|
|
|
|
|
# Try regexes... |
141
|
5607
|
|
|
|
|
3485
|
my $confidence = 0; |
142
|
|
|
|
|
|
|
|
143
|
5607
|
100
|
66
|
|
|
8129
|
if ( defined $headers_ref && exists $app_ref->{headers_rules} ) { |
144
|
887
|
|
|
|
|
541
|
my %headers_rules = %{ $app_ref->{headers_rules} }; |
|
887
|
|
|
|
|
2138
|
|
145
|
887
|
|
|
|
|
1477
|
while ( my ( $header, $rule ) = each %headers_rules ) { |
146
|
1090
|
100
|
|
|
|
2150
|
my $header_val = $headers_ref->{ $header } or next; |
147
|
|
|
|
|
|
|
|
148
|
329
|
100
|
|
|
|
1358
|
if ( $header_val =~ m/$rule->{re}/ ) { |
149
|
6
|
|
|
|
|
11
|
$confidence += $rule->{confidence}; |
150
|
6
|
50
|
|
|
|
9
|
if ( $confidence >= 100 ) { |
151
|
6
|
|
|
|
|
9
|
$detected = 1; |
152
|
6
|
|
|
|
|
12
|
last; |
153
|
|
|
|
|
|
|
} |
154
|
|
|
|
|
|
|
} |
155
|
|
|
|
|
|
|
} |
156
|
|
|
|
|
|
|
} |
157
|
|
|
|
|
|
|
|
158
|
5607
|
100
|
|
|
|
6032
|
unless ( $detected ) { |
159
|
|
|
|
|
|
|
# try from most to least relevant method |
160
|
|
|
|
|
|
|
RULES: |
161
|
5601
|
|
|
|
|
4271
|
for my $rule_type ( qw( html url ) ) { |
162
|
11196
|
|
|
|
|
8607
|
my $rule_name = $rule_type. '_rules'; |
163
|
11196
|
100
|
66
|
|
|
18185
|
if ( defined $params{ $rule_type } && exists $app_ref->{ $rule_name } ) { |
164
|
3391
|
|
|
|
|
2100
|
for my $rule ( @{ $app_ref->{ $rule_name } } ) { |
|
3391
|
|
|
|
|
4712
|
|
165
|
3505
|
100
|
|
|
|
28706
|
if ( $params{ $rule_type } =~ m/$rule->{re}/ ) { |
166
|
9
|
|
|
|
|
14
|
$confidence += $rule->{confidence}; |
167
|
9
|
100
|
|
|
|
19
|
if ( $confidence >= 100 ) { |
168
|
7
|
|
|
|
|
6
|
$detected = 1; |
169
|
7
|
|
|
|
|
15
|
last RULES; |
170
|
|
|
|
|
|
|
} |
171
|
|
|
|
|
|
|
} |
172
|
|
|
|
|
|
|
} |
173
|
|
|
|
|
|
|
} |
174
|
|
|
|
|
|
|
} |
175
|
|
|
|
|
|
|
} |
176
|
|
|
|
|
|
|
|
177
|
|
|
|
|
|
|
# Some speed optimizations |
178
|
5607
|
100
|
66
|
|
|
10382
|
if ( @cats > 1 && $app_ref->{multi_cat} ) { |
179
|
455
|
|
|
|
|
494
|
$tried_multi_cat_apps{ $app_ref->{name} } = $detected; |
180
|
|
|
|
|
|
|
} |
181
|
|
|
|
|
|
|
} |
182
|
|
|
|
|
|
|
|
183
|
6113
|
100
|
|
|
|
8592
|
next unless $detected; |
184
|
|
|
|
|
|
|
|
185
|
|
|
|
|
|
|
# Detected! |
186
|
13
|
|
|
|
|
14
|
push @{ $detected{ $cat } }, $app_ref->{name}; |
|
13
|
|
|
|
|
32
|
|
187
|
|
|
|
|
|
|
|
188
|
13
|
100
|
|
|
|
38
|
last APP unless $MULTIPLE_APP_CATS{ $cat }; |
189
|
|
|
|
|
|
|
} |
190
|
|
|
|
|
|
|
} |
191
|
|
|
|
|
|
|
|
192
|
9
|
|
|
|
|
138
|
return %detected; |
193
|
|
|
|
|
|
|
} |
194
|
|
|
|
|
|
|
|
195
|
|
|
|
|
|
|
=head2 get_categories |
196
|
|
|
|
|
|
|
|
197
|
|
|
|
|
|
|
my @cats = get_categories() |
198
|
|
|
|
|
|
|
|
199
|
|
|
|
|
|
|
Returns the array of all application categories. |
200
|
|
|
|
|
|
|
|
201
|
|
|
|
|
|
|
=cut |
202
|
|
|
|
|
|
|
|
203
|
|
|
|
|
|
|
sub get_categories { |
204
|
|
|
|
|
|
|
# Lazy load and process clues from JSON files |
205
|
8
|
100
|
|
8
|
1
|
22
|
_load_categories() unless scalar keys %_categories; |
206
|
|
|
|
|
|
|
|
207
|
8
|
|
|
|
|
144
|
return keys %_categories; |
208
|
|
|
|
|
|
|
} |
209
|
|
|
|
|
|
|
|
210
|
|
|
|
|
|
|
# Loads and processes clues from JSON files |
211
|
|
|
|
|
|
|
sub _load_categories { |
212
|
|
|
|
|
|
|
|
213
|
2
|
|
|
2
|
|
4
|
for my $clue_file ( @_clues_file_list ) { |
214
|
3
|
50
|
|
|
|
244
|
open my $fh, '<', $clue_file |
215
|
|
|
|
|
|
|
or die "Can not read clues file $clue_file."; |
216
|
|
|
|
|
|
|
|
217
|
3
|
|
|
|
|
14
|
local $/ = undef; |
218
|
3
|
|
|
|
|
1734
|
my $json = <$fh>; |
219
|
3
|
|
|
|
|
66
|
close $fh; |
220
|
|
|
|
|
|
|
|
221
|
|
|
|
|
|
|
# Replace html entities with oridinary symbols |
222
|
3
|
|
|
|
|
895
|
$json =~ s{>}{>}xig; |
223
|
3
|
|
|
|
|
901
|
$json =~ s{<}{<}xig; |
224
|
|
|
|
|
|
|
|
225
|
3
|
|
|
|
|
6
|
my $cfg_ref = eval { JSON::decode_json( $json ) }; |
|
3
|
|
|
|
|
4199
|
|
226
|
|
|
|
|
|
|
|
227
|
3
|
50
|
|
|
|
10
|
die "Can't parse clue file $clue_file: $@" if $@; |
228
|
|
|
|
|
|
|
|
229
|
|
|
|
|
|
|
my $cats_ref = $cfg_ref->{categories} |
230
|
3
|
50
|
|
|
|
12
|
or die "Broken clues file $clue_file. Can not find categories."; |
231
|
|
|
|
|
|
|
|
232
|
|
|
|
|
|
|
my $apps_ref = $cfg_ref->{apps} |
233
|
3
|
50
|
|
|
|
10
|
or die "Broken clues file $clue_file. Can not find applications."; |
234
|
|
|
|
|
|
|
|
235
|
|
|
|
|
|
|
# Process apps |
236
|
3
|
|
|
|
|
12
|
while ( my ( $app, $app_ref ) = each %$apps_ref ) { |
237
|
|
|
|
|
|
|
|
238
|
1695
|
50
|
|
|
|
2422
|
my $new_app_ref = _process_app_clues( $app, $app_ref ) or next; |
239
|
|
|
|
|
|
|
|
240
|
1695
|
50
|
|
|
|
1253
|
my @cats = @{ $app_ref->{cats} } or next; |
|
1695
|
|
|
|
|
4865
|
|
241
|
|
|
|
|
|
|
|
242
|
1695
|
100
|
|
|
|
2712
|
$new_app_ref->{multi_cat} = 1 if @cats > 1; |
243
|
|
|
|
|
|
|
|
244
|
1695
|
|
|
|
|
2120
|
for my $cat_id ( @cats ) { |
245
|
1855
|
50
|
|
|
|
4856
|
my $cat = $cats_ref->{ $cat_id } or next; |
246
|
|
|
|
|
|
|
|
247
|
1855
|
|
|
|
|
1326
|
push @{ $_categories{ $cat } }, $new_app_ref; |
|
1855
|
|
|
|
|
13677
|
|
248
|
|
|
|
|
|
|
} |
249
|
|
|
|
|
|
|
} |
250
|
|
|
|
|
|
|
} |
251
|
|
|
|
|
|
|
} |
252
|
|
|
|
|
|
|
|
253
|
|
|
|
|
|
|
# Process clues of given app |
254
|
|
|
|
|
|
|
sub _process_app_clues { |
255
|
1695
|
|
|
1695
|
|
1329
|
my ( $app, $app_ref ) = @_; |
256
|
|
|
|
|
|
|
|
257
|
1695
|
|
|
|
|
2807
|
my $new_app_ref = { name => $app }; |
258
|
|
|
|
|
|
|
|
259
|
1695
|
|
|
|
|
1845
|
my @fields = grep { exists $app_ref->{$_} } qw( script html meta headers url ); |
|
8475
|
|
|
|
|
10567
|
|
260
|
1695
|
|
|
|
|
1103
|
my @html_rules; |
261
|
|
|
|
|
|
|
# Precompile regexps |
262
|
1695
|
|
|
|
|
1585
|
for my $field ( @fields ) { |
263
|
2113
|
|
|
|
|
2854
|
my $rule_ref = $app_ref->{ $field }; |
264
|
|
|
|
|
|
|
my @rules_list = !ref $rule_ref ? _parse_rule( $rule_ref ) |
265
|
2113
|
100
|
|
|
|
4891
|
: ref $rule_ref eq 'ARRAY' ? ( map { _parse_rule( $_ ) } @$rule_ref ) |
|
186
|
100
|
|
|
|
300
|
|
266
|
|
|
|
|
|
|
: () ; |
267
|
|
|
|
|
|
|
|
268
|
2113
|
100
|
|
|
|
5215
|
if ( $field eq 'html' ) { |
|
|
100
|
|
|
|
|
|
|
|
100
|
|
|
|
|
|
|
|
100
|
|
|
|
|
|
|
|
50
|
|
|
|
|
|
269
|
523
|
|
|
|
|
767
|
push @html_rules, map { $_->{re} = qr/(?-x:$_->{re})/; $_ } @rules_list; |
|
567
|
|
|
|
|
15498
|
|
|
567
|
|
|
|
|
1999
|
|
270
|
|
|
|
|
|
|
} |
271
|
|
|
|
|
|
|
elsif ( $field eq 'script' ) { |
272
|
|
|
|
|
|
|
push @html_rules, |
273
|
|
|
|
|
|
|
map { |
274
|
547
|
|
|
|
|
738
|
$_->{re} = qr/ |
|
599
|
|
|
|
|
19281
|
|
275
|
|
|
|
|
|
|
< \s* script [^>]+ src \s* = \s* ["'] [^"']* (?-x:$_->{re}) [^"']* ["'] |
276
|
|
|
|
|
|
|
/x; |
277
|
599
|
|
|
|
|
2035
|
$_ |
278
|
|
|
|
|
|
|
} @rules_list; |
279
|
|
|
|
|
|
|
} |
280
|
|
|
|
|
|
|
elsif ( $field eq 'url' ) { |
281
|
72
|
|
|
|
|
130
|
my @url_rules = map { $_->{re} = qr/(?-x:$_->{re})/; $_ } @rules_list; |
|
72
|
|
|
|
|
1178
|
|
|
72
|
|
|
|
|
223
|
|
282
|
72
|
|
|
|
|
169
|
$new_app_ref->{url_rules} = _optimize_rules( \@url_rules ); |
283
|
|
|
|
|
|
|
} |
284
|
|
|
|
|
|
|
elsif ( $field eq 'meta' ) { |
285
|
365
|
|
|
|
|
1046
|
for my $key ( keys %$rule_ref ) { |
286
|
397
|
|
|
|
|
536
|
my $lc_key = lc $key; |
287
|
397
|
|
|
|
|
3127
|
my $name_re = qr/ name \s* = \s* ["']? $lc_key ["']? /x; |
288
|
397
|
|
|
|
|
777
|
my $rule = _parse_rule( $rule_ref->{$key} ); |
289
|
397
|
|
|
|
|
6128
|
$rule->{re} = qr/$rule->{re}/; |
290
|
397
|
|
|
|
|
9154
|
my $content_re = qr/ content \s* = \s* ["'] [^"']* (?-x:$rule->{re}) [^"']* ["'] /x; |
291
|
|
|
|
|
|
|
|
292
|
397
|
|
|
|
|
11794
|
$rule->{re} = qr/ |
293
|
|
|
|
|
|
|
< \s* meta \s+ |
294
|
|
|
|
|
|
|
(?: |
295
|
|
|
|
|
|
|
(?: $name_re \s+ $content_re ) |
296
|
|
|
|
|
|
|
# | (?: $content_re \s+ $name_re ) # hangs sometimes |
297
|
|
|
|
|
|
|
) |
298
|
|
|
|
|
|
|
/x; |
299
|
|
|
|
|
|
|
|
300
|
397
|
|
|
|
|
1920
|
push @html_rules, $rule; |
301
|
|
|
|
|
|
|
} |
302
|
|
|
|
|
|
|
} |
303
|
|
|
|
|
|
|
elsif ( $field eq 'headers' ) { |
304
|
606
|
|
|
|
|
1551
|
for my $key ( keys %$rule_ref ) { |
305
|
714
|
|
|
|
|
1204
|
my $rule = _parse_rule( $rule_ref->{$key} ); |
306
|
714
|
|
|
|
|
10745
|
$rule->{re} = qr/$rule->{re}/; |
307
|
714
|
|
|
|
|
2820
|
$new_app_ref->{headers_rules}{ lc $key } = $rule; |
308
|
|
|
|
|
|
|
} |
309
|
|
|
|
|
|
|
} |
310
|
|
|
|
|
|
|
} |
311
|
|
|
|
|
|
|
|
312
|
1695
|
100
|
|
|
|
2833
|
if ( @html_rules ) { |
313
|
1121
|
|
|
|
|
1646
|
$new_app_ref->{html_rules} = _optimize_rules( \@html_rules ); |
314
|
|
|
|
|
|
|
} |
315
|
|
|
|
|
|
|
|
316
|
1695
|
|
|
|
|
5062
|
return $new_app_ref; |
317
|
|
|
|
|
|
|
} |
318
|
|
|
|
|
|
|
|
319
|
|
|
|
|
|
|
# separate regexp and other optional parameters from the rule |
320
|
|
|
|
|
|
|
sub _parse_rule { |
321
|
2349
|
|
|
2349
|
|
2209
|
my ( $rule ) = @_; |
322
|
|
|
|
|
|
|
|
323
|
2349
|
|
|
|
|
5023
|
my ( $re, @params ) = split /\\;/, $rule; |
324
|
|
|
|
|
|
|
|
325
|
2349
|
|
|
|
|
1808
|
my $confidence; |
326
|
2349
|
|
|
|
|
2467
|
for my $param ( @params ) { |
327
|
628
|
100
|
|
|
|
1694
|
if ( ( $confidence ) = $param =~ /^\s*confidence\s*:\s*(\d+)\s*$/ ) { |
328
|
|
|
|
|
|
|
# supports only confidence for now |
329
|
58
|
|
|
|
|
74
|
last; |
330
|
|
|
|
|
|
|
} |
331
|
|
|
|
|
|
|
} |
332
|
|
|
|
|
|
|
|
333
|
|
|
|
|
|
|
return { |
334
|
2349
|
100
|
100
|
|
|
3814
|
re => _escape_re( defined( $re ) ? $re : '' ), |
335
|
|
|
|
|
|
|
confidence => $confidence || 100, |
336
|
|
|
|
|
|
|
}; |
337
|
|
|
|
|
|
|
} |
338
|
|
|
|
|
|
|
|
339
|
|
|
|
|
|
|
# Escape special symbols in regexp string of config file |
340
|
|
|
|
|
|
|
sub _escape_re { |
341
|
2349
|
|
|
2349
|
|
3313
|
my ( $re ) = @_; |
342
|
|
|
|
|
|
|
|
343
|
|
|
|
|
|
|
# Escape { } braces |
344
|
|
|
|
|
|
|
#$re =~ s/ ([{}]) /[$1]/xig; |
345
|
|
|
|
|
|
|
|
346
|
|
|
|
|
|
|
# Escape [^] |
347
|
2349
|
|
|
|
|
3245
|
$re =~ s{\Q[^]\E}{[\\^]}ig; |
348
|
|
|
|
|
|
|
|
349
|
|
|
|
|
|
|
# Escape \\1 |
350
|
2349
|
|
|
|
|
2001
|
$re =~ s{\Q\1\E}{\\\\1}ig; |
351
|
|
|
|
|
|
|
|
352
|
|
|
|
|
|
|
# Escape (?! |
353
|
2349
|
|
|
|
|
1850
|
$re =~ s{[(][?][!]}{([?]!}ig; |
354
|
|
|
|
|
|
|
|
355
|
|
|
|
|
|
|
# turn literals in regexp to lowercase to make case insensitive search |
356
|
|
|
|
|
|
|
# i flag will be slower because we makes many searches in one text |
357
|
2
|
|
|
2
|
|
14
|
no warnings 'redefine'; |
|
2
|
|
|
|
|
3
|
|
|
2
|
|
|
|
|
418
|
|
358
|
2349
|
|
|
0
|
|
7672
|
local *Regexp::Parser::warn = sub {}; # it may be too noisy |
359
|
|
|
|
|
|
|
|
360
|
2349
|
|
|
|
|
5924
|
my $parser = Regexp::Parser->new(); |
361
|
2349
|
100
|
|
|
|
1105100
|
if ( $parser->regex($re) ) { |
362
|
2263
|
|
|
|
|
1147955
|
$re = ''; |
363
|
|
|
|
|
|
|
|
364
|
2263
|
|
|
|
|
4191
|
while ( my $node = $parser->next ) { |
365
|
55012
|
|
|
|
|
3011125
|
my $ref = ref $node; |
366
|
55012
|
100
|
100
|
|
|
106465
|
if ( $ref eq 'Regexp::Parser::exact' || $ref eq 'Regexp::Parser::anyof_char' ) { |
367
|
45004
|
|
|
|
|
62226
|
$re .= lc $node->raw; |
368
|
|
|
|
|
|
|
} |
369
|
|
|
|
|
|
|
else { |
370
|
10008
|
|
|
|
|
15698
|
$re .= $node->raw; |
371
|
|
|
|
|
|
|
} |
372
|
|
|
|
|
|
|
} |
373
|
|
|
|
|
|
|
} |
374
|
|
|
|
|
|
|
|
375
|
2349
|
|
|
|
|
110111
|
return $re; |
376
|
|
|
|
|
|
|
} |
377
|
|
|
|
|
|
|
|
378
|
|
|
|
|
|
|
# If possible combine all rules in one regexp |
379
|
|
|
|
|
|
|
sub _optimize_rules { |
380
|
1193
|
|
|
1193
|
|
1169
|
my ( $rules ) = @_; |
381
|
|
|
|
|
|
|
|
382
|
1193
|
100
|
100
|
|
|
2700
|
if ( @$rules > 1 && @$rules == grep { $_->{confidence} == 100 } @$rules ) { |
|
784
|
|
|
|
|
1825
|
|
383
|
|
|
|
|
|
|
# can combine only if confidence for each is 100 |
384
|
317
|
|
|
|
|
624
|
my $re = join '|', map { $_->{re} } @$rules; |
|
720
|
|
|
|
|
1326
|
|
385
|
|
|
|
|
|
|
return [{ |
386
|
317
|
|
|
|
|
17361
|
re => qr/$re/, |
387
|
|
|
|
|
|
|
confidence => 100, |
388
|
|
|
|
|
|
|
}]; |
389
|
|
|
|
|
|
|
} |
390
|
|
|
|
|
|
|
|
391
|
876
|
|
|
|
|
1448
|
return $rules; |
392
|
|
|
|
|
|
|
} |
393
|
|
|
|
|
|
|
|
394
|
|
|
|
|
|
|
=head2 add_clues_file |
395
|
|
|
|
|
|
|
|
396
|
|
|
|
|
|
|
add_clues_file( $filepath ) |
397
|
|
|
|
|
|
|
|
398
|
|
|
|
|
|
|
Puts additional clues file to a list of processed clues files. |
399
|
|
|
|
|
|
|
See apps.json as format sample. |
400
|
|
|
|
|
|
|
|
401
|
|
|
|
|
|
|
=cut |
402
|
|
|
|
|
|
|
|
403
|
|
|
|
|
|
|
sub add_clues_file { |
404
|
1
|
|
|
1
|
1
|
510
|
my ( $filepath ) = @_; |
405
|
|
|
|
|
|
|
|
406
|
1
|
|
|
|
|
2
|
push @_clues_file_list, $filepath; |
407
|
|
|
|
|
|
|
|
408
|
|
|
|
|
|
|
# just clear out categories to lazy load later |
409
|
1
|
|
|
|
|
3008
|
%_categories = (); |
410
|
|
|
|
|
|
|
} |
411
|
|
|
|
|
|
|
|
412
|
|
|
|
|
|
|
=head1 AUTHOR |
413
|
|
|
|
|
|
|
|
414
|
|
|
|
|
|
|
Alexander Nalobin, C<< >> |
415
|
|
|
|
|
|
|
|
416
|
|
|
|
|
|
|
=head1 BUGS |
417
|
|
|
|
|
|
|
|
418
|
|
|
|
|
|
|
Please report any bugs or feature requests to C, or through |
419
|
|
|
|
|
|
|
the web interface at L. I will be notified, and then you'll |
420
|
|
|
|
|
|
|
automatically be notified of progress on your bug as I make changes. |
421
|
|
|
|
|
|
|
|
422
|
|
|
|
|
|
|
|
423
|
|
|
|
|
|
|
=head1 SUPPORT |
424
|
|
|
|
|
|
|
|
425
|
|
|
|
|
|
|
You can find documentation for this module with the perldoc command. |
426
|
|
|
|
|
|
|
|
427
|
|
|
|
|
|
|
perldoc WWW::Wappalyzer |
428
|
|
|
|
|
|
|
|
429
|
|
|
|
|
|
|
|
430
|
|
|
|
|
|
|
You can also look for information at: |
431
|
|
|
|
|
|
|
|
432
|
|
|
|
|
|
|
=over 4 |
433
|
|
|
|
|
|
|
|
434
|
|
|
|
|
|
|
=item * GitHub |
435
|
|
|
|
|
|
|
|
436
|
|
|
|
|
|
|
L |
437
|
|
|
|
|
|
|
|
438
|
|
|
|
|
|
|
=item * RT: CPAN's request tracker (report bugs here) |
439
|
|
|
|
|
|
|
|
440
|
|
|
|
|
|
|
L |
441
|
|
|
|
|
|
|
|
442
|
|
|
|
|
|
|
=item * AnnoCPAN: Annotated CPAN documentation |
443
|
|
|
|
|
|
|
|
444
|
|
|
|
|
|
|
L |
445
|
|
|
|
|
|
|
|
446
|
|
|
|
|
|
|
=item * CPAN Ratings |
447
|
|
|
|
|
|
|
|
448
|
|
|
|
|
|
|
L |
449
|
|
|
|
|
|
|
|
450
|
|
|
|
|
|
|
=item * Search CPAN |
451
|
|
|
|
|
|
|
|
452
|
|
|
|
|
|
|
L |
453
|
|
|
|
|
|
|
|
454
|
|
|
|
|
|
|
=back |
455
|
|
|
|
|
|
|
|
456
|
|
|
|
|
|
|
|
457
|
|
|
|
|
|
|
=head1 ACKNOWLEDGEMENTS |
458
|
|
|
|
|
|
|
|
459
|
|
|
|
|
|
|
|
460
|
|
|
|
|
|
|
=head1 LICENSE AND COPYRIGHT |
461
|
|
|
|
|
|
|
|
462
|
|
|
|
|
|
|
Copyright 2013-2015 Alexander Nalobin. |
463
|
|
|
|
|
|
|
|
464
|
|
|
|
|
|
|
This program is free software; you can redistribute it and/or modify it |
465
|
|
|
|
|
|
|
under the terms of either: the GNU General Public License as published |
466
|
|
|
|
|
|
|
by the Free Software Foundation; or the Artistic License. |
467
|
|
|
|
|
|
|
|
468
|
|
|
|
|
|
|
See http://dev.perl.org/licenses/ for more information. |
469
|
|
|
|
|
|
|
|
470
|
|
|
|
|
|
|
|
471
|
|
|
|
|
|
|
=cut |
472
|
|
|
|
|
|
|
|
473
|
|
|
|
|
|
|
1; # End of WWW::Wappalyzer |