File Coverage

blib/lib/Metabrik/Lookup/Threatlist.pm
Criterion Covered Total %
statement 9 86 10.4
branch 0 62 0.0
condition 0 21 0.0
subroutine 3 6 50.0
pod 1 3 33.3
total 13 178 7.3


line stmt bran cond sub pod time code
1             #
2             # $Id$
3             #
4             # lookup::threatlist Brik
5             #
6             package Metabrik::Lookup::Threatlist;
7 1     1   774 use strict;
  1         2  
  1         61  
8 1     1   7 use warnings;
  1         2  
  1         30  
9              
10 1     1   5 use base qw(Metabrik);
  1         2  
  1         1305  
11              
12             sub brik_properties {
13             return {
14 0     0 1   revision => '$Revision$',
15             tags => [ qw(unstable ipv4 ipv6 ip threat) ],
16             author => 'GomoR ',
17             license => 'http://opensource.org/licenses/BSD-3-Clause',
18             attributes => {
19             datadir => [ qw(datadir) ],
20             },
21             commands => {
22             update => [ ],
23             from_ipv4 => [ qw(ipv4_address) ],
24             },
25             require_modules => {
26             'Metabrik::Client::Www' => [ ],
27             'Metabrik::File::Compress' => [ ],
28             'Metabrik::File::Text' => [ ],
29             'Metabrik::Network::Address' => [ ],
30             },
31             };
32             }
33              
34             sub update {
35 0     0 0   my $self = shift;
36              
37 0           my $datadir = $self->datadir;
38              
39 0           my %mirror = (
40             'iblocklist-tgbankumtwtrzllndbmb.gz' => 'http://list.iblocklist.com/?list=logmein',
41             'iblocklist-nzldzlpkgrcncdomnttb.gz' => 'http://list.iblocklist.com/?list=nzldzlpkgrcncdomnttb',
42             'iblocklist-xoebmbyexwuiogmbyprb.gz' => 'http://list.iblocklist.com/?list=bt_proxy',
43             'iblocklist-zfucwtjkfwkalytktyiw.gz' => 'http://list.iblocklist.com/?list=zfucwtjkfwkalytktyiw',
44             'iblocklist-llvtlsjyoyiczbkjsxpf.gz' => 'http://list.iblocklist.com/?list=bt_spyware',
45             'iblocklist-togdoptykrlolpddwbvz.gz' => 'http://list.iblocklist.com/?list=tor',
46             'iblocklist-ghlzqtqxnzctvvajwwag.gz' => 'http://list.iblocklist.com/?list=ghlzqtqxnzctvvajwwag',
47             'sans-block.txt' => 'http://isc.sans.edu/block.txt',
48             'malwaredomains-domains.txt' => 'http://mirror1.malwaredomains.com/files/domains.txt',
49             'emergingthreats-compromised-ips.txt.gz' => 'http://rules.emergingthreats.net/blockrules/compromised-ips.txt',
50             'emergingthreats-emerging-Block-IPs.txt.gz' => 'http://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt',
51             'phishtank-verified_online.csv.gz' => 'http://data.phishtank.com/data/online-valid.csv.gz',
52             'abusech-palevotracker.txt.gz' => 'https://palevotracker.abuse.ch/blocklists.php?download=ipblocklist',
53             'abusech-spyeyetracker.txt.gz' => 'https://spyeyetracker.abuse.ch/blocklist.php?download=ipblocklist',
54             'abusech-zeustracker-badips.txt.gz' => 'https://zeustracker.abuse.ch/blocklist.php?download=badips',
55             'abusech-zeustracker.txt.gz' => 'https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist',
56             'iana-tlds-alpha-by-domain.txt' => 'http://data.iana.org/TLD/tlds-alpha-by-domain.txt',
57             'publicsuffix-effective_tld_names.dat.gz' => 'https://publicsuffix.org/list/effective_tld_names.dat',
58             );
59              
60             # IP Threatlist:
61             # "abusech-palevotracker.txt", # Palevo C&C
62             # "abusech-zeustracker-badips.txt", # Zeus IPs
63             # "abusech-zeustracker.txt", # Zeus IPs
64             # "emergingthreats-compromised-ips.txt", # Compromised IPs
65             # "emergingthreats-emerging-Block-IPs.txt", # Raw IPs from Spamhaus, DShield and Abuse.ch
66             # "iblocklist-ghlzqtqxnzctvvajwwag", # Various exploiters, scanner, spammers IPs
67             # "iblocklist-llvtlsjyoyiczbkjsxpf", # Various evil IPs (?)
68             # "iblocklist-xoebmbyexwuiogmbyprb", # Proxy and TOR IPs
69             # "sans-block.txt", # IP ranges to block for abuse reasons
70              
71             # Owner lists
72             # "iblocklist-nzldzlpkgrcncdomnttb", # ThePirateBay
73             # "iblocklist-togdoptykrlolpddwbvz", # TOR IPs
74             # "iblocklist-tgbankumtwtrzllndbmb", # LogMeIn IPs
75             # "iblocklist-zfucwtjkfwkalytktyiw", # RapidShare IPs
76             # "phishtank-verified_online.csv", # URLs hosting phishings
77             # "malwaredomains-domains.txt", # Malware domains
78              
79             # Other lists
80             # "top-1m.csv",
81             # "iana-tlds-alpha-by-domain.txt",
82             # "publicsuffix-effective_tld_names.dat",
83              
84 0 0         my $cw = Metabrik::Client::Www->new_from_brik_init($self) or return;
85 0           $cw->user_agent("Metabrik-Lookup-Threatlist-mirror/1.00");
86 0           $cw->datadir($datadir);
87              
88 0 0         my $fc = Metabrik::File::Compress->new_from_brik_init($self) or return;
89 0           $fc->datadir($datadir);
90              
91 0           my @updated = ();
92 0           for my $f (keys %mirror) {
93 0 0         my $files = $cw->mirror($mirror{$f}, $f) or next;
94 0           for my $file (@$files) {
95 0           my $outfile = $file;
96 0 0         if ($file =~ /\.gz$/) {
    0          
97 0           ($outfile = $file) =~ s/\.gz$//;
98 0 0         $fc->uncompress($file, $outfile) or next;
99             }
100             elsif ($file =~ /\.zip$/) {
101 0           ($outfile = $file) =~ s/\.zip$//;
102 0 0         $fc->uncompress($file, $outfile) or next;
103             }
104 0           push @updated, $outfile;
105             }
106             }
107              
108 0           return \@updated;
109             }
110              
111             sub from_ipv4 {
112 0     0 0   my $self = shift;
113 0           my ($ipv4) = @_;
114              
115 0 0         $self->brik_help_run_undef_arg('from_ipv4', $ipv4) or return;
116              
117 0 0         my $na = Metabrik::Network::Address->new_from_brik_init($self) or return;
118 0 0         if (! $na->is_ipv4($ipv4)) {
119 0           return $self->log->error("from_ipv4: not a valid IPv4 address [$ipv4]");
120             }
121              
122             # Keep only the IP part
123 0           ($ipv4) = $ipv4 =~ m{^(\d+\.\d+\.\d+\.\d+)/?.*$};
124              
125             # One IP per line format
126 0           my $lists_a = {
127             "abusech-palevotracker.txt" => 'Abuse.ch - Palevo C&C',
128             "abusech-zeustracker-badips.txt" => 'Abuse.ch - Zeus bad IPs',
129             "abusech-zeustracker.txt" => 'Abuse.ch - Zeus IPs',
130             "emergingthreats-compromised-ips.txt" => 'EmergingThreats - Compromised IPs',
131             "emergingthreats-emerging-Block-IPs.txt" => 'EmergingThreats - Spamhaus, DShield and Abuse.ch',
132             };
133              
134             # CSV-like format
135 0           my $lists_b = {
136             "iblocklist-ghlzqtqxnzctvvajwwag" => 'iblocklist - Exploiters, scanner and spammers',
137             "iblocklist-llvtlsjyoyiczbkjsxpf" => 'iblocklist - Malicious IPs',
138             "iblocklist-xoebmbyexwuiogmbyprb" => 'iblocklist - Proxy and TOR',
139             };
140              
141             # Custom format
142 0           my $lists_c = {
143             "sans-block.txt" => 'SANS - Malicious IPs',
144             };
145              
146 0           my $datadir = $self->datadir;
147              
148 0 0         my $ft = Metabrik::File::Text->new_from_brik_init($self) or return;
149 0           $ft->as_array(1);
150 0           $ft->strip_crlf(1);
151              
152 0           my $level = $self->log->level;
153 0           $self->log->level(1);
154              
155 0           my %threats = ();
156 0           for my $file (keys %$lists_a) {
157 0 0         my $data = $ft->read($datadir.'/'.$file) or next;
158 0           for (@$data) {
159 0 0         next if (/^\s*#/);
160 0 0         next if (/^\s*$/);
161 0 0 0       if ($na->is_ipv4_subnet($_) && $na->match($ipv4, $_)) {
    0 0        
162 0           $threats{$lists_a->{$file}}++;
163             }
164             elsif ($na->is_ipv4($_) && /^$ipv4$/) {
165 0           $threats{$lists_a->{$file}}++;
166             }
167             }
168             }
169              
170 0           for my $file (keys %$lists_b) {
171 0 0         my $data = $ft->read($datadir.'/'.$file) or next;
172 0           for (@$data) {
173 0 0         next if (/^\s*#/);
174 0 0         next if (/^\s*$/);
175 0           my @toks = split(/\s*:\s*/);
176 0 0 0       next unless (defined($toks[0]) && defined($toks[1]));
177 0           my $type = $toks[0]; # Exploit scanner, WebExploit, ...
178 0           my ($start, $end) = $toks[1] =~ m{^\s*(\d+\.\d+\.\d+\.\d+)\s*-\s*(\d+\.\d+\.\d+\.\d+)\s*$};
179 0 0 0       next unless (defined($start) && defined($end));
180 0 0 0       next unless ($na->is_ipv4($start) && $na->is_ipv4($end));
181 0 0         my $subnet = $na->range_to_cidr($start, $end) or next;
182 0           for my $this (@$subnet) {
183 0 0         if ($na->match($ipv4, $this)) {
184 0           $threats{$lists_b->{$file}}++;
185             }
186             }
187             }
188             }
189              
190 0           for my $file (keys %$lists_c) {
191 0 0         my $data = $ft->read($datadir.'/'.$file) or next;
192 0           for (@$data) {
193 0 0         next if (/^\s*#/);
194 0 0         next if (/^\s*$/);
195 0           my @toks = split(/\s+/);
196 0           my $start = $toks[0];
197 0           my $end = $toks[1];
198 0 0 0       next unless (defined($start) && defined($end));
199 0 0 0       next unless ($na->is_ipv4($start) && $na->is_ipv4($end));
200 0 0         my $subnet = $na->range_to_cidr($start, $end) or next;
201 0           for my $this (@$subnet) {
202 0 0         if ($na->match($ipv4, $this)) {
203 0           $threats{$lists_c->{$file}}++;
204             }
205             }
206             }
207             }
208              
209 0           $self->log->level($level);
210              
211 0           return [ keys %threats ];
212             }
213              
214             1;
215              
216             __END__