commit e7b7f84: [Fix] Various fixes to asn module
Vsevolod Stakhov
vsevolod at highsecure.ru
Mon Jul 22 12:14:04 UTC 2019
Author: Vsevolod Stakhov
Date: 2019-07-22 13:07:21 +0100
URL: https://github.com/rspamd/rspamd/commit/e7b7f84e3e31a252c87a75c1719555de039e9e72 (HEAD -> master)
[Fix] Various fixes to asn module
Contributed by: @citrin
---
utils/asn.pl | 416 ++++++++++++++++++++++++++---------------------------------
1 file changed, 186 insertions(+), 230 deletions(-)
diff --git a/utils/asn.pl b/utils/asn.pl
index 8cc6e4afd..15d7d7862 100644
--- a/utils/asn.pl
+++ b/utils/asn.pl
@@ -1,29 +1,31 @@
#!/usr/bin/env perl
+#
+# Ansible managed
+#
use warnings;
use strict;
+use autodie;
use File::Basename;
use File::Fetch;
use Getopt::Long;
-use IPC::Cmd qw/can_run/;
use Pod::Usage;
-use LWP::Simple;
-use PerlIO::gzip;
-use URI;
+use FindBin;
+use lib "$FindBin::Bin/extlib/lib/perl5";
-$LWP::Simple::ua->show_progress(1);
+use URI;
my %config = (
- asn_sources => [
- 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest',
- 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest',
- 'ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest',
- 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest',
- 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
- ],
- bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz']
+ asn_sources => [
+ 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest',
+ 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest',
+ 'http://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest',
+ 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest',
+ 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
+ ],
+ bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz']
);
my $download_asn = 0;
@@ -39,278 +41,230 @@ my $v6_zone = "asn6.rspamd.com";
my $v4_file = "asn.zone";
my $v6_file = "asn6.zone";
my $ns_servers = [ "asn-ns.rspamd.com", "asn-ns2.rspamd.com" ];
-my $use_bgpdump = 0;
GetOptions(
- "download-asn" => \$download_asn,
- "download-bgp" => \$download_bgp,
- "4!" => \$v4,
- "6!" => \$v6,
- "parse!" => \$parse,
- "target=s" => \$download_target,
- "zone-v4=s" => \$v4_zone,
- "zone-v6=s" => \$v6_zone,
- "file-v4=s" => \$v4_file,
- "file-v6=s" => \$v6_file,
- "ns-server=s@" => \$ns_servers,
- "help|?" => \$help,
- "man" => \$man,
-) or pod2usage(2);
+ "download-asn" => \$download_asn,
+ "download-bgp" => \$download_bgp,
+ "4!" => \$v4,
+ "6!" => \$v6,
+ "parse!" => \$parse,
+ "target=s" => \$download_target,
+ "zone-v4=s" => \$v4_zone,
+ "zone-v6=s" => \$v6_zone,
+ "file-v4=s" => \$v4_file,
+ "file-v6=s" => \$v6_file,
+ "ns-server=s@" => \$ns_servers,
+ "help|?" => \$help,
+ "man" => \$man
+) or
+ pod2usage(2);
pod2usage(1) if $help;
-pod2usage( -exitval => 0, -verbose => 2 ) if $man;
-
-my $bgpdump_path = can_run('bgpdump')
- or warn 'bgpdump is not found, will try to use Net::MRT instead; results can be incomplete';
-
-sub download_file {
- my ($u) = @_;
-
- print "Fetching $u\n";
- my $ff = File::Fetch->new( uri => $u );
- my $where = $ff->fetch( to => $download_target ) or die $ff->error;
-
- return $where;
-}
+pod2usage(-exitval => 0, -verbose => 2) if $man;
if ($download_asn) {
- foreach my $u ( @{ $config{'asn_sources'} } ) {
+ foreach my $u (@{ $config{'asn_sources'} }) {
download_file($u);
}
}
if ($download_bgp) {
- foreach my $u ( @{ $config{'bgp_sources'} } ) {
+ foreach my $u (@{ $config{'bgp_sources'} }) {
download_file($u);
}
}
-if ( !$parse ) {
+if (!$parse) {
exit 0;
}
-my $v4_fh;
-my $v6_fh;
-
-if ($v4) {
- open( $v4_fh, ">", $v4_file ) or die "Cannot open $v4_file for writing: $!";
- print $v4_fh "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n";
- foreach my $ns ( @{$ns_servers} ) {
- print $v4_fh "\$NS 43200 $ns\n";
- }
-}
-if ($v6) {
- open( $v6_fh, ">", $v6_file ) or die "Cannot open $v6_file for writing: $!";
- print $v6_fh "\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300\n";
- foreach my $ns ( @{$ns_servers} ) {
- print $v6_fh "\$NS 43200 $ns\n";
- }
-}
-
-sub is_bougus_asn {
- my $as = shift;
-
- # 64496-64511 Reserved for use in documentation and sample code
- # 64512-65534 Designated for private use
- # 65535 Reserved
- # 65536-65551 Reserved for use in documentation and sample code
- # 65552-131071 Reserved
- return 1 if $as >= 64496 && $as <= 131071;
-
- # 4294967295
- return 1 if $as == 4294967295;
+# Prefix to ASN map
+my $networks = { 4 => {}, 6 => {} };
- # AS0 is reserved
- # AS1 is legal AS, but in most cases used by others without permission
- # of owner (probably lame admins use AS1 as private AS).
- return 1 if $as <= 1;
+foreach my $u (@{ $config{'bgp_sources'} }) {
+ my $parsed = URI->new($u);
+ my $fname = $download_target . '/' . basename($parsed->path);
- return 0;
-}
+ use constant {
+ F_MARKER => 0,
+ F_TIMESTAMP => 1,
+ F_PEER_IP => 3,
+ F_PEER_AS => 4,
+ F_PREFIX => 5,
+ F_AS_PATH => 6,
+ F_ORIGIN => 7,
+ };
-# Now load BGP data
-my $networks = {};
+ open(my $bgpd, '-|', "bgpdump -v -M $fname") or die "can't start bgpdump: $!";
-foreach my $u ( @{ $config{'bgp_sources'} } ) {
- my $parsed = URI->new($u);
- my $fname = $download_target . '/' . basename( $parsed->path );
-
- if ($bgpdump_path) {
- use constant {
- F_MARKER => 0,
- F_TIMESTAMP => 1,
- F_PEER_IP => 3,
- F_PEER_AS => 4,
- F_PREFIX => 5,
- F_AS_PATH => 6,
- F_ORIGIN => 7,
- };
-
- open( my $bgpd, '-|', "$bgpdump_path -v -M $fname" ) or die "can't start bgpdump: $!";
-
- while (<$bgpd>) {
- chomp;
- my @e = split /\|/;
- if ( $e[F_MARKER] ne 'TABLE_DUMP2' ) {
- warn "bad line: $_\n";
- next;
- }
+ while (<$bgpd>) {
+ chomp;
+ my @e = split /\|/;
+ if ($e[F_MARKER] ne 'TABLE_DUMP2') {
+ warn "bad line: $_\n";
+ next;
+ }
- my $origin_as;
- my $prefix = $e[F_PREFIX];
- my $ipv6 = 0;
+ my $origin_as;
+ my $prefix = $e[F_PREFIX];
+ my $ip_ver = 6;
- if ( $prefix !~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/ ) {
- $ipv6 = 1;
- }
+ if ($prefix =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/) {
+ $ip_ver = 4;
+ }
- if ( $e[F_AS_PATH] ) {
+ if ($e[F_AS_PATH]) {
- # not empty AS_PATH
- my @as_path = split /\s/, $e[F_AS_PATH];
- $origin_as = pop @as_path;
+ # not empty AS_PATH
+ my @as_path = split /\s/, $e[F_AS_PATH];
+ $origin_as = pop @as_path;
- if ( substr( $origin_as, 0, 1 ) eq '{' ) {
+ if (substr($origin_as, 0, 1) eq '{') {
- # route is aggregated
- if ( $origin_as =~ /^{(\d+)}$/ ) {
+ # route is aggregated
+ if ($origin_as =~ /^{(\d+)}$/) {
- # single AS aggregated, just remove { } around
- $origin_as = $1;
- }
- else {
- # use previous AS from AS_PATH
- $origin_as = pop @as_path;
- }
- }
+ # single AS aggregated, just remove { } around
+ $origin_as = $1;
+ } else {
- # strip bogus AS
- while ( is_bougus_asn($origin_as) ) {
+ # use previous AS from AS_PATH
$origin_as = pop @as_path;
- last if scalar @as_path == 0;
}
}
- # empty AS_PATH or all AS_PATH elements is stripped as bogus - use PEER_AS is origin AS
- $origin_as //= $e[F_PEER_AS];
-
- if ( !$networks->{$origin_as} ) {
- if ( !$ipv6 ) {
- $networks->{$origin_as} = { nets_v4 => [$prefix], nets_v6 => [] };
- }
- else {
- $networks->{$origin_as} = { nets_v6 => [$prefix], nets_v4 => [] };
- }
- }
- else {
- if ( !$ipv6 ) {
- push @{ $networks->{$origin_as}->{'nets_v4'} }, $prefix;
- }
- else {
- push @{ $networks->{$origin_as}->{'nets_v6'} }, $prefix;
- }
- }
- }
- }
- else {
- require Net::MRT;
- $Net::MRT::USE_RFC4760 = -1;
-
- open( my $fh, "<:gzip", $fname )
- or die "Cannot open $fname: $!";
- while ( my $dd = eval { Net::MRT::mrt_read_next($fh) } ) {
- if ( $dd->{'prefix'} && $dd->{'bits'} ) {
- next if $dd->{'subtype'} == 2 and !$v4;
- next if $dd->{'subtype'} == 4 and !$v6;
- my $entry = $dd->{'entries'}->[0];
- my $net = $dd->{'prefix'} . '/' . $dd->{'bits'};
- if ( $entry && $entry->{'AS_PATH'} ) {
- my $as = $entry->{'AS_PATH'}->[-1];
- if ( ref($as) eq "ARRAY" ) {
- $as = @{$as}[0];
- }
-
- next if ( is_bougus_asn($as) );
-
- if ( !$networks->{$as} ) {
- if ( $dd->{'subtype'} == 2 ) {
- $networks->{$as} = { nets_v4 => [$net], nets_v6 => [] };
- }
- else {
- $networks->{$as} = { nets_v6 => [$net], nets_v4 => [] };
- }
- }
- else {
-
- if ( $dd->{'subtype'} == 2 ) {
- push @{ $networks->{$as}->{'nets_v4'} }, $net;
- }
- else {
- push @{ $networks->{$as}->{'nets_v6'} }, $net;
- }
- }
- }
+ # strip bogus AS
+ while (is_bougus_asn($origin_as)) {
+ $origin_as = pop @as_path;
+ last if scalar @as_path == 0;
}
}
+
+ # empty AS_PATH or all AS_PATH elements was stripped as bogus - use
+ # PEER_AS as origin AS
+ $origin_as //= $e[F_PEER_AS];
+
+ $networks->{$ip_ver}{$prefix} = int($origin_as);
}
}
+# Remove default routes
+delete $networks->{4}{'0.0.0.0/0'};
+delete $networks->{6}{'::/0'};
+
# Now roughly detect countries
-foreach my $u ( @{ $config{'asn_sources'} } ) {
+my $as_info = {};
+
+# RIR statistics exchange format
+# https://www.apnic.net/publications/media-library/documents/resource-guidelines/rir-statistics-exchange-format
+# https://www.arin.net/knowledge/statistics/nro_extended_stats_format.pdf
+# first 7 fields for this two formats are same
+use constant {
+ F_REGISTRY => 0, # {afrinic,apnic,arin,iana,lacnic,ripencc}
+ F_CC => 1, # ISO 3166 2-letter contry code
+ F_TYPE => 2, # {asn,ipv4,ipv6}
+ F_START => 3,
+ F_VALUE => 4,
+ F_DATE => 5,
+ F_STATUS => 6,
+};
+
+foreach my $u (@{ $config{'asn_sources'} }) {
my $parsed = URI->new($u);
- my $fname = $download_target . '/' . basename( $parsed->path );
- open( my $fh, "<", $fname ) or die "Cannot open $fname: $!";
+ my $fname = $download_target . '/' . basename($parsed->path);
+ open(my $fh, "<", $fname) or die "Cannot open $fname: $!";
while (<$fh>) {
next if /^\#/;
chomp;
my @elts = split /\|/;
- if ( $elts[2] eq 'asn' && $elts[3] ne '*' ) {
- my $as_start = int( $elts[3] );
- my $as_end = $as_start + int( $elts[4] );
-
- for ( my $as = $as_start ; $as < $as_end ; $as++ ) {
- my $real_as = $as;
+ if ($elts[F_TYPE] eq 'asn' && $elts[F_START] ne '*') {
+ my $as_start = int($elts[F_START]);
+ my $as_end = $as_start + int($elts[F_VALUE]) - 1;
- if ( ref($as) eq "ARRAY" ) {
- $real_as = @{$as}[0];
- }
-
- if ( $networks->{"$real_as"} ) {
- $networks->{"$real_as"}->{'country'} = $elts[1];
- $networks->{"$real_as"}->{'rir'} = $elts[0];
- }
+ for my $as ($as_start .. $as_end) {
+ $as_info->{$as}{'country'} = $elts[F_CC];
+ $as_info->{$as}{'rir'} = $elts[F_REGISTRY];
}
}
}
}
-while ( my ( $k, $v ) = each( %{$networks} ) ) {
- if ($v4) {
- foreach my $n ( @{ $v->{'nets_v4'} } ) {
+# Write zone files
+my $ns_list = join ' ', @{$ns_servers};
+my $zone_header = << "EOH";
+\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300
+\$NS 43200 $ns_list
+EOH
- # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
- if ( $v->{'country'} ) {
- printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'};
- }
- else {
- printf $v4_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN';
- }
- }
+if ($v4) {
+ open my $v4_fh, '>', ".$v4_file.tmp";
+ print $v4_fh $zone_header;
+
+ while (my ($net, $asn) = each %{ $networks->{4} }) {
+ my $country = $as_info->{$asn}{'country'} || 'UN';
+ my $rir = $as_info->{$asn}{'rir'} || 'UN';
+
+ # "15169|8.8.8.0/24|US|arin|" for 8.8.8.8
+ printf $v4_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir;
}
- if ($v6) {
- foreach my $n ( @{ $v->{'nets_v6'} } ) {
- # "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
- if ( $v->{'country'} ) {
- printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, $v->{'country'}, $v->{'rir'};
- }
- else {
- printf $v6_fh "%s %s|%s|%s|%s|\n", $n, $k, $n, 'UN', 'UN';
- }
- }
+ close $v4_fh;
+ rename ".$v4_file.tmp", $v4_file;
+}
+
+if ($v6) {
+ open my $v6_fh, '>', ".$v6_file.tmp";
+ print $v6_fh $zone_header;
+
+ while (my ($net, $asn) = each %{ $networks->{6} }) {
+ my $country = $as_info->{$asn}{'country'} || 'UN';
+ my $rir = $as_info->{$asn}{'rir'} || 'UN';
+
+ # "2606:4700:4700::/48 13335|2606:4700:4700::/48|US|arin|" for 2606:4700:4700::1111
+ printf $v6_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir;
}
+
+ close $v6_fh;
+ rename ".$v6_file.tmp", $v6_file;
+}
+
+exit 0;
+
+########################################################################
+
+sub download_file {
+ my ($url) = @_;
+
+ local $File::Fetch::WARN = 0;
+ local $File::Fetch::TIMEOUT = 180; # connectivity to ftp.lacnic.net is bad
+
+ my $ff = File::Fetch->new(uri => $url);
+ my $where = $ff->fetch(to => $download_target) or
+ die "$url: ", $ff->error;
+
+ return $where;
+}
+
+# Returns true if AS number is bogus
+# e. g. a private AS.
+# List of allocated and reserved AS:
+# https://www.iana.org/assignments/as-numbers/as-numbers.txt
+sub is_bougus_asn {
+ my $as = shift;
+
+ # 64496-64511 Reserved for use in documentation and sample code
+ # 64512-65534 Designated for private use
+ # 65535 Reserved
+ # 65536-65551 Reserved for use in documentation and sample code
+ # 65552-131071 Reserved
+ return 1 if $as >= 64496 && $as <= 131071;
+
+ # Reserved (RFC6996, RFC7300, RFC7607)
+ return 1 if $as == 0 || $as >= 4200000000;
+
+ return 0;
}
__END__
@@ -324,8 +278,8 @@ asn.pl - download and parse ASN data for Rspamd
asn.pl [options]
Options:
- --download-asn Download ASN data from RIR
- --download-bgp Download GeoIP data from Maxmind
+ --download-asn Download ASN data from RIRs
+ --download-bgp Download BGP full view dump from RIPE RIS
--target Where to download files (default: current dir)
--zone-v4 IPv4 zone (default: asn.rspamd.com)
--zone-v6 IPv6 zone (default: asn6.rspamd.com)
@@ -365,3 +319,5 @@ Prints the manual page and exits.
B<asn.pl> is intended to download ASN data and GeoIP data and create a rbldnsd zone.
=cut
+
+# vim: et:ts=4:sw=4
\ No newline at end of file
More information about the Commits
mailing list