提交: 7f7bbc7162727ce236b32d4b607f70445a4dd397 - addr2country (git)

Commit MetaInfo

• 修訂: 7f7bbc7162727ce236b32d4b607f70445a4dd397 (tree)
• 時間: 2022-04-10 02:19:25
• 作者: dyknon <dyknon@user...>
• Commiter: dyknon

Log Message

works

Change Summary

• modified: ip2cc_index.pl

差異

--- a/ip2cc_index.pl
+++ b/ip2cc_index.pl
@@ -2,11 +2,11 @@
 
 use warnings;
 use strict;
-use feature qw(say state postderef);
+use feature qw(say state postderef bitwise);
 
 use Getopt::Long;
 use Pod::Usage;
-use List::Util qw(reduce);
+use List::Util qw(reduce sum0);
 use JSON;
 use LWP::UserAgent ();
 use HTTP::Date;
@@ -22,7 +22,7 @@ my $asn;
 my $ipv4 = 1;
 my $ipv6 = 1;
 my $prefix = "";
-my @sort = ("start", "region");
+my $cidr;
 my $help;
 
 GetOptions(
@@ -34,17 +34,12 @@ GetOptions(
     "orig-rotate=i" => \$orig_rotate,
     "orig-prefix=s" => \$orig_prefix,
     "prefix=s" => \$prefix,
-    "sort=s" => \@sort,
+    "cidr!" => \$cidr,
     "help|h|?" => \$help,
 ) or pod2usage(16);
 pod2usage(0) if($help);
 pod2usage(17) if(@ARGV > 1);
 my $op = $ARGV[0] // "generate";
-my @ill_sort = grep{$_ ne "start" && $_ ne "region"}@sort;
-if(@ill_sort){
-    say STDERR "Unknown sort key: ", join(", ", @ill_sort), "\n";
-    pod2usage(18);
-}
 
 sub mydie{
     my ($exit, @msg) = @_;
@@ -158,19 +153,21 @@ my @bitmap1 = (
     "\x{0f}", "\x{1f}", "\x{3f}", "\x{7f}");
 sub leftfill{ "\x{0}" x ($_[1]-length($_[0])) . $_[0] }
 sub cntr0{
-    my $c;
+    my $c = 0;
     for my $v(reverse split(//, $_[0])){
         if($v eq "\x{00}"){
             $c += 8;
             next;
         }
         for(reverse 1 .. 7){
-            if(($v & $bitmap1[$_]) eq "\x{00}"){
+            if(($v &. $bitmap1[$_]) eq "\x{00}"){
                 $c += $_;
+                last;
             }
         }
-        return $c;
+        last;
     }
+    $c;
 }
 sub num2bin{ pack("L>", $_[0]) =~ s/^\x{0}+//r }
 sub bin2num{ reduce{$a << 8 | $b}unpack("C*", $_[0]) }
@@ -187,6 +184,17 @@ sub binadd{
     push @ret, $carry if($carry);
     join("", map{chr}@ret);
 }
+sub bindec{
+    my @v = map{ord}split(//, $_[0]);
+    my @r;
+    while($v[$#v] == 0){
+        pop @v;
+        push @r, "\x{ff}";
+        die if(!@v);
+    }
+    $v[$#v] -= 1;
+    join("", map{chr}(@v, @r));
+}
 my %ascii2addr = (
     asn => sub{pack("L>", $_[0])},
     ipv4 => sub{pack("C*", split(/\./, $_[0]))},
@@ -219,8 +227,205 @@ my %ascii2len = (
         leftfill($bitmap0[$i%8] . "\x{00}" x int($i/8), 16);
     },
 );
+my %len2ascii = (
+    asn  => sub{unpack("L>", $_[0])},
+    ipv4 => sub{unpack("L>", $_[0])},
+    ipv6 => sub{
+        for(1 .. 128){
+            return $_ if($_[0] eq $ascii2len{ipv6}->($_));
+        }
+        "*";
+    },
+);
+my %asciilen = (
+    asn => {
+        addr    => 10,
+        length  => 10,
+        cidrnm  => 2,
+    },
+    ipv4 => {
+        addr    => 15,
+        length  => 10,
+        cidrnm  => 2,
+    },
+    ipv6 => {
+        addr    => 39,
+        length  => 3,
+        cidrnm  => 3,
+    },
+);
+sub sort_by{
+    my ($arr, @k) = @_;
+    sort{reduce{$a || $b}map{$a->{$_} cmp $b->{$_}}@k}@$arr;
+}
+sub uniq_check{
+    my ($arr) = @_;
+    my @ret;
+    my $end = "\x{0}" x length $arr->[0]{start};
+    for($arr->@*){
+        if($end gt $_->{start}){
+            push @ret, $_;
+        }
+        my $new_end = binadd($_->{start}, $_->{length});
+        $end = $new_end if($new_end gt $end);
+    }
+    @ret;
+}
+sub merge_by{
+    my ($arr, @k) = @_;
+    my @ret;
+    my $end;
+    my %k;
+    for my $r(@$arr){
+        if(
+            defined $end
+            && $end eq $r->{start}
+            && reduce{$a && $b}map{$k{$_} eq $r->{$_}}@k
+        ){
+            $ret[$#ret] = {$ret[$#ret]->%*};
+            delete $ret[$#ret]{cidr};
+            $ret[$#ret]{length} = binadd($ret[$#ret]{length}, $r->{length});
+        }else{
+            $k{$_} = $r->{$_} for(@k);
+            push @ret, $r;
+        }
+        $end = binadd($r->{start}, $r->{length});
+    }
+    @ret;
+}
+sub get_ascii_formatter{
+    my ($type, $cidr, @vals) = @_;
+    my $addr2ascii = $addr2ascii{$type};
+    my $len2ascii = $len2ascii{$type};
+    @vals = map{$_->{sub}}@vals;
+    if($cidr){
+        sub{
+            my ($r) = @_;
+            map{
+                join("\t",
+                    (map{$_->($r)}@vals),
+                    $addr2ascii->($_->[0]) . "/" . $_->[1]
+                )
+            }$r->{cidr}->@*;
+        };
+    }else{
+        sub{
+            my ($r) = @_;
+            join("\t",
+                (map{$_->($r)}@vals),
+                $addr2ascii->($r->{start}),
+                $addr2ascii->(binadd($r->{start}, bindec($r->{length}))),
+            );
+        };
+    }
+}
+sub get_pad_ascii_formatter{
+    my ($type, $cidr, @vals) = @_;
+    my %len_part = $asciilen{$type}->%*;
+    my $len = @vals
+        + (sum0 map{$_->{len}}@vals)
+        + $len_part{addr}
+        + 1;
+    if($cidr){
+        $len += $len_part{cidrnm}+1;
+    }else{
+        $len += $len_part{addr}+1;
+    }
+    my $inner = get_ascii_formatter($type, $cidr, @vals);
+    sub{
+        my $line = $inner->(@_);
+        my $padn = $len - length $line;
+        die if($padn < 0);
+        "$line\t" . " "x$padn;
+    };
+}
+sub get_binary_formatter{
+    my ($type, $cidr, @vals) = @_;
+    @vals = map{$_->{sub}}@vals;
+    if($cidr){
+        sub{
+            my ($r) = @_;
+            map{
+                join("",
+                    (map{$_->($r)}@vals),
+                    $_->[0],
+                    chr($_->[1]),
+                )
+            }$r->{cidr}->@*;
+        };
+    }else{
+        sub{
+            my ($r) = @_;
+            join("",
+                (map{$_->($r)}@vals),
+                $r->{start},
+                binadd($r->{start}, bindec($r->{length})),
+            );
+        };
+    }
+}
+sub get_lf_appended{
+    my ($inner) = @_;
+    sub{ map{"$_\n"}$inner->(@_) };
+}
+my %region_formatter = (
+    sub => sub{$_[0]->{region}},
+    len => 2,
+);
+my %formatters = (
+    "cidr.txt" => sub{
+        get_lf_appended(get_ascii_formatter($_[0], 1, \%region_formatter));
+    },
+    "txt" => sub{
+        get_lf_appended(get_ascii_formatter($_[0], 0, \%region_formatter));
+    },
+    "cidr.pad.txt" => sub{
+        get_lf_appended(get_pad_ascii_formatter($_[0], 1, \%region_formatter));
+    },
+    "pad.txt" => sub{
+        get_lf_appended(get_pad_ascii_formatter($_[0], 0, \%region_formatter));
+    },
+    "cidr.bin" => sub{
+        get_binary_formatter($_[0], 1, \%region_formatter);
+    },
+    "bin" => sub{
+        get_binary_formatter($_[0], 0, \%region_formatter);
+    },
+);
+sub write_it{
+    my ($ranges, $file, $formatter) = @_;
+    open(my $wh, ">:raw", $file) or mydie(4, $!);
+    for(@$ranges){
+        $wh->print($formatter->($_));
+    }
+    $wh->close or mydie(4, $!);
+}
+sub gen_cidr_inplace{
+    my ($arr) = @_;
+    my $al = length $arr->[0]{start};
+    for my $r(@$arr){
+        next if($r->{cidr});
+        $r->{cidr} = [];
+        my $p = $r->{start};
+        my $end = binadd($p, $r->{length});
+        while($p ne $end){
+            for(reverse 0 .. cntr0($p)){
+                my $step = $bitmap0[$_%8] . "\x{00}" x int($_/8);
+                my $next = binadd($p, $step);
+                next if(length $next > $al);
+                next if($next gt $end);
+                push $r->{cidr}->@*, [$p, $al*8-$_];
+                $p = $next;
+                last;
+            }
+        }
+    }
+}
 sub do_load_files{
-    my %ret = (ipv4 => [], ipv6 => [], asn => []);
+    my %ret;
+    $ret{asn} = [] if($asn);
+    $ret{ipv4} = [] if($ipv4);
+    $ret{ipv6} = [] if($ipv6);
     for my $src(src_spec()->@*){
         my $fname = orig_name($src->{name});
         if(!-e $fname){
@@ -255,23 +460,6 @@ sub do_load_files{
     }
     %ret;
 }
-sub sort_by_keys{
-    my ($arr, @k) = @_;
-    sort{reduce{$a || $b}map{$a->{$_} cmp $b->{$_}}@k}@$arr;
-}
-sub uniq_check{
-    my ($arr) = @_;
-    my @ret;
-    my $end = "\x{0}" x length $arr->[0]{start};
-    for($arr->@*){
-        if($end gt $_->{start}){
-            push @ret, $_;
-        }
-        my $new_end = binadd($_->{start}, $_->{length});
-        $end = $new_end if($new_end gt $end);
-    }
-    @ret;
-}
 
 if($op eq "generate"){
     if($download){
@@ -283,11 +471,22 @@ if($op eq "generate"){
     }
     my %reglist = do_load_files();
     for my $k(keys %reglist){
-        #$reglist{$k}->@* = grep{$_->{region} ne "ZZ"}$reglist{$k}->@*;
-        $reglist{$k}->@* = sort_by_keys($reglist{$k}, "start");
+        $reglist{$k}->@* = grep{$_->{region} ne "ZZ"}$reglist{$k}->@*;
+        $reglist{$k}->@* = sort_by($reglist{$k}, qw(start));
         for(uniq_check($reglist{$k})){
             say "Overwrapping $k: ", $addr2ascii{$k}->($_->{start});
         }
+
+        gen_cidr_inplace($reglist{$k});
+        my @merged = merge_by($reglist{$k}, qw(region));
+        gen_cidr_inplace(\@merged);
+        my @regionsort = sort_by(\@merged, qw(region start));
+
+        write_it(
+            \@regionsort,
+            "$k-merged-region.cidr.txt",
+            $formatters{"cidr.txt"}->($k)
+        );
     }
 }elsif($op eq "head-all"){
     if(!defined $src_spec){
@@ -324,12 +523,12 @@ ip2cc_indexgen.pl - Generate index file to lookup country code from ip address
 =item ip2cc_indexgen.pl
 
     [--help] [--asn] [--no-ipv4] [--no-ipv6]
-    [--src-spec=F<src_spec.json>]
-    [--orig-rotate=I<number>]
-    [--orig-prefix=I<prefix>]
-    [--prefix=I<output>]
-    [--sort=B<sort key>,...]
-    [I<OPERATION>]
+    [--src-spec=src_spec.json]
+    [--orig-rotate=number]
+    [--orig-prefix=prefix]
+    [--prefix=output]
+    [--cidr]
+    [OPERATION]
 
 =back