benvin
92213090fe
Replace the (unconfirmed) iplocate API expander with a database-backed one that reads the iplocate ip-to-asn CSV (network,asn,...) proxied through the artifactapi github remote. It downloads and indexes the whole DB once (ASN -> CIDRs), serves every asn address group from the in-memory index, and rebuilds on a 24h TTL; refresh failures keep the last-good index (fail-safe). No API key needed. - Add IPLocateDB expander (zip + CSV parsing, ASN normalization). - Wire it in main (TOMSWALLAPI_IPLOCATE_DB_URL overrides the default artifactapi URL); remove the dead API client. - Unit tests: CSV indexing (incl. quoted org fields), zip extraction, missing columns, and ASN normalization.
86 lines
2.2 KiB
Go
86 lines
2.2 KiB
Go
package asnexpand
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
const sampleCSV = `network,asn,country_code,name,org,domain
|
|
1.0.0.0/24,13335,US,CLOUDFLARENET,"Cloudflare, Inc.",cloudflare.com
|
|
1.1.1.0/24,13335,US,CLOUDFLARENET,"Cloudflare, Inc.",cloudflare.com
|
|
1.0.4.0/24,38803,AU,GTELECOM-AS-AP,Gtelecom Pty Ltd,gtelecom.com.au
|
|
104.16.0.0/13,13335,US,CLOUDFLARENET,"Cloudflare, Inc.",cloudflare.com
|
|
`
|
|
|
|
func TestBuildIndex(t *testing.T) {
|
|
idx, err := buildIndex(strings.NewReader(sampleCSV))
|
|
if err != nil {
|
|
t.Fatalf("buildIndex: %v", err)
|
|
}
|
|
cf := idx["13335"]
|
|
sort.Strings(cf)
|
|
want := []string{"1.0.0.0/24", "1.1.1.0/24", "104.16.0.0/13"}
|
|
sort.Strings(want)
|
|
if strings.Join(cf, ",") != strings.Join(want, ",") {
|
|
t.Errorf("AS13335 prefixes = %v, want %v", cf, want)
|
|
}
|
|
if got := idx["38803"]; len(got) != 1 || got[0] != "1.0.4.0/24" {
|
|
t.Errorf("AS38803 prefixes = %v", got)
|
|
}
|
|
// The quoted org field with an embedded comma must not shift columns.
|
|
if len(idx) != 2 {
|
|
t.Errorf("expected 2 distinct ASNs, got %d", len(idx))
|
|
}
|
|
}
|
|
|
|
func TestBuildIndexMissingColumns(t *testing.T) {
|
|
if _, err := buildIndex(strings.NewReader("foo,bar\n1,2\n")); err == nil {
|
|
t.Fatal("expected error when network/asn columns are absent")
|
|
}
|
|
}
|
|
|
|
func TestIndexFromZip(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
zw := zip.NewWriter(&buf)
|
|
w, _ := zw.Create("ip-to-asn-20260721.csv")
|
|
_, _ = w.Write([]byte(sampleCSV))
|
|
if err := zw.Close(); err != nil {
|
|
t.Fatalf("zip close: %v", err)
|
|
}
|
|
|
|
idx, err := indexFromZip(buf.Bytes())
|
|
if err != nil {
|
|
t.Fatalf("indexFromZip: %v", err)
|
|
}
|
|
if len(idx["13335"]) != 3 {
|
|
t.Errorf("expected 3 cloudflare prefixes from zip, got %d", len(idx["13335"]))
|
|
}
|
|
}
|
|
|
|
func TestIndexFromZipNoCSV(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
zw := zip.NewWriter(&buf)
|
|
w, _ := zw.Create("readme.txt")
|
|
_, _ = w.Write([]byte("no csv here"))
|
|
_ = zw.Close()
|
|
if _, err := indexFromZip(buf.Bytes()); err == nil {
|
|
t.Fatal("expected error when archive has no .csv")
|
|
}
|
|
}
|
|
|
|
func TestNormalizeASN(t *testing.T) {
|
|
for in, want := range map[string]string{
|
|
"13335": "13335",
|
|
"AS13335": "13335",
|
|
"as13335": "13335",
|
|
" 13335 ": "13335",
|
|
} {
|
|
if got := normalizeASN(in); got != want {
|
|
t.Errorf("normalizeASN(%q) = %q, want %q", in, got, want)
|
|
}
|
|
}
|
|
}
|