From 92213090fe3993eb42dd78b5e5ae6dd4e34de89b Mon Sep 17 00:00:00 2001 From: benvin Date: Tue, 21 Jul 2026 22:36:19 +1000 Subject: [PATCH] Expand ASN groups from the iplocate ip-to-asn database Replace the (unconfirmed) iplocate API expander with a database-backed one that reads the iplocate ip-to-asn CSV (network,asn,...) proxied through the artifactapi github remote. It downloads and indexes the whole DB once (ASN -> CIDRs), serves every asn address group from the in-memory index, and rebuilds on a 24h TTL; refresh failures keep the last-good index (fail-safe). No API key needed. - Add IPLocateDB expander (zip + CSV parsing, ASN normalization). - Wire it in main (TOMSWALLAPI_IPLOCATE_DB_URL overrides the default artifactapi URL); remove the dead API client. - Unit tests: CSV indexing (incl. quoted org fields), zip extraction, missing columns, and ASN normalization. --- cmd/tomswallapi/main.go | 19 ++- internal/asnexpand/iplocate.go | 94 -------------- internal/asnexpand/iplocate_db.go | 173 +++++++++++++++++++++++++ internal/asnexpand/iplocate_db_test.go | 85 ++++++++++++ internal/config/config.go | 7 +- 5 files changed, 272 insertions(+), 106 deletions(-) delete mode 100644 internal/asnexpand/iplocate.go create mode 100644 internal/asnexpand/iplocate_db.go create mode 100644 internal/asnexpand/iplocate_db_test.go diff --git a/cmd/tomswallapi/main.go b/cmd/tomswallapi/main.go index ba36218..7ec8a6a 100644 --- a/cmd/tomswallapi/main.go +++ b/cmd/tomswallapi/main.go @@ -54,18 +54,15 @@ func main() { slog.Warn("TOMSWALLAPI_AGENT_TOKEN is not set; agent config endpoint is disabled") } - // Start the central ASN expander when a key is configured. Without one, asn - // address groups simply stay unexpanded (their sets render empty and inert). - if cfg.IPLocateAPIKey != "" { - refresher := &asnexpand.Refresher{ - Store: store.New(db.Pool), - Expander: asnexpand.NewIPLocate(cfg.IPLocateAPIKey), - } - go refresher.Run(ctx) - slog.Info("started ASN expander") - } else { - slog.Warn("TOMSWALLAPI_IPLOCATE_API_KEY is not set; asn address groups will not be expanded") + // Start the central ASN expander, which reads the iplocate ip-to-asn database + // (proxied via artifactapi) and expands asn address groups into prefixes. + expander := asnexpand.NewIPLocateDB(cfg.IPLocateDBURL) + refresher := &asnexpand.Refresher{ + Store: store.New(db.Pool), + Expander: expander, } + go refresher.Run(ctx) + slog.Info("started ASN expander", "source", "iplocate-db", "url", expander.URL) srv := server.New(server.Options{ DB: db, diff --git a/internal/asnexpand/iplocate.go b/internal/asnexpand/iplocate.go deleted file mode 100644 index ac7b519..0000000 --- a/internal/asnexpand/iplocate.go +++ /dev/null @@ -1,94 +0,0 @@ -package asnexpand - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" -) - -// IPLocate expands ASNs via the iplocate.io IP-intelligence API. -// -// NOTE: iplocate's documented ASN data type is returned as part of an IP lookup -// (it carries the single `route` for that IP), so the exact ASN->prefixes -// endpoint depends on the account/plan. This client is deliberately tolerant of -// response shape and endpoint-configurable so it can be pointed at the correct -// endpoint once the API key and plan are confirmed. It expects a JSON body -// containing a list of prefixes under any of: "prefixes", "routes", "cidrs". -type IPLocate struct { - // BaseURL is the API root, default https://iplocate.io/api. The ASN path is - // BaseURL + "/asn/" + asn. - BaseURL string - // APIKey is sent as the ?apikey= query parameter (iplocate's scheme). - APIKey string - HTTP *http.Client -} - -// NewIPLocate builds a client with sane defaults. -func NewIPLocate(apiKey string) *IPLocate { - return &IPLocate{ - BaseURL: "https://iplocate.io/api", - APIKey: apiKey, - HTTP: &http.Client{Timeout: 15 * time.Second}, - } -} - -// asnResponse tolerates the several field names an ASN-prefix payload might use. -type asnResponse struct { - Prefixes []string `json:"prefixes"` - Routes []string `json:"routes"` - CIDRs []string `json:"cidrs"` -} - -func (r asnResponse) list() []string { - switch { - case len(r.Prefixes) > 0: - return r.Prefixes - case len(r.Routes) > 0: - return r.Routes - default: - return r.CIDRs - } -} - -// Prefixes implements Expander. -func (c *IPLocate) Prefixes(ctx context.Context, asn string) ([]string, error) { - if c.APIKey == "" { - return nil, fmt.Errorf("iplocate: no API key configured") - } - asn = strings.TrimPrefix(strings.ToUpper(strings.TrimSpace(asn)), "AS") - - u, err := url.Parse(fmt.Sprintf("%s/asn/AS%s", strings.TrimRight(c.BaseURL, "/"), asn)) - if err != nil { - return nil, err - } - q := u.Query() - q.Set("apikey", c.APIKey) - u.RawQuery = q.Encode() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - return nil, err - } - req.Header.Set("Accept", "application/json") - - resp, err := c.HTTP.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("iplocate: AS%s: status %d: %s", asn, resp.StatusCode, strings.TrimSpace(string(body))) - } - - var parsed asnResponse - if err := json.Unmarshal(body, &parsed); err != nil { - return nil, fmt.Errorf("iplocate: AS%s: decoding response: %w", asn, err) - } - return parsed.list(), nil -} diff --git a/internal/asnexpand/iplocate_db.go b/internal/asnexpand/iplocate_db.go new file mode 100644 index 0000000..6151e33 --- /dev/null +++ b/internal/asnexpand/iplocate_db.go @@ -0,0 +1,173 @@ +package asnexpand + +import ( + "archive/zip" + "bufio" + "bytes" + "context" + "encoding/csv" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" +) + +// DefaultIPLocateDBURL is the iplocate ip-to-asn CSV database, proxied through +// the artifactapi github remote. The files are Git-LFS, so the /raw/ path +// redirects to the LFS media host — artifactapi follows that redirect. +const DefaultIPLocateDBURL = "https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes/github/files/iplocate/ip-address-databases/raw/main/ip-to-asn/ip-to-asn.csv.zip" + +// IPLocateDB expands ASNs by reading iplocate's ip-to-asn CSV (CIDR,asn,...). It +// downloads and indexes the whole database once, then serves every ASN group +// from the in-memory index, rebuilding when the index goes stale. This is far +// cheaper than a per-ASN API call and needs no API key. +type IPLocateDB struct { + URL string + TTL time.Duration + HTTP *http.Client + + mu sync.Mutex + index map[string][]string + loadedAt time.Time +} + +// NewIPLocateDB builds a DB-backed expander. An empty url uses the default. +func NewIPLocateDB(url string) *IPLocateDB { + if url == "" { + url = DefaultIPLocateDBURL + } + return &IPLocateDB{ + URL: url, + TTL: 24 * time.Hour, + HTTP: &http.Client{Timeout: 5 * time.Minute}, + } +} + +// Prefixes implements Expander: it returns the CIDRs announced by the ASN. +func (d *IPLocateDB) Prefixes(ctx context.Context, asn string) ([]string, error) { + if err := d.ensureIndex(ctx); err != nil { + return nil, err + } + d.mu.Lock() + defer d.mu.Unlock() + return d.index[normalizeASN(asn)], nil +} + +// ensureIndex (re)builds the index when it is missing or stale. On a refresh +// failure it keeps the prior index (fail-safe) and only errors when there is +// nothing cached yet. +func (d *IPLocateDB) ensureIndex(ctx context.Context) error { + d.mu.Lock() + fresh := d.index != nil && time.Since(d.loadedAt) < d.TTL + hadIndex := d.index != nil + d.mu.Unlock() + if fresh { + return nil + } + + idx, err := d.download(ctx) + if err != nil { + if hadIndex { + return nil + } + return err + } + d.mu.Lock() + d.index = idx + d.loadedAt = time.Now() + d.mu.Unlock() + return nil +} + +func (d *IPLocateDB) download(ctx context.Context) (map[string][]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, d.URL, nil) + if err != nil { + return nil, err + } + resp, err := d.HTTP.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("iplocate db: status %d", resp.StatusCode) + } + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return indexFromZip(data) +} + +// indexFromZip finds the single .csv in the archive and indexes it. +func indexFromZip(data []byte) (map[string][]string, error) { + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, fmt.Errorf("iplocate db: opening zip: %w", err) + } + for _, f := range zr.File { + if !strings.HasSuffix(f.Name, ".csv") { + continue + } + rc, err := f.Open() + if err != nil { + return nil, err + } + defer rc.Close() + return buildIndex(rc) + } + return nil, fmt.Errorf("iplocate db: no .csv entry in archive") +} + +// buildIndex reads the ip-to-asn CSV (header: network,asn,country_code,name,org, +// domain) and returns a map of ASN -> announced CIDRs. +func buildIndex(r io.Reader) (map[string][]string, error) { + cr := csv.NewReader(bufio.NewReaderSize(r, 1<<20)) + cr.ReuseRecord = true + cr.FieldsPerRecord = -1 // tolerate quoted commas (org names) + + header, err := cr.Read() + if err != nil { + return nil, fmt.Errorf("iplocate db: reading header: %w", err) + } + netIdx := columnIndex(header, "network") + asnIdx := columnIndex(header, "asn") + if netIdx < 0 || asnIdx < 0 { + return nil, fmt.Errorf("iplocate db: header missing network/asn columns: %v", header) + } + + index := map[string][]string{} + for { + rec, err := cr.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("iplocate db: reading row: %w", err) + } + if netIdx >= len(rec) || asnIdx >= len(rec) { + continue + } + // Records are reused across Read calls, so clone the retained fields. + asn := strings.Clone(rec[asnIdx]) + index[asn] = append(index[asn], strings.Clone(rec[netIdx])) + } + return index, nil +} + +func columnIndex(header []string, name string) int { + for i, h := range header { + if strings.EqualFold(strings.TrimSpace(h), name) { + return i + } + } + return -1 +} + +// normalizeASN strips an optional AS prefix so "AS13335" and "13335" both match +// the bare numbers in the database. +func normalizeASN(asn string) string { + return strings.TrimPrefix(strings.ToUpper(strings.TrimSpace(asn)), "AS") +} diff --git a/internal/asnexpand/iplocate_db_test.go b/internal/asnexpand/iplocate_db_test.go new file mode 100644 index 0000000..ab732c1 --- /dev/null +++ b/internal/asnexpand/iplocate_db_test.go @@ -0,0 +1,85 @@ +package asnexpand + +import ( + "archive/zip" + "bytes" + "sort" + "strings" + "testing" +) + +const sampleCSV = `network,asn,country_code,name,org,domain +1.0.0.0/24,13335,US,CLOUDFLARENET,"Cloudflare, Inc.",cloudflare.com +1.1.1.0/24,13335,US,CLOUDFLARENET,"Cloudflare, Inc.",cloudflare.com +1.0.4.0/24,38803,AU,GTELECOM-AS-AP,Gtelecom Pty Ltd,gtelecom.com.au +104.16.0.0/13,13335,US,CLOUDFLARENET,"Cloudflare, Inc.",cloudflare.com +` + +func TestBuildIndex(t *testing.T) { + idx, err := buildIndex(strings.NewReader(sampleCSV)) + if err != nil { + t.Fatalf("buildIndex: %v", err) + } + cf := idx["13335"] + sort.Strings(cf) + want := []string{"1.0.0.0/24", "1.1.1.0/24", "104.16.0.0/13"} + sort.Strings(want) + if strings.Join(cf, ",") != strings.Join(want, ",") { + t.Errorf("AS13335 prefixes = %v, want %v", cf, want) + } + if got := idx["38803"]; len(got) != 1 || got[0] != "1.0.4.0/24" { + t.Errorf("AS38803 prefixes = %v", got) + } + // The quoted org field with an embedded comma must not shift columns. + if len(idx) != 2 { + t.Errorf("expected 2 distinct ASNs, got %d", len(idx)) + } +} + +func TestBuildIndexMissingColumns(t *testing.T) { + if _, err := buildIndex(strings.NewReader("foo,bar\n1,2\n")); err == nil { + t.Fatal("expected error when network/asn columns are absent") + } +} + +func TestIndexFromZip(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create("ip-to-asn-20260721.csv") + _, _ = w.Write([]byte(sampleCSV)) + if err := zw.Close(); err != nil { + t.Fatalf("zip close: %v", err) + } + + idx, err := indexFromZip(buf.Bytes()) + if err != nil { + t.Fatalf("indexFromZip: %v", err) + } + if len(idx["13335"]) != 3 { + t.Errorf("expected 3 cloudflare prefixes from zip, got %d", len(idx["13335"])) + } +} + +func TestIndexFromZipNoCSV(t *testing.T) { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, _ := zw.Create("readme.txt") + _, _ = w.Write([]byte("no csv here")) + _ = zw.Close() + if _, err := indexFromZip(buf.Bytes()); err == nil { + t.Fatal("expected error when archive has no .csv") + } +} + +func TestNormalizeASN(t *testing.T) { + for in, want := range map[string]string{ + "13335": "13335", + "AS13335": "13335", + "as13335": "13335", + " 13335 ": "13335", + } { + if got := normalizeASN(in); got != want { + t.Errorf("normalizeASN(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/config/config.go b/internal/config/config.go index d5140ec..70c5fa8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -23,8 +23,12 @@ type Config struct { // AgentToken guards the per-device config endpoint (used by tomswall agents). AgentToken string - // IPLocateAPIKey is used to expand ASN address groups into prefixes. + // IPLocateAPIKey is used to expand ASN address groups into prefixes via the + // iplocate API (legacy path). The DB expander below is preferred. IPLocateAPIKey string + // IPLocateDBURL points at the iplocate ip-to-asn CSV database (proxied via + // artifactapi). Empty uses the built-in default. + IPLocateDBURL string } // Load reads configuration from the environment, applying defaults. @@ -40,6 +44,7 @@ func Load() (*Config, error) { WriteToken: os.Getenv("TOMSWALLAPI_WRITE_TOKEN"), AgentToken: os.Getenv("TOMSWALLAPI_AGENT_TOKEN"), IPLocateAPIKey: os.Getenv("TOMSWALLAPI_IPLOCATE_API_KEY"), + IPLocateDBURL: os.Getenv("TOMSWALLAPI_IPLOCATE_DB_URL"), } if c.DBName == "" { return nil, fmt.Errorf("TOMSWALLAPI_DB_NAME must not be empty")