Expand ASN groups from the iplocate ip-to-asn database

Replace the (unconfirmed) iplocate API expander with a database-backed one that
reads the iplocate ip-to-asn CSV (network,asn,...) proxied through the artifactapi
github remote. It downloads and indexes the whole DB once (ASN -> CIDRs), serves
every asn address group from the in-memory index, and rebuilds on a 24h TTL;
refresh failures keep the last-good index (fail-safe). No API key needed.

- Add IPLocateDB expander (zip + CSV parsing, ASN normalization).
- Wire it in main (TOMSWALLAPI_IPLOCATE_DB_URL overrides the default artifactapi
  URL); remove the dead API client.
- Unit tests: CSV indexing (incl. quoted org fields), zip extraction, missing
  columns, and ASN normalization.

cmd/tomswallapi/main.go

@@ -54,18 +54,15 @@ func main() {
 		slog.Warn("TOMSWALLAPI_AGENT_TOKEN is not set; agent config endpoint is disabled")
 	}
 
-	// Start the central ASN expander when a key is configured. Without one, asn
-	// address groups simply stay unexpanded (their sets render empty and inert).
-	if cfg.IPLocateAPIKey != "" {
+	// Start the central ASN expander, which reads the iplocate ip-to-asn database
+	// (proxied via artifactapi) and expands asn address groups into prefixes.
+	expander := asnexpand.NewIPLocateDB(cfg.IPLocateDBURL)
 	refresher := &asnexpand.Refresher{
 		Store:    store.New(db.Pool),
-			Expander: asnexpand.NewIPLocate(cfg.IPLocateAPIKey),
+		Expander: expander,
 	}
 	go refresher.Run(ctx)
-		slog.Info("started ASN expander")
-	} else {
-		slog.Warn("TOMSWALLAPI_IPLOCATE_API_KEY is not set; asn address groups will not be expanded")
-	}
+	slog.Info("started ASN expander", "source", "iplocate-db", "url", expander.URL)
 
 	srv := server.New(server.Options{
 		DB:         db,

internal/asnexpand/iplocate.go

@@ -1,94 +0,0 @@
-package asnexpand
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"strings"
-	"time"
-)
-
-// IPLocate expands ASNs via the iplocate.io IP-intelligence API.
-//
-// NOTE: iplocate's documented ASN data type is returned as part of an IP lookup
-// (it carries the single `route` for that IP), so the exact ASN->prefixes
-// endpoint depends on the account/plan. This client is deliberately tolerant of
-// response shape and endpoint-configurable so it can be pointed at the correct
-// endpoint once the API key and plan are confirmed. It expects a JSON body
-// containing a list of prefixes under any of: "prefixes", "routes", "cidrs".
-type IPLocate struct {
-	// BaseURL is the API root, default https://iplocate.io/api. The ASN path is
-	// BaseURL + "/asn/" + asn.
-	BaseURL string
-	// APIKey is sent as the ?apikey= query parameter (iplocate's scheme).
-	APIKey string
-	HTTP   *http.Client
-}
-
-// NewIPLocate builds a client with sane defaults.
-func NewIPLocate(apiKey string) *IPLocate {
-	return &IPLocate{
-		BaseURL: "https://iplocate.io/api",
-		APIKey:  apiKey,
-		HTTP:    &http.Client{Timeout: 15 * time.Second},
-	}
-}
-
-// asnResponse tolerates the several field names an ASN-prefix payload might use.
-type asnResponse struct {
-	Prefixes []string `json:"prefixes"`
-	Routes   []string `json:"routes"`
-	CIDRs    []string `json:"cidrs"`
-}
-
-func (r asnResponse) list() []string {
-	switch {
-	case len(r.Prefixes) > 0:
-		return r.Prefixes
-	case len(r.Routes) > 0:
-		return r.Routes
-	default:
-		return r.CIDRs
-	}
-}
-
-// Prefixes implements Expander.
-func (c *IPLocate) Prefixes(ctx context.Context, asn string) ([]string, error) {
-	if c.APIKey == "" {
-		return nil, fmt.Errorf("iplocate: no API key configured")
-	}
-	asn = strings.TrimPrefix(strings.ToUpper(strings.TrimSpace(asn)), "AS")
-
-	u, err := url.Parse(fmt.Sprintf("%s/asn/AS%s", strings.TrimRight(c.BaseURL, "/"), asn))
-	if err != nil {
-		return nil, err
-	}
-	q := u.Query()
-	q.Set("apikey", c.APIKey)
-	u.RawQuery = q.Encode()
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("Accept", "application/json")
-
-	resp, err := c.HTTP.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("iplocate: AS%s: status %d: %s", asn, resp.StatusCode, strings.TrimSpace(string(body)))
-	}
-
-	var parsed asnResponse
-	if err := json.Unmarshal(body, &parsed); err != nil {
-		return nil, fmt.Errorf("iplocate: AS%s: decoding response: %w", asn, err)
-	}
-	return parsed.list(), nil
-}

internal/asnexpand/iplocate_db.go

@@ -0,0 +1,173 @@
+package asnexpand
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/csv"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+)
+
+// DefaultIPLocateDBURL is the iplocate ip-to-asn CSV database, proxied through
+// the artifactapi github remote. The files are Git-LFS, so the /raw/ path
+// redirects to the LFS media host — artifactapi follows that redirect.
+const DefaultIPLocateDBURL = "https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes/github/files/iplocate/ip-address-databases/raw/main/ip-to-asn/ip-to-asn.csv.zip"
+
+// IPLocateDB expands ASNs by reading iplocate's ip-to-asn CSV (CIDR,asn,...). It
+// downloads and indexes the whole database once, then serves every ASN group
+// from the in-memory index, rebuilding when the index goes stale. This is far
+// cheaper than a per-ASN API call and needs no API key.
+type IPLocateDB struct {
+	URL  string
+	TTL  time.Duration
+	HTTP *http.Client
+
+	mu       sync.Mutex
+	index    map[string][]string
+	loadedAt time.Time
+}
+
+// NewIPLocateDB builds a DB-backed expander. An empty url uses the default.
+func NewIPLocateDB(url string) *IPLocateDB {
+	if url == "" {
+		url = DefaultIPLocateDBURL
+	}
+	return &IPLocateDB{
+		URL:  url,
+		TTL:  24 * time.Hour,
+		HTTP: &http.Client{Timeout: 5 * time.Minute},
+	}
+}
+
+// Prefixes implements Expander: it returns the CIDRs announced by the ASN.
+func (d *IPLocateDB) Prefixes(ctx context.Context, asn string) ([]string, error) {
+	if err := d.ensureIndex(ctx); err != nil {
+		return nil, err
+	}
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.index[normalizeASN(asn)], nil
+}
+
+// ensureIndex (re)builds the index when it is missing or stale. On a refresh
+// failure it keeps the prior index (fail-safe) and only errors when there is
+// nothing cached yet.
+func (d *IPLocateDB) ensureIndex(ctx context.Context) error {
+	d.mu.Lock()
+	fresh := d.index != nil && time.Since(d.loadedAt) < d.TTL
+	hadIndex := d.index != nil
+	d.mu.Unlock()
+	if fresh {
+		return nil
+	}
+
+	idx, err := d.download(ctx)
+	if err != nil {
+		if hadIndex {
+			return nil
+		}
+		return err
+	}
+	d.mu.Lock()
+	d.index = idx
+	d.loadedAt = time.Now()
+	d.mu.Unlock()
+	return nil
+}
+
+func (d *IPLocateDB) download(ctx context.Context) (map[string][]string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, d.URL, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := d.HTTP.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("iplocate db: status %d", resp.StatusCode)
+	}
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	return indexFromZip(data)
+}
+
+// indexFromZip finds the single .csv in the archive and indexes it.
+func indexFromZip(data []byte) (map[string][]string, error) {
+	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		return nil, fmt.Errorf("iplocate db: opening zip: %w", err)
+	}
+	for _, f := range zr.File {
+		if !strings.HasSuffix(f.Name, ".csv") {
+			continue
+		}
+		rc, err := f.Open()
+		if err != nil {
+			return nil, err
+		}
+		defer rc.Close()
+		return buildIndex(rc)
+	}
+	return nil, fmt.Errorf("iplocate db: no .csv entry in archive")
+}
+
+// buildIndex reads the ip-to-asn CSV (header: network,asn,country_code,name,org,
+// domain) and returns a map of ASN -> announced CIDRs.
+func buildIndex(r io.Reader) (map[string][]string, error) {
+	cr := csv.NewReader(bufio.NewReaderSize(r, 1<<20))
+	cr.ReuseRecord = true
+	cr.FieldsPerRecord = -1 // tolerate quoted commas (org names)
+
+	header, err := cr.Read()
+	if err != nil {
+		return nil, fmt.Errorf("iplocate db: reading header: %w", err)
+	}
+	netIdx := columnIndex(header, "network")
+	asnIdx := columnIndex(header, "asn")
+	if netIdx < 0 || asnIdx < 0 {
+		return nil, fmt.Errorf("iplocate db: header missing network/asn columns: %v", header)
+	}
+
+	index := map[string][]string{}
+	for {
+		rec, err := cr.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("iplocate db: reading row: %w", err)
+		}
+		if netIdx >= len(rec) || asnIdx >= len(rec) {
+			continue
+		}
+		// Records are reused across Read calls, so clone the retained fields.
+		asn := strings.Clone(rec[asnIdx])
+		index[asn] = append(index[asn], strings.Clone(rec[netIdx]))
+	}
+	return index, nil
+}
+
+func columnIndex(header []string, name string) int {
+	for i, h := range header {
+		if strings.EqualFold(strings.TrimSpace(h), name) {
+			return i
+		}
+	}
+	return -1
+}
+
+// normalizeASN strips an optional AS prefix so "AS13335" and "13335" both match
+// the bare numbers in the database.
+func normalizeASN(asn string) string {
+	return strings.TrimPrefix(strings.ToUpper(strings.TrimSpace(asn)), "AS")
+}

internal/asnexpand/iplocate_db_test.go

@@ -0,0 +1,85 @@
+package asnexpand
+
+import (
+	"archive/zip"
+	"bytes"
+	"sort"
+	"strings"
+	"testing"
+)
+
+const sampleCSV = `network,asn,country_code,name,org,domain
+1.0.0.0/24,13335,US,CLOUDFLARENET,"Cloudflare, Inc.",cloudflare.com
+1.1.1.0/24,13335,US,CLOUDFLARENET,"Cloudflare, Inc.",cloudflare.com
+1.0.4.0/24,38803,AU,GTELECOM-AS-AP,Gtelecom Pty Ltd,gtelecom.com.au
+104.16.0.0/13,13335,US,CLOUDFLARENET,"Cloudflare, Inc.",cloudflare.com
+`
+
+func TestBuildIndex(t *testing.T) {
+	idx, err := buildIndex(strings.NewReader(sampleCSV))
+	if err != nil {
+		t.Fatalf("buildIndex: %v", err)
+	}
+	cf := idx["13335"]
+	sort.Strings(cf)
+	want := []string{"1.0.0.0/24", "1.1.1.0/24", "104.16.0.0/13"}
+	sort.Strings(want)
+	if strings.Join(cf, ",") != strings.Join(want, ",") {
+		t.Errorf("AS13335 prefixes = %v, want %v", cf, want)
+	}
+	if got := idx["38803"]; len(got) != 1 || got[0] != "1.0.4.0/24" {
+		t.Errorf("AS38803 prefixes = %v", got)
+	}
+	// The quoted org field with an embedded comma must not shift columns.
+	if len(idx) != 2 {
+		t.Errorf("expected 2 distinct ASNs, got %d", len(idx))
+	}
+}
+
+func TestBuildIndexMissingColumns(t *testing.T) {
+	if _, err := buildIndex(strings.NewReader("foo,bar\n1,2\n")); err == nil {
+		t.Fatal("expected error when network/asn columns are absent")
+	}
+}
+
+func TestIndexFromZip(t *testing.T) {
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, _ := zw.Create("ip-to-asn-20260721.csv")
+	_, _ = w.Write([]byte(sampleCSV))
+	if err := zw.Close(); err != nil {
+		t.Fatalf("zip close: %v", err)
+	}
+
+	idx, err := indexFromZip(buf.Bytes())
+	if err != nil {
+		t.Fatalf("indexFromZip: %v", err)
+	}
+	if len(idx["13335"]) != 3 {
+		t.Errorf("expected 3 cloudflare prefixes from zip, got %d", len(idx["13335"]))
+	}
+}
+
+func TestIndexFromZipNoCSV(t *testing.T) {
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, _ := zw.Create("readme.txt")
+	_, _ = w.Write([]byte("no csv here"))
+	_ = zw.Close()
+	if _, err := indexFromZip(buf.Bytes()); err == nil {
+		t.Fatal("expected error when archive has no .csv")
+	}
+}
+
+func TestNormalizeASN(t *testing.T) {
+	for in, want := range map[string]string{
+		"13335":   "13335",
+		"AS13335": "13335",
+		"as13335": "13335",
+		" 13335 ": "13335",
+	} {
+		if got := normalizeASN(in); got != want {
+			t.Errorf("normalizeASN(%q) = %q, want %q", in, got, want)
+		}
+	}
+}

internal/config/config.go

@@ -23,8 +23,12 @@ type Config struct {
 	// AgentToken guards the per-device config endpoint (used by tomswall agents).
 	AgentToken string
 
-	// IPLocateAPIKey is used to expand ASN address groups into prefixes.
+	// IPLocateAPIKey is used to expand ASN address groups into prefixes via the
+	// iplocate API (legacy path). The DB expander below is preferred.
 	IPLocateAPIKey string
+	// IPLocateDBURL points at the iplocate ip-to-asn CSV database (proxied via
+	// artifactapi). Empty uses the built-in default.
+	IPLocateDBURL string
 }
 
 // Load reads configuration from the environment, applying defaults.
@@ -40,6 +44,7 @@ func Load() (*Config, error) {
 		WriteToken:     os.Getenv("TOMSWALLAPI_WRITE_TOKEN"),
 		AgentToken:     os.Getenv("TOMSWALLAPI_AGENT_TOKEN"),
 		IPLocateAPIKey: os.Getenv("TOMSWALLAPI_IPLOCATE_API_KEY"),
+		IPLocateDBURL:  os.Getenv("TOMSWALLAPI_IPLOCATE_DB_URL"),
 	}
 	if c.DBName == "" {
 		return nil, fmt.Errorf("TOMSWALLAPI_DB_NAME must not be empty")