From 5e5524ad3df50fe29f0f095aa061f20906453979 Mon Sep 17 00:00:00 2001 From: Rustam <16064414+rusq@users.noreply.github.com> Date: Thu, 19 Mar 2026 18:25:32 +1000 Subject: [PATCH] fix 406 error and migrate to new Auckland API address --- .gitignore | 1 + AGENTS.md | 214 +++++++++++++++++++++++++++++++++++++++++ addr.go | 9 +- aklapi.go | 30 ++++++ cmd/aklapi/handlers.go | 12 ++- rubbish.go | 4 +- rubbish_test.go | 4 +- 7 files changed, 262 insertions(+), 12 deletions(-) create mode 100644 AGENTS.md diff --git a/.gitignore b/.gitignore index 6ab1466..f5e7e75 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ aklrubbish # binary files aklapi +.env diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..00258d3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,214 @@ +# AGENTS.md — Coding Agent Instructions for `aklapi` + +This document provides guidance for agentic coding assistants operating in this repository. + +--- + +## Project Overview + +`aklapi` is a Go library and HTTP server that exposes Auckland Council APIs +(rubbish collection schedules, property address lookup) as a simple REST service. + +- **Module:** `github.com/rusq/aklapi` (`go 1.24`) +- **Library package:** root (`aklapi`) +- **Binary:** `cmd/aklapi/` — standard HTTP server on port 8080 +- **Language:** Go only — no TypeScript, JavaScript, or Node tooling + +--- + +## Build, Run & Test Commands + +```sh +# Build the server binary +go build -o server ./cmd/aklapi + +# Run the server (port defaults to 8080) +./server + +# Run all tests +go test ./... + +# Run all tests with verbose output +go test -v ./... + +# Run a single test by name (supports regex) +go test -v -run TestFunctionName ./... + +# Run a single test in a specific package +go test -v -run TestCollectionDayDetail ./cmd/aklapi/ + +# Run tests with race detector +go test -race ./... + +# Build all packages (verify compilation) +go build -v ./... + +# Format code (use goimports, not gofmt) +goimports -w . + +# Lint (golangci-lint with default config) +golangci-lint run ./... + +# Docker build +docker build -t aklapi . + +# Make targets +make server # go build -o server ./cmd/aklapi +make docker # docker build -t aklapi . +``` + +> **To run a single test:** use `go test -v -run <./package/path>` +> Example: `go test -v -run TestNextRubbish .` + +--- + +## Code Style Guidelines + +### Formatting + +- Use **`goimports`** (not plain `gofmt`) — it manages imports automatically. +- Indentation: **tabs** (Go standard). +- VS Code devcontainer is configured with `"editor.formatOnSave": true` using `goimports`. +- No trailing whitespace; no blank lines at end of file. + +### Imports + +Group imports in two blocks separated by a blank line: +1. Standard library +2. Third-party packages + +```go +import ( + "context" + "encoding/json" + "net/http" + + "github.com/PuerkitoBio/goquery" +) +``` + +- Use blank imports only where required: `_ "time/tzdata"`, `_ "embed"`. +- Never use dot imports (`.`). +- Alias imports only when disambiguation is genuinely needed. + +### Naming Conventions + +| Element | Convention | Example | +|---|---|---| +| Exported types | PascalCase | `AddrRequest`, `RubbishCollection` | +| Unexported types | camelCase | `refuseParser`, `lruCache` | +| Exported functions | PascalCase | `AddressLookup`, `CollectionDayDetail` | +| Unexported functions | camelCase | `fetchandparse`, `oneAddress` | +| Receiver names | Short (1–2 chars) | `(r *RubbishCollection)`, `(c *lruCache[K,V])` | +| Package-level vars | camelCase | `addrCache`, `defaultLoc` | +| Unexported constants | camelCase | `defCacheSz`, `dateLayout` | +| Acronyms | Go convention | `addrURI` (not `addrUrl`), `ID` (not `Id`) | + +### Types & Structs + +- Add JSON struct tags to all exported response types: `json:"field,omitempty"`. +- Prefer pointer receivers for types that may mutate state or are large. +- Use **generics** for reusable containers (see `lruCache[K comparable, V any]`). +- Use a stateful parser type (struct with fields for state, error, and results) when + parsing multi-step data (see `refuseParser`). + +### Error Handling + +- Always check errors: `if err != nil { return nil, err }`. +- No `panic` in production code. +- Use `errors.New("...")` for static error messages. +- Use string concatenation (not `fmt.Sprintf`) for simple dynamic error strings: + ```go + errors.New("address API returned status code: " + strconv.Itoa(resp.StatusCode)) + ``` +- Prefer `fmt.Errorf("context: %w", err)` for wrapping errors that need context. +- Use package-level sentinel errors for flow control: + ```go + var errSkip = errors.New("skip this date") + ``` +- Use `errors.Is` for sentinel error comparisons. +- HTTP handlers: use `http.Error(w, msg, code)` or a typed `respond(w, body, code)` helper. + +### HTTP & Networking + +- Use **standard library `net/http` only** — no external router (no Gin, Echo, Chi). +- Register routes with `http.HandleFunc` on the default mux. +- Always pass context to outgoing HTTP requests: + ```go + req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) + ``` +- Always `defer resp.Body.Close()` immediately after a successful response. +- Decode JSON responses with `json.NewDecoder(resp.Body).Decode(&v)`. + +### Logging + +- Use **`log/slog`** for all logging — not `log.Printf`, `fmt.Println`, etc. +- Prefer context-aware variants: `slog.DebugContext(ctx, ...)`, `slog.InfoContext(ctx, ...)`. +- Add structured key-value pairs for observability: + ```go + start := time.Now() + // ... operation ... + slog.DebugContext(ctx, "fetched addresses", "count", len(results), "duration", time.Since(start)) + ``` + +### Dependency Injection & Testability + +- Declare external URLs as **package-level `var`** (not `const`) so tests can override them: + ```go + var addrURI = `https://example.com/api/addresses` + ``` +- Inject time via a replaceable variable: `var now = time.Now`. +- Injectable function-type variables enable handler testing without real upstream calls: + ```go + var addressLookup = aklapi.AddressLookup + ``` +- Restore overridden vars with `defer`: + ```go + old := addrURI + addrURI = ts.URL + defer func() { addrURI = old }() + ``` + +--- + +## Testing Guidelines + +### Style + +- Use **table-driven tests** for all non-trivial functions. +- Table entry struct fields: `name string`, `args`, `want`, `wantErr bool`. +- Field names may be omitted for the `name` field in composite literals. +- Prefer `github.com/stretchr/testify/assert` for assertions in new tests (avoid raw + `reflect.DeepEqual` + `t.Errorf` patterns from older tests). +- Use `t.Context()` (Go 1.24+) for context in subtests. +- Use `t.Cleanup(func() {...})` for teardown instead of `defer` in the test function body + when working with subtests. + +### HTTP Testing + +- Use `net/http/httptest.NewServer` to mock upstream APIs. +- Use `httptest.NewRequest` + `httptest.NewRecorder` for handler unit tests. + +### Test Fixtures + +- Embed HTML fixture files with `//go:embed`: + ```go + //go:embed test_assets/some-page.html + var fixtureHTML []byte + ``` +- Fixtures are refreshed by `//go:generate` directives that `curl` the live page. + +### Subtests + +- Always run subtests with `t.Run(tt.name, func(t *testing.T) { ... })`. +- Use `t.Helper()` in assertion helper functions. + +--- + +## Repository Conventions + +- **One concern per file:** `addr.go`, `rubbish.go`, `caches.go`, `time.go`. +- **Library in root, binary in `cmd/`:** follows standard Go project layout. +- CI runs on `push` and `pull_request` to `master` (see `.github/workflows/go.yml`): + `go build -v ./...` then `go test -v ./...`. +- Docker images are published to `ffffuuu/aklapi` on GitHub Release events. diff --git a/addr.go b/addr.go index a51b69d..fa4958b 100644 --- a/addr.go +++ b/addr.go @@ -12,7 +12,7 @@ import ( var ( // defined as a variable so it can be overridden in tests. - addrURI = `https://www.aucklandcouncil.govt.nz/nextapi/property` + addrURI = `https://experience.aucklandcouncil.govt.nz/nextapi/property` ) // AddrRequest is the address request. @@ -23,8 +23,8 @@ type AddrRequest struct { // Address is the address and its unique identifier (rate account key). type Address struct { - ID string `json:"ID"` - Address string `json:"Address"` + ID string `json:"id"` + Address string `json:"address"` } // AddrResponse is the address response. @@ -61,8 +61,7 @@ func MatchingPropertyAddresses(ctx context.Context, addrReq *AddrRequest) (*Addr req.URL.RawQuery = q.Encode() start := time.Now() - client := &http.Client{} - resp, err := client.Do(req) + resp, err := aklClient.Do(req) if err != nil { return nil, err } diff --git a/aklapi.go b/aklapi.go index 21988ea..f22b2d3 100644 --- a/aklapi.go +++ b/aklapi.go @@ -1,9 +1,39 @@ package aklapi import ( + "net/http" "time" ) var ( defaultLoc, _ = time.LoadLocation("Pacific/Auckland") // Auckland is in NZ. ) + +// userAgent is sent with all outgoing HTTP requests. The Auckland Council +// website CDN (Fastly) returns 406 for requests that identify as Go's default +// http client, so we send a browser-compatible value instead. +const userAgent = "Mozilla/5.0 (compatible; aklapi/1.0)" + +// aklClient is a shared HTTP client that injects the required headers on every +// outgoing request so that the Auckland Council CDN does not reject them. +var aklClient = &http.Client{ + Transport: &browserTransport{wrapped: http.DefaultTransport}, +} + +// browserTransport is an http.RoundTripper that adds browser-like headers to +// every request before forwarding it to the underlying transport. +type browserTransport struct { + wrapped http.RoundTripper +} + +func (t *browserTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // Clone the request so we don't mutate the caller's copy. + r := req.Clone(req.Context()) + if r.Header.Get("User-Agent") == "" { + r.Header.Set("User-Agent", userAgent) + } + if r.Header.Get("Accept") == "" { + r.Header.Set("Accept", "application/json, text/html, */*") + } + return t.wrapped.RoundTrip(r) +} diff --git a/cmd/aklapi/handlers.go b/cmd/aklapi/handlers.go index d5ebe04..8e161a4 100644 --- a/cmd/aklapi/handlers.go +++ b/cmd/aklapi/handlers.go @@ -10,6 +10,12 @@ import ( "github.com/rusq/aklapi" ) +// injectable for testing +var ( + addressLookup = aklapi.AddressLookup + collectionDayDetail = aklapi.CollectionDayDetail +) + const dttmLayout = "2006-01-02" type rrResponse struct { @@ -35,7 +41,7 @@ func rubbish(r *http.Request) (*aklapi.CollectionDayDetailResult, error) { if addr == "" { return nil, errors.New(http.StatusText(http.StatusBadRequest)) } - return aklapi.CollectionDayDetail(r.Context(), addr) + return collectionDayDetail(r.Context(), addr) } func addrHandler(w http.ResponseWriter, r *http.Request) { @@ -44,10 +50,10 @@ func addrHandler(w http.ResponseWriter, r *http.Request) { http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) return } - resp, err := aklapi.AddressLookup(r.Context(), addr) + resp, err := addressLookup(r.Context(), addr) if err != nil { slog.Error("address lookup failed", "error", err) - http.NotFound(w, r) + http.Error(w, err.Error(), http.StatusBadGateway) return } respond(w, resp, http.StatusOK) diff --git a/rubbish.go b/rubbish.go index c1371b0..a78fcc5 100644 --- a/rubbish.go +++ b/rubbish.go @@ -22,7 +22,7 @@ const ( var ( // defined as a variable so it can be overridden in tests. - collectionDayURI = `https://new.aucklandcouncil.govt.nz/en/rubbish-recycling/rubbish-recycling-collections/rubbish-recycling-collection-days/%s.html` + collectionDayURI = `https://www.aucklandcouncil.govt.nz/en/rubbish-recycling/rubbish-recycling-collections/rubbish-recycling-collection-days/%s.html` ) var errSkip = errors.New("skip this date") @@ -118,7 +118,7 @@ func fetchandparse(ctx context.Context, addressID string) (*CollectionDayDetailR if err != nil { return nil, err } - resp, err := http.DefaultClient.Do(req) + resp, err := aklClient.Do(req) if err != nil { return nil, err } diff --git a/rubbish_test.go b/rubbish_test.go index 0ee7865..046ddc8 100644 --- a/rubbish_test.go +++ b/rubbish_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/assert" ) -//go:generate curl -L https://new.aucklandcouncil.govt.nz/en/rubbish-recycling/rubbish-recycling-collections/rubbish-recycling-collection-days/12342478585.html -o test_assets/500-queen-street.html -//go:generate curl -L https://new.aucklandcouncil.govt.nz/en/rubbish-recycling/rubbish-recycling-collections/rubbish-recycling-collection-days/12341511281.html -o test_assets/1-luanda-drive.html +//go:generate curl -L https://www.aucklandcouncil.govt.nz/en/rubbish-recycling/rubbish-recycling-collections/rubbish-recycling-collection-days/12342478585.html -o test_assets/500-queen-street.html +//go:generate curl -L https://www.aucklandcouncil.govt.nz/en/rubbish-recycling/rubbish-recycling-collections/rubbish-recycling-collection-days/12341511281.html -o test_assets/1-luanda-drive.html // Test data, run go:generate to update, then update dates in tests // accordingly.