From c54feb36ebd6a0da24cf8a2459d9b8f521c3ca98 Mon Sep 17 00:00:00 2001 From: Bar Malka Date: Tue, 30 Jun 2026 08:58:35 +0300 Subject: [PATCH 1/2] Fix login failures caused by Apple's native auth endpoint migration. Normalize authenticateAccount from Bag responses, discover alternate endpoints from malformed login responses, handle browser sign-in redirects, and remove stale zero-byte cookie-jar lock files. Co-authored-by: Cursor --- cmd/common.go | 16 ++++++- pkg/appstore/appstore_bag.go | 5 +- pkg/appstore/appstore_bag_test.go | 27 ++++++++++- pkg/appstore/appstore_login.go | 11 ++++- pkg/appstore/appstore_login_test.go | 49 ++++++++++++++++++++ pkg/appstore/auth_endpoint.go | 71 +++++++++++++++++++++++++++++ pkg/appstore/auth_endpoint_test.go | 58 +++++++++++++++++++++++ pkg/appstore/constants.go | 6 +++ pkg/http/client.go | 52 ++++++++++++++++++++- pkg/http/client_test.go | 32 ++++++++++++- 10 files changed, 319 insertions(+), 8 deletions(-) create mode 100644 pkg/appstore/auth_endpoint.go create mode 100644 pkg/appstore/auth_endpoint_test.go diff --git a/cmd/common.go b/cmd/common.go index 3b3a3dec..112eee34 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -54,11 +54,25 @@ func newLogger(format OutputFormat, verbose bool) log.Logger { // newCookieJar returns a new cookie jar instance. func newCookieJar(machine machine.Machine) http.CookieJar { + cookiePath := filepath.Join(machine.HomeDirectory(), ConfigDirectoryName, CookieJarFileName) + removeStaleZeroByteLock(cookiePath + ".lock") return util.Must(cookiejar.New(&cookiejar.Options{ - Filename: filepath.Join(machine.HomeDirectory(), ConfigDirectoryName, CookieJarFileName), + Filename: cookiePath, })) } +// removeStaleZeroByteLock removes a zero-byte lock file left behind when a process +// is killed between O_TRUNC and writing its PID. The juju/go4/lock library only +// performs PID-based stale detection when the file has content. +func removeStaleZeroByteLock(lockPath string) { + fi, err := os.Stat(lockPath) + if err != nil || fi.Size() != 0 { + return + } + + _ = os.Remove(lockPath) +} + // newKeychain returns a new keychain instance. func newKeychain(machine machine.Machine, logger log.Logger, interactive bool) keychain.Keychain { ring := util.Must(keyring.Open(keyring.Config{ diff --git a/pkg/appstore/appstore_bag.go b/pkg/appstore/appstore_bag.go index 29108a2f..8de570c5 100644 --- a/pkg/appstore/appstore_bag.go +++ b/pkg/appstore/appstore_bag.go @@ -33,12 +33,13 @@ func (t *appstore) Bag(input BagInput) (BagOutput, error) { } return BagOutput{ - AuthEndpoint: res.Data.URLBag.AuthEndpoint, + AuthEndpoint: normalizeAuthEndpoint(res.Data.AuthEndpoint, res.Data.URLBag.AuthEndpoint), }, nil } type bagResult struct { - URLBag urlBag `plist:"urlBag,omitempty"` + URLBag urlBag `plist:"urlBag,omitempty"` + AuthEndpoint string `plist:"authenticateAccount,omitempty"` } type urlBag struct { diff --git a/pkg/appstore/appstore_bag_test.go b/pkg/appstore/appstore_bag_test.go index ae9f1f27..15d94aed 100644 --- a/pkg/appstore/appstore_bag_test.go +++ b/pkg/appstore/appstore_bag_test.go @@ -118,6 +118,29 @@ var _ = Describe("AppStore (Bag)", func() { }) }) + When("request is successful with authenticateAccount at the root", func() { + BeforeEach(func() { + mockMachine.EXPECT(). + MacAddress(). + Return("aa:bb:cc:dd:ee:ff", nil) + + mockBagClient.EXPECT(). + Send(gomock.Any()). + Return(http.Result[bagResult]{ + StatusCode: gohttp.StatusOK, + Data: bagResult{ + AuthEndpoint: "https://auth.itunes.apple.com/auth/v1/native", + }, + }, nil) + }) + + It("returns the normalized native auth endpoint", func() { + out, err := as.Bag(BagInput{}) + Expect(err).ToNot(HaveOccurred()) + Expect(out.AuthEndpoint).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/")) + }) + }) + When("request is successful but authenticateAccount is empty", func() { BeforeEach(func() { mockMachine.EXPECT(). @@ -132,10 +155,10 @@ var _ = Describe("AppStore (Bag)", func() { }, nil) }) - It("returns empty auth endpoint", func() { + It("returns the fallback auth endpoint", func() { out, err := as.Bag(BagInput{}) Expect(err).ToNot(HaveOccurred()) - Expect(out.AuthEndpoint).To(BeEmpty()) + Expect(out.AuthEndpoint).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/")) }) }) }) diff --git a/pkg/appstore/appstore_login.go b/pkg/appstore/appstore_login.go index 419fdeaa..43080d58 100644 --- a/pkg/appstore/appstore_login.go +++ b/pkg/appstore/appstore_login.go @@ -65,6 +65,7 @@ type loginResult struct { func (t *appstore) login(email, password, authCode, guid, endpoint string) (Account, error) { redirect := "" + authEndpoint := normalizeAuthEndpoint(endpoint) var ( err error @@ -74,11 +75,17 @@ func (t *appstore) login(email, password, authCode, guid, endpoint string) (Acco retry := true for attempt := 1; retry && attempt <= 4; attempt++ { - request := t.loginRequest(email, password, authCode, guid, endpoint, attempt) + request := t.loginRequest(email, password, authCode, guid, authEndpoint, attempt) request.URL, _ = util.IfEmpty(redirect, request.URL), "" res, err = t.loginClient.Send(request) if err != nil { + if discoveredEndpoint := authEndpointFromResponseError(err); discoveredEndpoint != "" && discoveredEndpoint != authEndpoint { + authEndpoint = discoveredEndpoint + redirect = "" + continue + } + return Account{}, fmt.Errorf("request failed: %w", err) } @@ -144,6 +151,8 @@ func (t *appstore) parseLoginResponse(res *http.Result[loginResult], attempt int err = ErrAuthCodeRequired } else if res.Data.FailureType == "" && res.Data.CustomerMessage == CustomerMessageAccountDisabled { err = NewErrorWithMetadata(errors.New("account is disabled"), res) + } else if res.Data.FailureType == "" && res.Data.CustomerMessage == CustomerMessageActionSignInPage { + err = NewErrorWithMetadata(errors.New("account requires browser sign-in (2FA or Apple ID review required)"), res) } else if res.Data.FailureType != "" { if res.Data.CustomerMessage != "" { err = NewErrorWithMetadata(errors.New(res.Data.CustomerMessage), res) diff --git a/pkg/appstore/appstore_login_test.go b/pkg/appstore/appstore_login_test.go index 3521eeb1..f0c0ad22 100644 --- a/pkg/appstore/appstore_login_test.go +++ b/pkg/appstore/appstore_login_test.go @@ -73,6 +73,9 @@ var _ = Describe("AppStore (Login)", func() { BeforeEach(func() { mockClient.EXPECT(). Send(gomock.Any()). + Do(func(req http.Request) { + Expect(req.URL).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/")) + }). Return(http.Result[loginResult]{}, errors.New("")) }) @@ -164,6 +167,26 @@ var _ = Describe("AppStore (Login)", func() { }) }) + When("store API requires browser sign-in", func() { + BeforeEach(func() { + mockClient.EXPECT(). + Send(gomock.Any()). + Return(http.Result[loginResult]{ + Data: loginResult{ + CustomerMessage: CustomerMessageActionSignInPage, + }, + }, nil) + }) + + It("returns browser sign-in error", func() { + _, err := as.Login(LoginInput{ + Password: testPassword, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("browser sign-in")) + }) + }) + When("store API redirects", func() { const ( testRedirectLocation = "https://test-redirect-url.com" @@ -201,6 +224,32 @@ var _ = Describe("AppStore (Login)", func() { }) }) + When("store API response contains a new auth endpoint", func() { + BeforeEach(func() { + firstCall := mockClient.EXPECT(). + Send(gomock.Any()). + Return(http.Result[loginResult]{}, &http.ResponseDecodeError{ + Cause: errors.New("decode failed"), + URLs: []string{"https://auth.itunes.apple.com/auth/v1/native"}, + }) + secondCall := mockClient.EXPECT(). + Send(gomock.Any()). + Do(func(req http.Request) { + Expect(req.URL).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/")) + }). + Return(http.Result[loginResult]{}, errors.New("test complete")) + gomock.InOrder(firstCall, secondCall) + }) + + It("retries with the discovered endpoint", func() { + _, err := as.Login(LoginInput{ + Endpoint: "https://example.com/authenticate", + Password: testPassword, + }) + Expect(err).To(MatchError(ContainSubstring("test complete"))) + }) + }) + When("store API redirects too much", func() { BeforeEach(func() { mockClient.EXPECT(). diff --git a/pkg/appstore/auth_endpoint.go b/pkg/appstore/auth_endpoint.go new file mode 100644 index 00000000..6f4f2309 --- /dev/null +++ b/pkg/appstore/auth_endpoint.go @@ -0,0 +1,71 @@ +package appstore + +import ( + "errors" + "html" + "net/url" + "strings" + + "github.com/majd/ipatool/v2/pkg/http" +) + +func normalizeAuthEndpoint(endpoints ...string) string { + for _, endpoint := range endpoints { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + continue + } + + if normalized := normalizeNativeAuthEndpoint(endpoint); normalized != "" { + return normalized + } + + return endpoint + } + + return defaultNativeAuthEndpoint +} + +func authEndpointFromResponseError(err error) string { + var decodeErr *http.ResponseDecodeError + if !errors.As(err, &decodeErr) { + return "" + } + + parts := make([]string, 0, len(decodeErr.URLs)+1) + parts = append(parts, decodeErr.URLs...) + if decodeErr.Body != "" { + parts = append(parts, decodeErr.Body) + } + if len(parts) == 0 { + return "" + } + + return authEndpointFromText(strings.Join(parts, " ")) +} + +func authEndpointFromText(text string) string { + text = html.UnescapeString(strings.ReplaceAll(text, `\/`, `/`)) + for _, match := range http.ExtractURLs([]byte(text)) { + if endpoint := normalizeNativeAuthEndpoint(strings.TrimRight(match, ".,;)")); endpoint != "" { + return endpoint + } + } + + return "" +} + +func normalizeNativeAuthEndpoint(endpoint string) string { + parsed, err := url.Parse(endpoint) + if err != nil || parsed.Host != PrivateAuthDomain { + return "" + } + + path := strings.TrimRight(parsed.Path, "/") + if !strings.HasSuffix(path, "/fast") { + path = strings.TrimRight(path, "/") + "/fast" + } + parsed.Path = path + "/" + + return parsed.String() +} diff --git a/pkg/appstore/auth_endpoint_test.go b/pkg/appstore/auth_endpoint_test.go new file mode 100644 index 00000000..8dfa1cff --- /dev/null +++ b/pkg/appstore/auth_endpoint_test.go @@ -0,0 +1,58 @@ +package appstore + +import ( + "errors" + + "github.com/majd/ipatool/v2/pkg/http" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Auth endpoint", func() { + It("falls back to the native auth endpoint", func() { + Expect(normalizeAuthEndpoint()).To(Equal(defaultNativeAuthEndpoint)) + }) + + It("normalizes Apple's native auth endpoint with the fast path and trailing slash", func() { + Expect(normalizeAuthEndpoint("https://auth.itunes.apple.com/auth/v1/native")). + To(Equal(defaultNativeAuthEndpoint)) + Expect(normalizeAuthEndpoint("https://auth.itunes.apple.com/auth/v1/native/fast")). + To(Equal(defaultNativeAuthEndpoint)) + }) + + It("keeps legacy endpoints unchanged", func() { + endpoint := "https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate" + Expect(normalizeAuthEndpoint(endpoint)).To(Equal(endpoint)) + }) + + It("prefers the first non-empty endpoint", func() { + Expect(normalizeAuthEndpoint( + "https://auth.itunes.apple.com/auth/v1/native", + "https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate", + )).To(Equal(defaultNativeAuthEndpoint)) + }) + + It("extracts a native endpoint from an escaped response body", func() { + body := `{"authenticateAccount":"https:\/\/auth.itunes.apple.com\/auth\/v1\/native"}` + Expect(authEndpointFromText(body)).To(Equal(defaultNativeAuthEndpoint)) + }) + + It("extracts a native endpoint from a response decode error", func() { + err := &http.ResponseDecodeError{ + Cause: errors.New("decode failed"), + URLs: []string{"https://auth.itunes.apple.com/auth/v1/native"}, + } + Expect(authEndpointFromResponseError(err)).To(Equal(defaultNativeAuthEndpoint)) + }) + + It("does not mutate URLs on the decode error", func() { + urls := []string{"https://auth.itunes.apple.com/auth/v1/native"} + err := &http.ResponseDecodeError{ + Cause: errors.New("decode failed"), + URLs: urls, + Body: "ignored", + } + _ = authEndpointFromResponseError(err) + Expect(urls).To(Equal([]string{"https://auth.itunes.apple.com/auth/v1/native"})) + }) +}) diff --git a/pkg/appstore/constants.go b/pkg/appstore/constants.go index 66201514..3748154c 100644 --- a/pkg/appstore/constants.go +++ b/pkg/appstore/constants.go @@ -13,6 +13,7 @@ const ( CustomerMessageAccountDisabled = "Your account is disabled." CustomerMessageSubscriptionRequired = "Subscription Required" CustomerMessagePasswordChanged = "Your password has changed." + CustomerMessageActionSignInPage = "AMD-Action::SP" iTunesAPIDomain = "itunes.apple.com" iTunesAPIPathSearch = "/search" @@ -25,6 +26,11 @@ const ( PrivateAppStoreAPIPathPurchase = "/WebObjects/MZFinance.woa/wa/buyProduct" PrivateAppStoreAPIPathDownload = "/WebObjects/MZFinance.woa/wa/volumeStoreDownloadProduct" + PrivateAuthDomain = "auth." + iTunesAPIDomain + PrivateAuthPathNative = "/auth/v1/native/fast/" + + defaultNativeAuthEndpoint = "https://" + PrivateAuthDomain + PrivateAuthPathNative + HTTPHeaderStoreFront = "X-Set-Apple-Store-Front" HTTPHeaderPod = "pod" diff --git a/pkg/http/client.go b/pkg/http/client.go index 22de26b1..8acb03b2 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -8,6 +8,7 @@ import ( "net/http" "regexp" "strings" + "unicode/utf8" "howett.net/plist" ) @@ -20,8 +21,27 @@ var ( documentXMLPattern = regexp.MustCompile(`(?is)]*>(.*)`) plistXMLPattern = regexp.MustCompile(`(?is)]*>.*?`) dictXMLPattern = regexp.MustCompile(`(?is)]*>.*`) + urlPattern = regexp.MustCompile(`https?://[^\s"'<>]+`) ) +// ResponseDecodeError is returned when an XML/plist response body cannot be decoded. +// URLs and Body may contain hints (such as an alternate auth endpoint) for callers to inspect. +type ResponseDecodeError struct { + Cause error + StatusCode int + ContentType string + Body string + URLs []string +} + +func (e *ResponseDecodeError) Error() string { + return fmt.Sprintf("failed to unmarshal xml: %v", e.Cause) +} + +func (e *ResponseDecodeError) Unwrap() error { + return e.Cause +} + //go:generate go run go.uber.org/mock/mockgen -source=client.go -destination=client_mock.go -package=http type Client[R interface{}] interface { Send(request Request) (Result[R], error) @@ -170,7 +190,13 @@ func (c *client[R]) handleXMLResponse(res *http.Response) (Result[R], error) { _, err = plist.Unmarshal(normalizedBody, &data) if err != nil { - return Result[R]{}, fmt.Errorf("failed to unmarshal xml: %w", err) + return Result[R]{}, &ResponseDecodeError{ + Cause: err, + StatusCode: res.StatusCode, + ContentType: res.Header.Get("Content-Type"), + Body: truncateBody(body, 500), + URLs: ExtractURLs(body), + } } headers := map[string]string{} @@ -185,6 +211,30 @@ func (c *client[R]) handleXMLResponse(res *http.Response) (Result[R], error) { }, nil } +// ExtractURLs returns HTTP(S) URLs found in a response body. +func ExtractURLs(body []byte) []string { + matches := urlPattern.FindAll(body, -1) + urls := make([]string, 0, len(matches)) + for _, match := range matches { + urls = append(urls, string(match)) + } + + return urls +} + +func truncateBody(body []byte, max int) string { + trimmed := strings.TrimSpace(string(body)) + if len(trimmed) <= max { + return trimmed + } + + for max > 0 && !utf8.RuneStart(trimmed[max]) { + max-- + } + + return trimmed[:max] + "..." +} + func normalizeXMLPlistBody(body []byte) []byte { normalized := bytes.TrimSpace(body) if len(normalized) == 0 { diff --git a/pkg/http/client_test.go b/pkg/http/client_test.go index cb0407c2..b2b65515 100644 --- a/pkg/http/client_test.go +++ b/pkg/http/client_test.go @@ -203,7 +203,7 @@ var _ = Describe("Client", Ordered, func() { }) When("payload fails to decode", func() { - It("returns error", func() { + It("returns error when payload cannot be encoded", func() { sut := NewClient[xmlResult](Args{ CookieJar: mockCookieJar, }) @@ -220,5 +220,35 @@ var _ = Describe("Client", Ordered, func() { Expect(err).To(HaveOccurred()) }) + + When("response body is not valid XML", func() { + BeforeEach(func() { + mockCookieJar.EXPECT(). + Save(). + Return(nil) + }) + + It("returns ResponseDecodeError with extracted URLs", func() { + mockHandler = func(w http.ResponseWriter, _r *http.Request) { + w.Header().Add("Content-Type", "application/xml") + _, err := w.Write([]byte(`redirect to https://auth.itunes.apple.com/auth/v1/native`)) + Expect(err).ToNot(HaveOccurred()) + } + + sut := NewClient[xmlResult](Args{ + CookieJar: mockCookieJar, + }) + _, err := sut.Send(Request{ + URL: srv.URL, + Method: MethodPOST, + ResponseFormat: ResponseFormatXML, + }) + + var decodeErr *ResponseDecodeError + Expect(errors.As(err, &decodeErr)).To(BeTrue()) + Expect(decodeErr.URLs).To(ContainElement("https://auth.itunes.apple.com/auth/v1/native")) + Expect(errors.Unwrap(decodeErr)).To(HaveOccurred()) + }) + }) }) }) From 13016ff085abf0fd092907073c37a30be9385145 Mon Sep 17 00:00:00 2001 From: Bar Malka Date: Tue, 30 Jun 2026 09:08:52 +0300 Subject: [PATCH 2/2] Add live integration tests for native auth endpoint verification. Exercise Bag and Login against Apple's production endpoints to confirm the normalized auth URL is used and login no longer fails with plist decode errors. Co-authored-by: Cursor --- pkg/appstore/appstore_live_test.go | 89 ++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 pkg/appstore/appstore_live_test.go diff --git a/pkg/appstore/appstore_live_test.go b/pkg/appstore/appstore_live_test.go new file mode 100644 index 00000000..2d3429b0 --- /dev/null +++ b/pkg/appstore/appstore_live_test.go @@ -0,0 +1,89 @@ +//go:build integration + +package appstore + +import ( + "errors" + "net/http" + "net/http/cookiejar" + "net/url" + "strings" + "testing" + + pkghttp "github.com/majd/ipatool/v2/pkg/http" + "github.com/majd/ipatool/v2/pkg/keychain" + "github.com/majd/ipatool/v2/pkg/util/machine" + "github.com/majd/ipatool/v2/pkg/util/operatingsystem" +) + +type integrationCookieJar struct { + jar *cookiejar.Jar +} + +func (c integrationCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie) { + c.jar.SetCookies(u, cookies) +} + +func (c integrationCookieJar) Cookies(u *url.URL) []*http.Cookie { + return c.jar.Cookies(u) +} + +func (integrationCookieJar) Save() error { return nil } + +func newIntegrationAppStore(t *testing.T) AppStore { + t.Helper() + + jar, err := cookiejar.New(nil) + if err != nil { + t.Fatalf("cookie jar: %v", err) + } + + return NewAppStore(Args{ + CookieJar: integrationCookieJar{jar: jar}, + OperatingSystem: operatingsystem.New(), + Machine: machine.New(machine.Args{OS: operatingsystem.New()}), + Keychain: keychain.New(keychain.Args{}), + }) +} + +func TestLiveBagReturnsNativeAuthEndpoint(t *testing.T) { + as := newIntegrationAppStore(t) + + out, err := as.Bag(BagInput{}) + if err != nil { + t.Fatalf("Bag: %v", err) + } + + if out.AuthEndpoint != defaultNativeAuthEndpoint { + t.Fatalf("AuthEndpoint = %q, want %q", out.AuthEndpoint, defaultNativeAuthEndpoint) + } +} + +func TestLiveLoginUsesNativeAuthEndpoint(t *testing.T) { + as := newIntegrationAppStore(t) + + bag, err := as.Bag(BagInput{}) + if err != nil { + t.Fatalf("Bag: %v", err) + } + + _, err = as.Login(LoginInput{ + Email: "ipatool-integration-test@example.com", + Password: "not-a-real-password", + Endpoint: bag.AuthEndpoint, + }) + if err == nil { + t.Fatal("expected login to fail with invalid credentials") + } + + var decodeErr *pkghttp.ResponseDecodeError + if errors.As(err, &decodeErr) { + t.Fatalf("login returned plist decode error (wrong endpoint or non-plist body): %v", err) + } + + if strings.Contains(err.Error(), "failed to unmarshal xml") { + t.Fatalf("login hit legacy plist parse failure: %v", err) + } + + t.Logf("login failed as expected with invalid credentials: %v", err) +}