Fix login failures caused by Apple's native auth endpoint migration

cmd/common.go

@@ -54,11 +54,25 @@ func newLogger(format OutputFormat, verbose bool) log.Logger {
 
 // newCookieJar returns a new cookie jar instance.
 func newCookieJar(machine machine.Machine) http.CookieJar {
+	cookiePath := filepath.Join(machine.HomeDirectory(), ConfigDirectoryName, CookieJarFileName)
+	removeStaleZeroByteLock(cookiePath + ".lock")
 	return util.Must(cookiejar.New(&cookiejar.Options{
-		Filename: filepath.Join(machine.HomeDirectory(), ConfigDirectoryName, CookieJarFileName),
+		Filename: cookiePath,
 	}))
 }
 
+// removeStaleZeroByteLock removes a zero-byte lock file left behind when a process
+// is killed between O_TRUNC and writing its PID. The juju/go4/lock library only
+// performs PID-based stale detection when the file has content.
+func removeStaleZeroByteLock(lockPath string) {
+	fi, err := os.Stat(lockPath)
+	if err != nil || fi.Size() != 0 {
+		return
+	}
+
+	_ = os.Remove(lockPath)
+}
+
 // newKeychain returns a new keychain instance.
 func newKeychain(machine machine.Machine, logger log.Logger, interactive bool) keychain.Keychain {
 	ring := util.Must(keyring.Open(keyring.Config{

pkg/appstore/appstore_bag.go

@@ -33,12 +33,13 @@ func (t *appstore) Bag(input BagInput) (BagOutput, error) {
 	}
 
 	return BagOutput{
-		AuthEndpoint: res.Data.URLBag.AuthEndpoint,
+		AuthEndpoint: normalizeAuthEndpoint(res.Data.AuthEndpoint, res.Data.URLBag.AuthEndpoint),
 	}, nil
 }
 
 type bagResult struct {
-	URLBag urlBag `plist:"urlBag,omitempty"`
+	URLBag       urlBag `plist:"urlBag,omitempty"`
+	AuthEndpoint string `plist:"authenticateAccount,omitempty"`
 }
 
 type urlBag struct {

pkg/appstore/appstore_bag_test.go

@@ -118,6 +118,29 @@ var _ = Describe("AppStore (Bag)", func() {
 		})
 	})
 
+	When("request is successful with authenticateAccount at the root", func() {
+		BeforeEach(func() {
+			mockMachine.EXPECT().
+				MacAddress().
+				Return("aa:bb:cc:dd:ee:ff", nil)
+
+			mockBagClient.EXPECT().
+				Send(gomock.Any()).
+				Return(http.Result[bagResult]{
+					StatusCode: gohttp.StatusOK,
+					Data: bagResult{
+						AuthEndpoint: "https://auth.itunes.apple.com/auth/v1/native",
+					},
+				}, nil)
+		})
+
+		It("returns the normalized native auth endpoint", func() {
+			out, err := as.Bag(BagInput{})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out.AuthEndpoint).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/"))
+		})
+	})
+
 	When("request is successful but authenticateAccount is empty", func() {
 		BeforeEach(func() {
 			mockMachine.EXPECT().
@@ -132,10 +155,10 @@ var _ = Describe("AppStore (Bag)", func() {
 				}, nil)
 		})
 
-		It("returns empty auth endpoint", func() {
+		It("returns the fallback auth endpoint", func() {
 			out, err := as.Bag(BagInput{})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(out.AuthEndpoint).To(BeEmpty())
+			Expect(out.AuthEndpoint).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/"))
 		})
 	})
 })

pkg/appstore/appstore_live_test.go

@@ -0,0 +1,89 @@
+//go:build integration
+
+package appstore
+
+import (
+	"errors"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"strings"
+	"testing"
+
+	pkghttp "github.com/majd/ipatool/v2/pkg/http"
+	"github.com/majd/ipatool/v2/pkg/keychain"
+	"github.com/majd/ipatool/v2/pkg/util/machine"
+	"github.com/majd/ipatool/v2/pkg/util/operatingsystem"
+)
+
+type integrationCookieJar struct {
+	jar *cookiejar.Jar
+}
+
+func (c integrationCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie) {
+	c.jar.SetCookies(u, cookies)
+}
+
+func (c integrationCookieJar) Cookies(u *url.URL) []*http.Cookie {
+	return c.jar.Cookies(u)
+}
+
+func (integrationCookieJar) Save() error { return nil }
+
+func newIntegrationAppStore(t *testing.T) AppStore {
+	t.Helper()
+
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		t.Fatalf("cookie jar: %v", err)
+	}
+
+	return NewAppStore(Args{
+		CookieJar:       integrationCookieJar{jar: jar},
+		OperatingSystem: operatingsystem.New(),
+		Machine:         machine.New(machine.Args{OS: operatingsystem.New()}),
+		Keychain:        keychain.New(keychain.Args{}),
+	})
+}
+
+func TestLiveBagReturnsNativeAuthEndpoint(t *testing.T) {
+	as := newIntegrationAppStore(t)
+
+	out, err := as.Bag(BagInput{})
+	if err != nil {
+		t.Fatalf("Bag: %v", err)
+	}
+
+	if out.AuthEndpoint != defaultNativeAuthEndpoint {
+		t.Fatalf("AuthEndpoint = %q, want %q", out.AuthEndpoint, defaultNativeAuthEndpoint)
+	}
+}
+
+func TestLiveLoginUsesNativeAuthEndpoint(t *testing.T) {
+	as := newIntegrationAppStore(t)
+
+	bag, err := as.Bag(BagInput{})
+	if err != nil {
+		t.Fatalf("Bag: %v", err)
+	}
+
+	_, err = as.Login(LoginInput{
+		Email:    "ipatool-integration-test@example.com",
+		Password: "not-a-real-password",
+		Endpoint: bag.AuthEndpoint,
+	})
+	if err == nil {
+		t.Fatal("expected login to fail with invalid credentials")
+	}
+
+	var decodeErr *pkghttp.ResponseDecodeError
+	if errors.As(err, &decodeErr) {
+		t.Fatalf("login returned plist decode error (wrong endpoint or non-plist body): %v", err)
+	}
+
+	if strings.Contains(err.Error(), "failed to unmarshal xml") {
+		t.Fatalf("login hit legacy plist parse failure: %v", err)
+	}
+
+	t.Logf("login failed as expected with invalid credentials: %v", err)
+}

pkg/appstore/appstore_login.go

@@ -65,6 +65,7 @@ type loginResult struct {
 
 func (t *appstore) login(email, password, authCode, guid, endpoint string) (Account, error) {
 	redirect := ""
+	authEndpoint := normalizeAuthEndpoint(endpoint)
 
 	var (
 		err error
@@ -74,11 +75,17 @@ func (t *appstore) login(email, password, authCode, guid, endpoint string) (Acco
 	retry := true
 
 	for attempt := 1; retry && attempt <= 4; attempt++ {
-		request := t.loginRequest(email, password, authCode, guid, endpoint, attempt)
+		request := t.loginRequest(email, password, authCode, guid, authEndpoint, attempt)
 		request.URL, _ = util.IfEmpty(redirect, request.URL), ""
 		res, err = t.loginClient.Send(request)
 
 		if err != nil {
+			if discoveredEndpoint := authEndpointFromResponseError(err); discoveredEndpoint != "" && discoveredEndpoint != authEndpoint {
+				authEndpoint = discoveredEndpoint
+				redirect = ""
+				continue
+			}
+
 			return Account{}, fmt.Errorf("request failed: %w", err)
 		}
 
@@ -144,6 +151,8 @@ func (t *appstore) parseLoginResponse(res *http.Result[loginResult], attempt int
 		err = ErrAuthCodeRequired
 	} else if res.Data.FailureType == "" && res.Data.CustomerMessage == CustomerMessageAccountDisabled {
 		err = NewErrorWithMetadata(errors.New("account is disabled"), res)
+	} else if res.Data.FailureType == "" && res.Data.CustomerMessage == CustomerMessageActionSignInPage {
+		err = NewErrorWithMetadata(errors.New("account requires browser sign-in (2FA or Apple ID review required)"), res)
 	} else if res.Data.FailureType != "" {
 		if res.Data.CustomerMessage != "" {
 			err = NewErrorWithMetadata(errors.New(res.Data.CustomerMessage), res)

pkg/appstore/appstore_login_test.go

@@ -73,6 +73,9 @@ var _ = Describe("AppStore (Login)", func() {
 			BeforeEach(func() {
 				mockClient.EXPECT().
 					Send(gomock.Any()).
+					Do(func(req http.Request) {
+						Expect(req.URL).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/"))
+					}).
 					Return(http.Result[loginResult]{}, errors.New(""))
 			})
 
@@ -164,6 +167,26 @@ var _ = Describe("AppStore (Login)", func() {
 			})
 		})
 
+		When("store API requires browser sign-in", func() {
+			BeforeEach(func() {
+				mockClient.EXPECT().
+					Send(gomock.Any()).
+					Return(http.Result[loginResult]{
+						Data: loginResult{
+							CustomerMessage: CustomerMessageActionSignInPage,
+						},
+					}, nil)
+			})
+
+			It("returns browser sign-in error", func() {
+				_, err := as.Login(LoginInput{
+					Password: testPassword,
+				})
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(ContainSubstring("browser sign-in"))
+			})
+		})
+
 		When("store API redirects", func() {
 			const (
 				testRedirectLocation = "https://test-redirect-url.com"
@@ -201,6 +224,32 @@ var _ = Describe("AppStore (Login)", func() {
 			})
 		})
 
+		When("store API response contains a new auth endpoint", func() {
+			BeforeEach(func() {
+				firstCall := mockClient.EXPECT().
+					Send(gomock.Any()).
+					Return(http.Result[loginResult]{}, &http.ResponseDecodeError{
+						Cause: errors.New("decode failed"),
+						URLs:  []string{"https://auth.itunes.apple.com/auth/v1/native"},
+					})
+				secondCall := mockClient.EXPECT().
+					Send(gomock.Any()).
+					Do(func(req http.Request) {
+						Expect(req.URL).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/"))
+					}).
+					Return(http.Result[loginResult]{}, errors.New("test complete"))
+				gomock.InOrder(firstCall, secondCall)
+			})
+
+			It("retries with the discovered endpoint", func() {
+				_, err := as.Login(LoginInput{
+					Endpoint: "https://example.com/authenticate",
+					Password: testPassword,
+				})
+				Expect(err).To(MatchError(ContainSubstring("test complete")))
+			})
+		})
+
 		When("store API redirects too much", func() {
 			BeforeEach(func() {
 				mockClient.EXPECT().

pkg/appstore/auth_endpoint.go

@@ -0,0 +1,71 @@
+package appstore
+
+import (
+	"errors"
+	"html"
+	"net/url"
+	"strings"
+
+	"github.com/majd/ipatool/v2/pkg/http"
+)
+
+func normalizeAuthEndpoint(endpoints ...string) string {
+	for _, endpoint := range endpoints {
+		endpoint = strings.TrimSpace(endpoint)
+		if endpoint == "" {
+			continue
+		}
+
+		if normalized := normalizeNativeAuthEndpoint(endpoint); normalized != "" {
+			return normalized
+		}
+
+		return endpoint
+	}
+
+	return defaultNativeAuthEndpoint
+}
+
+func authEndpointFromResponseError(err error) string {
+	var decodeErr *http.ResponseDecodeError
+	if !errors.As(err, &decodeErr) {
+		return ""
+	}
+
+	parts := make([]string, 0, len(decodeErr.URLs)+1)
+	parts = append(parts, decodeErr.URLs...)
+	if decodeErr.Body != "" {
+		parts = append(parts, decodeErr.Body)
+	}
+	if len(parts) == 0 {
+		return ""
+	}
+
+	return authEndpointFromText(strings.Join(parts, " "))
+}
+
+func authEndpointFromText(text string) string {
+	text = html.UnescapeString(strings.ReplaceAll(text, `\/`, `/`))
+	for _, match := range http.ExtractURLs([]byte(text)) {
+		if endpoint := normalizeNativeAuthEndpoint(strings.TrimRight(match, ".,;)")); endpoint != "" {
+			return endpoint
+		}
+	}
+
+	return ""
+}
+
+func normalizeNativeAuthEndpoint(endpoint string) string {
+	parsed, err := url.Parse(endpoint)
+	if err != nil || parsed.Host != PrivateAuthDomain {
+		return ""
+	}
+
+	path := strings.TrimRight(parsed.Path, "/")
+	if !strings.HasSuffix(path, "/fast") {
+		path = strings.TrimRight(path, "/") + "/fast"
+	}
+	parsed.Path = path + "/"
+
+	return parsed.String()
+}