diff --git a/cmd/common.go b/cmd/common.go
index 3b3a3dec..112eee34 100644
--- a/cmd/common.go
+++ b/cmd/common.go
@@ -54,11 +54,25 @@ func newLogger(format OutputFormat, verbose bool) log.Logger {
// newCookieJar returns a new cookie jar instance.
func newCookieJar(machine machine.Machine) http.CookieJar {
+ cookiePath := filepath.Join(machine.HomeDirectory(), ConfigDirectoryName, CookieJarFileName)
+ removeStaleZeroByteLock(cookiePath + ".lock")
return util.Must(cookiejar.New(&cookiejar.Options{
- Filename: filepath.Join(machine.HomeDirectory(), ConfigDirectoryName, CookieJarFileName),
+ Filename: cookiePath,
}))
}
+// removeStaleZeroByteLock removes a zero-byte lock file left behind when a process
+// is killed between O_TRUNC and writing its PID. The juju/go4/lock library only
+// performs PID-based stale detection when the file has content.
+func removeStaleZeroByteLock(lockPath string) {
+ fi, err := os.Stat(lockPath)
+ if err != nil || fi.Size() != 0 {
+ return
+ }
+
+ _ = os.Remove(lockPath)
+}
+
// newKeychain returns a new keychain instance.
func newKeychain(machine machine.Machine, logger log.Logger, interactive bool) keychain.Keychain {
ring := util.Must(keyring.Open(keyring.Config{
diff --git a/pkg/appstore/appstore_bag.go b/pkg/appstore/appstore_bag.go
index 29108a2f..8de570c5 100644
--- a/pkg/appstore/appstore_bag.go
+++ b/pkg/appstore/appstore_bag.go
@@ -33,12 +33,13 @@ func (t *appstore) Bag(input BagInput) (BagOutput, error) {
}
return BagOutput{
- AuthEndpoint: res.Data.URLBag.AuthEndpoint,
+ AuthEndpoint: normalizeAuthEndpoint(res.Data.AuthEndpoint, res.Data.URLBag.AuthEndpoint),
}, nil
}
type bagResult struct {
- URLBag urlBag `plist:"urlBag,omitempty"`
+ URLBag urlBag `plist:"urlBag,omitempty"`
+ AuthEndpoint string `plist:"authenticateAccount,omitempty"`
}
type urlBag struct {
diff --git a/pkg/appstore/appstore_bag_test.go b/pkg/appstore/appstore_bag_test.go
index ae9f1f27..15d94aed 100644
--- a/pkg/appstore/appstore_bag_test.go
+++ b/pkg/appstore/appstore_bag_test.go
@@ -118,6 +118,29 @@ var _ = Describe("AppStore (Bag)", func() {
})
})
+ When("request is successful with authenticateAccount at the root", func() {
+ BeforeEach(func() {
+ mockMachine.EXPECT().
+ MacAddress().
+ Return("aa:bb:cc:dd:ee:ff", nil)
+
+ mockBagClient.EXPECT().
+ Send(gomock.Any()).
+ Return(http.Result[bagResult]{
+ StatusCode: gohttp.StatusOK,
+ Data: bagResult{
+ AuthEndpoint: "https://auth.itunes.apple.com/auth/v1/native",
+ },
+ }, nil)
+ })
+
+ It("returns the normalized native auth endpoint", func() {
+ out, err := as.Bag(BagInput{})
+ Expect(err).ToNot(HaveOccurred())
+ Expect(out.AuthEndpoint).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/"))
+ })
+ })
+
When("request is successful but authenticateAccount is empty", func() {
BeforeEach(func() {
mockMachine.EXPECT().
@@ -132,10 +155,10 @@ var _ = Describe("AppStore (Bag)", func() {
}, nil)
})
- It("returns empty auth endpoint", func() {
+ It("returns the fallback auth endpoint", func() {
out, err := as.Bag(BagInput{})
Expect(err).ToNot(HaveOccurred())
- Expect(out.AuthEndpoint).To(BeEmpty())
+ Expect(out.AuthEndpoint).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/"))
})
})
})
diff --git a/pkg/appstore/appstore_live_test.go b/pkg/appstore/appstore_live_test.go
new file mode 100644
index 00000000..2d3429b0
--- /dev/null
+++ b/pkg/appstore/appstore_live_test.go
@@ -0,0 +1,89 @@
+//go:build integration
+
+package appstore
+
+import (
+ "errors"
+ "net/http"
+ "net/http/cookiejar"
+ "net/url"
+ "strings"
+ "testing"
+
+ pkghttp "github.com/majd/ipatool/v2/pkg/http"
+ "github.com/majd/ipatool/v2/pkg/keychain"
+ "github.com/majd/ipatool/v2/pkg/util/machine"
+ "github.com/majd/ipatool/v2/pkg/util/operatingsystem"
+)
+
+type integrationCookieJar struct {
+ jar *cookiejar.Jar
+}
+
+func (c integrationCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie) {
+ c.jar.SetCookies(u, cookies)
+}
+
+func (c integrationCookieJar) Cookies(u *url.URL) []*http.Cookie {
+ return c.jar.Cookies(u)
+}
+
+func (integrationCookieJar) Save() error { return nil }
+
+func newIntegrationAppStore(t *testing.T) AppStore {
+ t.Helper()
+
+ jar, err := cookiejar.New(nil)
+ if err != nil {
+ t.Fatalf("cookie jar: %v", err)
+ }
+
+ return NewAppStore(Args{
+ CookieJar: integrationCookieJar{jar: jar},
+ OperatingSystem: operatingsystem.New(),
+ Machine: machine.New(machine.Args{OS: operatingsystem.New()}),
+ Keychain: keychain.New(keychain.Args{}),
+ })
+}
+
+func TestLiveBagReturnsNativeAuthEndpoint(t *testing.T) {
+ as := newIntegrationAppStore(t)
+
+ out, err := as.Bag(BagInput{})
+ if err != nil {
+ t.Fatalf("Bag: %v", err)
+ }
+
+ if out.AuthEndpoint != defaultNativeAuthEndpoint {
+ t.Fatalf("AuthEndpoint = %q, want %q", out.AuthEndpoint, defaultNativeAuthEndpoint)
+ }
+}
+
+func TestLiveLoginUsesNativeAuthEndpoint(t *testing.T) {
+ as := newIntegrationAppStore(t)
+
+ bag, err := as.Bag(BagInput{})
+ if err != nil {
+ t.Fatalf("Bag: %v", err)
+ }
+
+ _, err = as.Login(LoginInput{
+ Email: "ipatool-integration-test@example.com",
+ Password: "not-a-real-password",
+ Endpoint: bag.AuthEndpoint,
+ })
+ if err == nil {
+ t.Fatal("expected login to fail with invalid credentials")
+ }
+
+ var decodeErr *pkghttp.ResponseDecodeError
+ if errors.As(err, &decodeErr) {
+ t.Fatalf("login returned plist decode error (wrong endpoint or non-plist body): %v", err)
+ }
+
+ if strings.Contains(err.Error(), "failed to unmarshal xml") {
+ t.Fatalf("login hit legacy plist parse failure: %v", err)
+ }
+
+ t.Logf("login failed as expected with invalid credentials: %v", err)
+}
diff --git a/pkg/appstore/appstore_login.go b/pkg/appstore/appstore_login.go
index 419fdeaa..43080d58 100644
--- a/pkg/appstore/appstore_login.go
+++ b/pkg/appstore/appstore_login.go
@@ -65,6 +65,7 @@ type loginResult struct {
func (t *appstore) login(email, password, authCode, guid, endpoint string) (Account, error) {
redirect := ""
+ authEndpoint := normalizeAuthEndpoint(endpoint)
var (
err error
@@ -74,11 +75,17 @@ func (t *appstore) login(email, password, authCode, guid, endpoint string) (Acco
retry := true
for attempt := 1; retry && attempt <= 4; attempt++ {
- request := t.loginRequest(email, password, authCode, guid, endpoint, attempt)
+ request := t.loginRequest(email, password, authCode, guid, authEndpoint, attempt)
request.URL, _ = util.IfEmpty(redirect, request.URL), ""
res, err = t.loginClient.Send(request)
if err != nil {
+ if discoveredEndpoint := authEndpointFromResponseError(err); discoveredEndpoint != "" && discoveredEndpoint != authEndpoint {
+ authEndpoint = discoveredEndpoint
+ redirect = ""
+ continue
+ }
+
return Account{}, fmt.Errorf("request failed: %w", err)
}
@@ -144,6 +151,8 @@ func (t *appstore) parseLoginResponse(res *http.Result[loginResult], attempt int
err = ErrAuthCodeRequired
} else if res.Data.FailureType == "" && res.Data.CustomerMessage == CustomerMessageAccountDisabled {
err = NewErrorWithMetadata(errors.New("account is disabled"), res)
+ } else if res.Data.FailureType == "" && res.Data.CustomerMessage == CustomerMessageActionSignInPage {
+ err = NewErrorWithMetadata(errors.New("account requires browser sign-in (2FA or Apple ID review required)"), res)
} else if res.Data.FailureType != "" {
if res.Data.CustomerMessage != "" {
err = NewErrorWithMetadata(errors.New(res.Data.CustomerMessage), res)
diff --git a/pkg/appstore/appstore_login_test.go b/pkg/appstore/appstore_login_test.go
index 3521eeb1..f0c0ad22 100644
--- a/pkg/appstore/appstore_login_test.go
+++ b/pkg/appstore/appstore_login_test.go
@@ -73,6 +73,9 @@ var _ = Describe("AppStore (Login)", func() {
BeforeEach(func() {
mockClient.EXPECT().
Send(gomock.Any()).
+ Do(func(req http.Request) {
+ Expect(req.URL).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/"))
+ }).
Return(http.Result[loginResult]{}, errors.New(""))
})
@@ -164,6 +167,26 @@ var _ = Describe("AppStore (Login)", func() {
})
})
+ When("store API requires browser sign-in", func() {
+ BeforeEach(func() {
+ mockClient.EXPECT().
+ Send(gomock.Any()).
+ Return(http.Result[loginResult]{
+ Data: loginResult{
+ CustomerMessage: CustomerMessageActionSignInPage,
+ },
+ }, nil)
+ })
+
+ It("returns browser sign-in error", func() {
+ _, err := as.Login(LoginInput{
+ Password: testPassword,
+ })
+ Expect(err).To(HaveOccurred())
+ Expect(err.Error()).To(ContainSubstring("browser sign-in"))
+ })
+ })
+
When("store API redirects", func() {
const (
testRedirectLocation = "https://test-redirect-url.com"
@@ -201,6 +224,32 @@ var _ = Describe("AppStore (Login)", func() {
})
})
+ When("store API response contains a new auth endpoint", func() {
+ BeforeEach(func() {
+ firstCall := mockClient.EXPECT().
+ Send(gomock.Any()).
+ Return(http.Result[loginResult]{}, &http.ResponseDecodeError{
+ Cause: errors.New("decode failed"),
+ URLs: []string{"https://auth.itunes.apple.com/auth/v1/native"},
+ })
+ secondCall := mockClient.EXPECT().
+ Send(gomock.Any()).
+ Do(func(req http.Request) {
+ Expect(req.URL).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/"))
+ }).
+ Return(http.Result[loginResult]{}, errors.New("test complete"))
+ gomock.InOrder(firstCall, secondCall)
+ })
+
+ It("retries with the discovered endpoint", func() {
+ _, err := as.Login(LoginInput{
+ Endpoint: "https://example.com/authenticate",
+ Password: testPassword,
+ })
+ Expect(err).To(MatchError(ContainSubstring("test complete")))
+ })
+ })
+
When("store API redirects too much", func() {
BeforeEach(func() {
mockClient.EXPECT().
diff --git a/pkg/appstore/auth_endpoint.go b/pkg/appstore/auth_endpoint.go
new file mode 100644
index 00000000..6f4f2309
--- /dev/null
+++ b/pkg/appstore/auth_endpoint.go
@@ -0,0 +1,71 @@
+package appstore
+
+import (
+ "errors"
+ "html"
+ "net/url"
+ "strings"
+
+ "github.com/majd/ipatool/v2/pkg/http"
+)
+
+func normalizeAuthEndpoint(endpoints ...string) string {
+ for _, endpoint := range endpoints {
+ endpoint = strings.TrimSpace(endpoint)
+ if endpoint == "" {
+ continue
+ }
+
+ if normalized := normalizeNativeAuthEndpoint(endpoint); normalized != "" {
+ return normalized
+ }
+
+ return endpoint
+ }
+
+ return defaultNativeAuthEndpoint
+}
+
+func authEndpointFromResponseError(err error) string {
+ var decodeErr *http.ResponseDecodeError
+ if !errors.As(err, &decodeErr) {
+ return ""
+ }
+
+ parts := make([]string, 0, len(decodeErr.URLs)+1)
+ parts = append(parts, decodeErr.URLs...)
+ if decodeErr.Body != "" {
+ parts = append(parts, decodeErr.Body)
+ }
+ if len(parts) == 0 {
+ return ""
+ }
+
+ return authEndpointFromText(strings.Join(parts, " "))
+}
+
+func authEndpointFromText(text string) string {
+ text = html.UnescapeString(strings.ReplaceAll(text, `\/`, `/`))
+ for _, match := range http.ExtractURLs([]byte(text)) {
+ if endpoint := normalizeNativeAuthEndpoint(strings.TrimRight(match, ".,;)")); endpoint != "" {
+ return endpoint
+ }
+ }
+
+ return ""
+}
+
+func normalizeNativeAuthEndpoint(endpoint string) string {
+ parsed, err := url.Parse(endpoint)
+ if err != nil || parsed.Host != PrivateAuthDomain {
+ return ""
+ }
+
+ path := strings.TrimRight(parsed.Path, "/")
+ if !strings.HasSuffix(path, "/fast") {
+ path = strings.TrimRight(path, "/") + "/fast"
+ }
+ parsed.Path = path + "/"
+
+ return parsed.String()
+}
diff --git a/pkg/appstore/auth_endpoint_test.go b/pkg/appstore/auth_endpoint_test.go
new file mode 100644
index 00000000..8dfa1cff
--- /dev/null
+++ b/pkg/appstore/auth_endpoint_test.go
@@ -0,0 +1,58 @@
+package appstore
+
+import (
+ "errors"
+
+ "github.com/majd/ipatool/v2/pkg/http"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+var _ = Describe("Auth endpoint", func() {
+ It("falls back to the native auth endpoint", func() {
+ Expect(normalizeAuthEndpoint()).To(Equal(defaultNativeAuthEndpoint))
+ })
+
+ It("normalizes Apple's native auth endpoint with the fast path and trailing slash", func() {
+ Expect(normalizeAuthEndpoint("https://auth.itunes.apple.com/auth/v1/native")).
+ To(Equal(defaultNativeAuthEndpoint))
+ Expect(normalizeAuthEndpoint("https://auth.itunes.apple.com/auth/v1/native/fast")).
+ To(Equal(defaultNativeAuthEndpoint))
+ })
+
+ It("keeps legacy endpoints unchanged", func() {
+ endpoint := "https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate"
+ Expect(normalizeAuthEndpoint(endpoint)).To(Equal(endpoint))
+ })
+
+ It("prefers the first non-empty endpoint", func() {
+ Expect(normalizeAuthEndpoint(
+ "https://auth.itunes.apple.com/auth/v1/native",
+ "https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate",
+ )).To(Equal(defaultNativeAuthEndpoint))
+ })
+
+ It("extracts a native endpoint from an escaped response body", func() {
+ body := `{"authenticateAccount":"https:\/\/auth.itunes.apple.com\/auth\/v1\/native"}`
+ Expect(authEndpointFromText(body)).To(Equal(defaultNativeAuthEndpoint))
+ })
+
+ It("extracts a native endpoint from a response decode error", func() {
+ err := &http.ResponseDecodeError{
+ Cause: errors.New("decode failed"),
+ URLs: []string{"https://auth.itunes.apple.com/auth/v1/native"},
+ }
+ Expect(authEndpointFromResponseError(err)).To(Equal(defaultNativeAuthEndpoint))
+ })
+
+ It("does not mutate URLs on the decode error", func() {
+ urls := []string{"https://auth.itunes.apple.com/auth/v1/native"}
+ err := &http.ResponseDecodeError{
+ Cause: errors.New("decode failed"),
+ URLs: urls,
+ Body: "ignored",
+ }
+ _ = authEndpointFromResponseError(err)
+ Expect(urls).To(Equal([]string{"https://auth.itunes.apple.com/auth/v1/native"}))
+ })
+})
diff --git a/pkg/appstore/constants.go b/pkg/appstore/constants.go
index 66201514..3748154c 100644
--- a/pkg/appstore/constants.go
+++ b/pkg/appstore/constants.go
@@ -13,6 +13,7 @@ const (
CustomerMessageAccountDisabled = "Your account is disabled."
CustomerMessageSubscriptionRequired = "Subscription Required"
CustomerMessagePasswordChanged = "Your password has changed."
+ CustomerMessageActionSignInPage = "AMD-Action::SP"
iTunesAPIDomain = "itunes.apple.com"
iTunesAPIPathSearch = "/search"
@@ -25,6 +26,11 @@ const (
PrivateAppStoreAPIPathPurchase = "/WebObjects/MZFinance.woa/wa/buyProduct"
PrivateAppStoreAPIPathDownload = "/WebObjects/MZFinance.woa/wa/volumeStoreDownloadProduct"
+ PrivateAuthDomain = "auth." + iTunesAPIDomain
+ PrivateAuthPathNative = "/auth/v1/native/fast/"
+
+ defaultNativeAuthEndpoint = "https://" + PrivateAuthDomain + PrivateAuthPathNative
+
HTTPHeaderStoreFront = "X-Set-Apple-Store-Front"
HTTPHeaderPod = "pod"
diff --git a/pkg/http/client.go b/pkg/http/client.go
index 22de26b1..8acb03b2 100644
--- a/pkg/http/client.go
+++ b/pkg/http/client.go
@@ -8,6 +8,7 @@ import (
"net/http"
"regexp"
"strings"
+ "unicode/utf8"
"howett.net/plist"
)
@@ -20,8 +21,27 @@ var (
documentXMLPattern = regexp.MustCompile(`(?is)]*>(.*)`)
plistXMLPattern = regexp.MustCompile(`(?is)]*>.*?`)
dictXMLPattern = regexp.MustCompile(`(?is)]*>.*`)
+ urlPattern = regexp.MustCompile(`https?://[^\s"'<>]+`)
)
+// ResponseDecodeError is returned when an XML/plist response body cannot be decoded.
+// URLs and Body may contain hints (such as an alternate auth endpoint) for callers to inspect.
+type ResponseDecodeError struct {
+ Cause error
+ StatusCode int
+ ContentType string
+ Body string
+ URLs []string
+}
+
+func (e *ResponseDecodeError) Error() string {
+ return fmt.Sprintf("failed to unmarshal xml: %v", e.Cause)
+}
+
+func (e *ResponseDecodeError) Unwrap() error {
+ return e.Cause
+}
+
//go:generate go run go.uber.org/mock/mockgen -source=client.go -destination=client_mock.go -package=http
type Client[R interface{}] interface {
Send(request Request) (Result[R], error)
@@ -170,7 +190,13 @@ func (c *client[R]) handleXMLResponse(res *http.Response) (Result[R], error) {
_, err = plist.Unmarshal(normalizedBody, &data)
if err != nil {
- return Result[R]{}, fmt.Errorf("failed to unmarshal xml: %w", err)
+ return Result[R]{}, &ResponseDecodeError{
+ Cause: err,
+ StatusCode: res.StatusCode,
+ ContentType: res.Header.Get("Content-Type"),
+ Body: truncateBody(body, 500),
+ URLs: ExtractURLs(body),
+ }
}
headers := map[string]string{}
@@ -185,6 +211,30 @@ func (c *client[R]) handleXMLResponse(res *http.Response) (Result[R], error) {
}, nil
}
+// ExtractURLs returns HTTP(S) URLs found in a response body.
+func ExtractURLs(body []byte) []string {
+ matches := urlPattern.FindAll(body, -1)
+ urls := make([]string, 0, len(matches))
+ for _, match := range matches {
+ urls = append(urls, string(match))
+ }
+
+ return urls
+}
+
+func truncateBody(body []byte, max int) string {
+ trimmed := strings.TrimSpace(string(body))
+ if len(trimmed) <= max {
+ return trimmed
+ }
+
+ for max > 0 && !utf8.RuneStart(trimmed[max]) {
+ max--
+ }
+
+ return trimmed[:max] + "..."
+}
+
func normalizeXMLPlistBody(body []byte) []byte {
normalized := bytes.TrimSpace(body)
if len(normalized) == 0 {
diff --git a/pkg/http/client_test.go b/pkg/http/client_test.go
index cb0407c2..b2b65515 100644
--- a/pkg/http/client_test.go
+++ b/pkg/http/client_test.go
@@ -203,7 +203,7 @@ var _ = Describe("Client", Ordered, func() {
})
When("payload fails to decode", func() {
- It("returns error", func() {
+ It("returns error when payload cannot be encoded", func() {
sut := NewClient[xmlResult](Args{
CookieJar: mockCookieJar,
})
@@ -220,5 +220,35 @@ var _ = Describe("Client", Ordered, func() {
Expect(err).To(HaveOccurred())
})
+
+ When("response body is not valid XML", func() {
+ BeforeEach(func() {
+ mockCookieJar.EXPECT().
+ Save().
+ Return(nil)
+ })
+
+ It("returns ResponseDecodeError with extracted URLs", func() {
+ mockHandler = func(w http.ResponseWriter, _r *http.Request) {
+ w.Header().Add("Content-Type", "application/xml")
+ _, err := w.Write([]byte(`redirect to https://auth.itunes.apple.com/auth/v1/native`))
+ Expect(err).ToNot(HaveOccurred())
+ }
+
+ sut := NewClient[xmlResult](Args{
+ CookieJar: mockCookieJar,
+ })
+ _, err := sut.Send(Request{
+ URL: srv.URL,
+ Method: MethodPOST,
+ ResponseFormat: ResponseFormatXML,
+ })
+
+ var decodeErr *ResponseDecodeError
+ Expect(errors.As(err, &decodeErr)).To(BeTrue())
+ Expect(decodeErr.URLs).To(ContainElement("https://auth.itunes.apple.com/auth/v1/native"))
+ Expect(errors.Unwrap(decodeErr)).To(HaveOccurred())
+ })
+ })
})
})