diff --git a/cmd/common.go b/cmd/common.go index 3b3a3dec..112eee34 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -54,11 +54,25 @@ func newLogger(format OutputFormat, verbose bool) log.Logger { // newCookieJar returns a new cookie jar instance. func newCookieJar(machine machine.Machine) http.CookieJar { + cookiePath := filepath.Join(machine.HomeDirectory(), ConfigDirectoryName, CookieJarFileName) + removeStaleZeroByteLock(cookiePath + ".lock") return util.Must(cookiejar.New(&cookiejar.Options{ - Filename: filepath.Join(machine.HomeDirectory(), ConfigDirectoryName, CookieJarFileName), + Filename: cookiePath, })) } +// removeStaleZeroByteLock removes a zero-byte lock file left behind when a process +// is killed between O_TRUNC and writing its PID. The juju/go4/lock library only +// performs PID-based stale detection when the file has content. +func removeStaleZeroByteLock(lockPath string) { + fi, err := os.Stat(lockPath) + if err != nil || fi.Size() != 0 { + return + } + + _ = os.Remove(lockPath) +} + // newKeychain returns a new keychain instance. func newKeychain(machine machine.Machine, logger log.Logger, interactive bool) keychain.Keychain { ring := util.Must(keyring.Open(keyring.Config{ diff --git a/pkg/appstore/appstore_bag.go b/pkg/appstore/appstore_bag.go index 29108a2f..8de570c5 100644 --- a/pkg/appstore/appstore_bag.go +++ b/pkg/appstore/appstore_bag.go @@ -33,12 +33,13 @@ func (t *appstore) Bag(input BagInput) (BagOutput, error) { } return BagOutput{ - AuthEndpoint: res.Data.URLBag.AuthEndpoint, + AuthEndpoint: normalizeAuthEndpoint(res.Data.AuthEndpoint, res.Data.URLBag.AuthEndpoint), }, nil } type bagResult struct { - URLBag urlBag `plist:"urlBag,omitempty"` + URLBag urlBag `plist:"urlBag,omitempty"` + AuthEndpoint string `plist:"authenticateAccount,omitempty"` } type urlBag struct { diff --git a/pkg/appstore/appstore_bag_test.go b/pkg/appstore/appstore_bag_test.go index ae9f1f27..15d94aed 100644 --- a/pkg/appstore/appstore_bag_test.go +++ b/pkg/appstore/appstore_bag_test.go @@ -118,6 +118,29 @@ var _ = Describe("AppStore (Bag)", func() { }) }) + When("request is successful with authenticateAccount at the root", func() { + BeforeEach(func() { + mockMachine.EXPECT(). + MacAddress(). + Return("aa:bb:cc:dd:ee:ff", nil) + + mockBagClient.EXPECT(). + Send(gomock.Any()). + Return(http.Result[bagResult]{ + StatusCode: gohttp.StatusOK, + Data: bagResult{ + AuthEndpoint: "https://auth.itunes.apple.com/auth/v1/native", + }, + }, nil) + }) + + It("returns the normalized native auth endpoint", func() { + out, err := as.Bag(BagInput{}) + Expect(err).ToNot(HaveOccurred()) + Expect(out.AuthEndpoint).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/")) + }) + }) + When("request is successful but authenticateAccount is empty", func() { BeforeEach(func() { mockMachine.EXPECT(). @@ -132,10 +155,10 @@ var _ = Describe("AppStore (Bag)", func() { }, nil) }) - It("returns empty auth endpoint", func() { + It("returns the fallback auth endpoint", func() { out, err := as.Bag(BagInput{}) Expect(err).ToNot(HaveOccurred()) - Expect(out.AuthEndpoint).To(BeEmpty()) + Expect(out.AuthEndpoint).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/")) }) }) }) diff --git a/pkg/appstore/appstore_live_test.go b/pkg/appstore/appstore_live_test.go new file mode 100644 index 00000000..2d3429b0 --- /dev/null +++ b/pkg/appstore/appstore_live_test.go @@ -0,0 +1,89 @@ +//go:build integration + +package appstore + +import ( + "errors" + "net/http" + "net/http/cookiejar" + "net/url" + "strings" + "testing" + + pkghttp "github.com/majd/ipatool/v2/pkg/http" + "github.com/majd/ipatool/v2/pkg/keychain" + "github.com/majd/ipatool/v2/pkg/util/machine" + "github.com/majd/ipatool/v2/pkg/util/operatingsystem" +) + +type integrationCookieJar struct { + jar *cookiejar.Jar +} + +func (c integrationCookieJar) SetCookies(u *url.URL, cookies []*http.Cookie) { + c.jar.SetCookies(u, cookies) +} + +func (c integrationCookieJar) Cookies(u *url.URL) []*http.Cookie { + return c.jar.Cookies(u) +} + +func (integrationCookieJar) Save() error { return nil } + +func newIntegrationAppStore(t *testing.T) AppStore { + t.Helper() + + jar, err := cookiejar.New(nil) + if err != nil { + t.Fatalf("cookie jar: %v", err) + } + + return NewAppStore(Args{ + CookieJar: integrationCookieJar{jar: jar}, + OperatingSystem: operatingsystem.New(), + Machine: machine.New(machine.Args{OS: operatingsystem.New()}), + Keychain: keychain.New(keychain.Args{}), + }) +} + +func TestLiveBagReturnsNativeAuthEndpoint(t *testing.T) { + as := newIntegrationAppStore(t) + + out, err := as.Bag(BagInput{}) + if err != nil { + t.Fatalf("Bag: %v", err) + } + + if out.AuthEndpoint != defaultNativeAuthEndpoint { + t.Fatalf("AuthEndpoint = %q, want %q", out.AuthEndpoint, defaultNativeAuthEndpoint) + } +} + +func TestLiveLoginUsesNativeAuthEndpoint(t *testing.T) { + as := newIntegrationAppStore(t) + + bag, err := as.Bag(BagInput{}) + if err != nil { + t.Fatalf("Bag: %v", err) + } + + _, err = as.Login(LoginInput{ + Email: "ipatool-integration-test@example.com", + Password: "not-a-real-password", + Endpoint: bag.AuthEndpoint, + }) + if err == nil { + t.Fatal("expected login to fail with invalid credentials") + } + + var decodeErr *pkghttp.ResponseDecodeError + if errors.As(err, &decodeErr) { + t.Fatalf("login returned plist decode error (wrong endpoint or non-plist body): %v", err) + } + + if strings.Contains(err.Error(), "failed to unmarshal xml") { + t.Fatalf("login hit legacy plist parse failure: %v", err) + } + + t.Logf("login failed as expected with invalid credentials: %v", err) +} diff --git a/pkg/appstore/appstore_login.go b/pkg/appstore/appstore_login.go index 419fdeaa..43080d58 100644 --- a/pkg/appstore/appstore_login.go +++ b/pkg/appstore/appstore_login.go @@ -65,6 +65,7 @@ type loginResult struct { func (t *appstore) login(email, password, authCode, guid, endpoint string) (Account, error) { redirect := "" + authEndpoint := normalizeAuthEndpoint(endpoint) var ( err error @@ -74,11 +75,17 @@ func (t *appstore) login(email, password, authCode, guid, endpoint string) (Acco retry := true for attempt := 1; retry && attempt <= 4; attempt++ { - request := t.loginRequest(email, password, authCode, guid, endpoint, attempt) + request := t.loginRequest(email, password, authCode, guid, authEndpoint, attempt) request.URL, _ = util.IfEmpty(redirect, request.URL), "" res, err = t.loginClient.Send(request) if err != nil { + if discoveredEndpoint := authEndpointFromResponseError(err); discoveredEndpoint != "" && discoveredEndpoint != authEndpoint { + authEndpoint = discoveredEndpoint + redirect = "" + continue + } + return Account{}, fmt.Errorf("request failed: %w", err) } @@ -144,6 +151,8 @@ func (t *appstore) parseLoginResponse(res *http.Result[loginResult], attempt int err = ErrAuthCodeRequired } else if res.Data.FailureType == "" && res.Data.CustomerMessage == CustomerMessageAccountDisabled { err = NewErrorWithMetadata(errors.New("account is disabled"), res) + } else if res.Data.FailureType == "" && res.Data.CustomerMessage == CustomerMessageActionSignInPage { + err = NewErrorWithMetadata(errors.New("account requires browser sign-in (2FA or Apple ID review required)"), res) } else if res.Data.FailureType != "" { if res.Data.CustomerMessage != "" { err = NewErrorWithMetadata(errors.New(res.Data.CustomerMessage), res) diff --git a/pkg/appstore/appstore_login_test.go b/pkg/appstore/appstore_login_test.go index 3521eeb1..f0c0ad22 100644 --- a/pkg/appstore/appstore_login_test.go +++ b/pkg/appstore/appstore_login_test.go @@ -73,6 +73,9 @@ var _ = Describe("AppStore (Login)", func() { BeforeEach(func() { mockClient.EXPECT(). Send(gomock.Any()). + Do(func(req http.Request) { + Expect(req.URL).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/")) + }). Return(http.Result[loginResult]{}, errors.New("")) }) @@ -164,6 +167,26 @@ var _ = Describe("AppStore (Login)", func() { }) }) + When("store API requires browser sign-in", func() { + BeforeEach(func() { + mockClient.EXPECT(). + Send(gomock.Any()). + Return(http.Result[loginResult]{ + Data: loginResult{ + CustomerMessage: CustomerMessageActionSignInPage, + }, + }, nil) + }) + + It("returns browser sign-in error", func() { + _, err := as.Login(LoginInput{ + Password: testPassword, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("browser sign-in")) + }) + }) + When("store API redirects", func() { const ( testRedirectLocation = "https://test-redirect-url.com" @@ -201,6 +224,32 @@ var _ = Describe("AppStore (Login)", func() { }) }) + When("store API response contains a new auth endpoint", func() { + BeforeEach(func() { + firstCall := mockClient.EXPECT(). + Send(gomock.Any()). + Return(http.Result[loginResult]{}, &http.ResponseDecodeError{ + Cause: errors.New("decode failed"), + URLs: []string{"https://auth.itunes.apple.com/auth/v1/native"}, + }) + secondCall := mockClient.EXPECT(). + Send(gomock.Any()). + Do(func(req http.Request) { + Expect(req.URL).To(Equal("https://auth.itunes.apple.com/auth/v1/native/fast/")) + }). + Return(http.Result[loginResult]{}, errors.New("test complete")) + gomock.InOrder(firstCall, secondCall) + }) + + It("retries with the discovered endpoint", func() { + _, err := as.Login(LoginInput{ + Endpoint: "https://example.com/authenticate", + Password: testPassword, + }) + Expect(err).To(MatchError(ContainSubstring("test complete"))) + }) + }) + When("store API redirects too much", func() { BeforeEach(func() { mockClient.EXPECT(). diff --git a/pkg/appstore/auth_endpoint.go b/pkg/appstore/auth_endpoint.go new file mode 100644 index 00000000..6f4f2309 --- /dev/null +++ b/pkg/appstore/auth_endpoint.go @@ -0,0 +1,71 @@ +package appstore + +import ( + "errors" + "html" + "net/url" + "strings" + + "github.com/majd/ipatool/v2/pkg/http" +) + +func normalizeAuthEndpoint(endpoints ...string) string { + for _, endpoint := range endpoints { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + continue + } + + if normalized := normalizeNativeAuthEndpoint(endpoint); normalized != "" { + return normalized + } + + return endpoint + } + + return defaultNativeAuthEndpoint +} + +func authEndpointFromResponseError(err error) string { + var decodeErr *http.ResponseDecodeError + if !errors.As(err, &decodeErr) { + return "" + } + + parts := make([]string, 0, len(decodeErr.URLs)+1) + parts = append(parts, decodeErr.URLs...) + if decodeErr.Body != "" { + parts = append(parts, decodeErr.Body) + } + if len(parts) == 0 { + return "" + } + + return authEndpointFromText(strings.Join(parts, " ")) +} + +func authEndpointFromText(text string) string { + text = html.UnescapeString(strings.ReplaceAll(text, `\/`, `/`)) + for _, match := range http.ExtractURLs([]byte(text)) { + if endpoint := normalizeNativeAuthEndpoint(strings.TrimRight(match, ".,;)")); endpoint != "" { + return endpoint + } + } + + return "" +} + +func normalizeNativeAuthEndpoint(endpoint string) string { + parsed, err := url.Parse(endpoint) + if err != nil || parsed.Host != PrivateAuthDomain { + return "" + } + + path := strings.TrimRight(parsed.Path, "/") + if !strings.HasSuffix(path, "/fast") { + path = strings.TrimRight(path, "/") + "/fast" + } + parsed.Path = path + "/" + + return parsed.String() +} diff --git a/pkg/appstore/auth_endpoint_test.go b/pkg/appstore/auth_endpoint_test.go new file mode 100644 index 00000000..8dfa1cff --- /dev/null +++ b/pkg/appstore/auth_endpoint_test.go @@ -0,0 +1,58 @@ +package appstore + +import ( + "errors" + + "github.com/majd/ipatool/v2/pkg/http" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Auth endpoint", func() { + It("falls back to the native auth endpoint", func() { + Expect(normalizeAuthEndpoint()).To(Equal(defaultNativeAuthEndpoint)) + }) + + It("normalizes Apple's native auth endpoint with the fast path and trailing slash", func() { + Expect(normalizeAuthEndpoint("https://auth.itunes.apple.com/auth/v1/native")). + To(Equal(defaultNativeAuthEndpoint)) + Expect(normalizeAuthEndpoint("https://auth.itunes.apple.com/auth/v1/native/fast")). + To(Equal(defaultNativeAuthEndpoint)) + }) + + It("keeps legacy endpoints unchanged", func() { + endpoint := "https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate" + Expect(normalizeAuthEndpoint(endpoint)).To(Equal(endpoint)) + }) + + It("prefers the first non-empty endpoint", func() { + Expect(normalizeAuthEndpoint( + "https://auth.itunes.apple.com/auth/v1/native", + "https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate", + )).To(Equal(defaultNativeAuthEndpoint)) + }) + + It("extracts a native endpoint from an escaped response body", func() { + body := `{"authenticateAccount":"https:\/\/auth.itunes.apple.com\/auth\/v1\/native"}` + Expect(authEndpointFromText(body)).To(Equal(defaultNativeAuthEndpoint)) + }) + + It("extracts a native endpoint from a response decode error", func() { + err := &http.ResponseDecodeError{ + Cause: errors.New("decode failed"), + URLs: []string{"https://auth.itunes.apple.com/auth/v1/native"}, + } + Expect(authEndpointFromResponseError(err)).To(Equal(defaultNativeAuthEndpoint)) + }) + + It("does not mutate URLs on the decode error", func() { + urls := []string{"https://auth.itunes.apple.com/auth/v1/native"} + err := &http.ResponseDecodeError{ + Cause: errors.New("decode failed"), + URLs: urls, + Body: "ignored", + } + _ = authEndpointFromResponseError(err) + Expect(urls).To(Equal([]string{"https://auth.itunes.apple.com/auth/v1/native"})) + }) +}) diff --git a/pkg/appstore/constants.go b/pkg/appstore/constants.go index 66201514..3748154c 100644 --- a/pkg/appstore/constants.go +++ b/pkg/appstore/constants.go @@ -13,6 +13,7 @@ const ( CustomerMessageAccountDisabled = "Your account is disabled." CustomerMessageSubscriptionRequired = "Subscription Required" CustomerMessagePasswordChanged = "Your password has changed." + CustomerMessageActionSignInPage = "AMD-Action::SP" iTunesAPIDomain = "itunes.apple.com" iTunesAPIPathSearch = "/search" @@ -25,6 +26,11 @@ const ( PrivateAppStoreAPIPathPurchase = "/WebObjects/MZFinance.woa/wa/buyProduct" PrivateAppStoreAPIPathDownload = "/WebObjects/MZFinance.woa/wa/volumeStoreDownloadProduct" + PrivateAuthDomain = "auth." + iTunesAPIDomain + PrivateAuthPathNative = "/auth/v1/native/fast/" + + defaultNativeAuthEndpoint = "https://" + PrivateAuthDomain + PrivateAuthPathNative + HTTPHeaderStoreFront = "X-Set-Apple-Store-Front" HTTPHeaderPod = "pod" diff --git a/pkg/http/client.go b/pkg/http/client.go index 22de26b1..8acb03b2 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -8,6 +8,7 @@ import ( "net/http" "regexp" "strings" + "unicode/utf8" "howett.net/plist" ) @@ -20,8 +21,27 @@ var ( documentXMLPattern = regexp.MustCompile(`(?is)]*>(.*)`) plistXMLPattern = regexp.MustCompile(`(?is)]*>.*?`) dictXMLPattern = regexp.MustCompile(`(?is)]*>.*`) + urlPattern = regexp.MustCompile(`https?://[^\s"'<>]+`) ) +// ResponseDecodeError is returned when an XML/plist response body cannot be decoded. +// URLs and Body may contain hints (such as an alternate auth endpoint) for callers to inspect. +type ResponseDecodeError struct { + Cause error + StatusCode int + ContentType string + Body string + URLs []string +} + +func (e *ResponseDecodeError) Error() string { + return fmt.Sprintf("failed to unmarshal xml: %v", e.Cause) +} + +func (e *ResponseDecodeError) Unwrap() error { + return e.Cause +} + //go:generate go run go.uber.org/mock/mockgen -source=client.go -destination=client_mock.go -package=http type Client[R interface{}] interface { Send(request Request) (Result[R], error) @@ -170,7 +190,13 @@ func (c *client[R]) handleXMLResponse(res *http.Response) (Result[R], error) { _, err = plist.Unmarshal(normalizedBody, &data) if err != nil { - return Result[R]{}, fmt.Errorf("failed to unmarshal xml: %w", err) + return Result[R]{}, &ResponseDecodeError{ + Cause: err, + StatusCode: res.StatusCode, + ContentType: res.Header.Get("Content-Type"), + Body: truncateBody(body, 500), + URLs: ExtractURLs(body), + } } headers := map[string]string{} @@ -185,6 +211,30 @@ func (c *client[R]) handleXMLResponse(res *http.Response) (Result[R], error) { }, nil } +// ExtractURLs returns HTTP(S) URLs found in a response body. +func ExtractURLs(body []byte) []string { + matches := urlPattern.FindAll(body, -1) + urls := make([]string, 0, len(matches)) + for _, match := range matches { + urls = append(urls, string(match)) + } + + return urls +} + +func truncateBody(body []byte, max int) string { + trimmed := strings.TrimSpace(string(body)) + if len(trimmed) <= max { + return trimmed + } + + for max > 0 && !utf8.RuneStart(trimmed[max]) { + max-- + } + + return trimmed[:max] + "..." +} + func normalizeXMLPlistBody(body []byte) []byte { normalized := bytes.TrimSpace(body) if len(normalized) == 0 { diff --git a/pkg/http/client_test.go b/pkg/http/client_test.go index cb0407c2..b2b65515 100644 --- a/pkg/http/client_test.go +++ b/pkg/http/client_test.go @@ -203,7 +203,7 @@ var _ = Describe("Client", Ordered, func() { }) When("payload fails to decode", func() { - It("returns error", func() { + It("returns error when payload cannot be encoded", func() { sut := NewClient[xmlResult](Args{ CookieJar: mockCookieJar, }) @@ -220,5 +220,35 @@ var _ = Describe("Client", Ordered, func() { Expect(err).To(HaveOccurred()) }) + + When("response body is not valid XML", func() { + BeforeEach(func() { + mockCookieJar.EXPECT(). + Save(). + Return(nil) + }) + + It("returns ResponseDecodeError with extracted URLs", func() { + mockHandler = func(w http.ResponseWriter, _r *http.Request) { + w.Header().Add("Content-Type", "application/xml") + _, err := w.Write([]byte(`redirect to https://auth.itunes.apple.com/auth/v1/native`)) + Expect(err).ToNot(HaveOccurred()) + } + + sut := NewClient[xmlResult](Args{ + CookieJar: mockCookieJar, + }) + _, err := sut.Send(Request{ + URL: srv.URL, + Method: MethodPOST, + ResponseFormat: ResponseFormatXML, + }) + + var decodeErr *ResponseDecodeError + Expect(errors.As(err, &decodeErr)).To(BeTrue()) + Expect(decodeErr.URLs).To(ContainElement("https://auth.itunes.apple.com/auth/v1/native")) + Expect(errors.Unwrap(decodeErr)).To(HaveOccurred()) + }) + }) }) })