diff --git a/packages/network-controller/CHANGELOG.md b/packages/network-controller/CHANGELOG.md index fcff6929fd..2e3ebbb0b1 100644 --- a/packages/network-controller/CHANGELOG.md +++ b/packages/network-controller/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The constructor argument `isRpcFailoverEnabled` is no longer available. - `RemoteFeatureFlagController:stateChange` and `RemoteFeatureFlagController:getState` are now required. - Drop `async-mutex` dependency, which was no longer used in source ([#9064](https://github.com/MetaMask/core/pull/9064)) +- Only consider failover endpoints when using Infura ([#9125](https://github.com/MetaMask/core/pull/9125)) ### Removed diff --git a/packages/network-controller/src/create-network-client-tests/rpc-endpoint-events.test.ts b/packages/network-controller/src/create-network-client-tests/rpc-endpoint-events.test.ts index 868048a6da..fa1c5587f7 100644 --- a/packages/network-controller/src/create-network-client-tests/rpc-endpoint-events.test.ts +++ b/packages/network-controller/src/create-network-client-tests/rpc-endpoint-events.test.ts @@ -20,1078 +20,1080 @@ describe('createNetworkClient - RPC endpoint events', () => { const blockNumber = '0x100'; const backoffDuration = 100; - describe('with RPC failover', () => { - it('publishes the NetworkController:rpcEndpointChainUnavailable event only when the max number of consecutive request failures is reached for all of the endpoints in a chain of endpoints', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; - const request = { - method: 'eth_gasPrice', - params: [], - }; - const expectedError = createResourceUnavailableError(503); - const expectedUnavailableError = new HttpError(503); - - await withMockedCommunications( - { providerType: networkClientType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, - }, - async (failoverComms) => { - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: DEFAULT_MAX_CONSECUTIVE_FAILURES, - response: { - httpStatus: 503, - }, - }); - failoverComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: DEFAULT_MAX_CONSECUTIVE_FAILURES, - response: { - httpStatus: 503, - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointChainUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointChainUnavailable', - rpcEndpointChainUnavailableEventHandler, - ); - - await withNetworkClient( - { - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - providerType: networkClientType, - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - isOffline: (): boolean => false, - }), - }, - async ({ makeRpcCall, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - jest.advanceTimersByTime(backoffDuration); - }, - ); - - // Hit the primary and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the primary and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the primary and exceed the max number of retries, - // breaking the circuit; then hit the failover and exceed - // the max of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the failover and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the failover and exceed the max number of retries, - // breaking the circuit - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - - expect( - rpcEndpointChainUnavailableEventHandler, - ).toHaveBeenCalledTimes(1); - expect( - rpcEndpointChainUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - error: expectedUnavailableError, + if (networkClientType !== NetworkClientType.Custom) { + describe('with RPC failover', () => { + it('publishes the NetworkController:rpcEndpointChainUnavailable event only when the max number of consecutive request failures is reached for all of the endpoints in a chain of endpoints', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + const request = { + method: 'eth_gasPrice', + params: [], + }; + const expectedError = createResourceUnavailableError(503); + const expectedUnavailableError = new HttpError(503); + + await withMockedCommunications( + { providerType: networkClientType }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async (failoverComms) => { + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: DEFAULT_MAX_CONSECUTIVE_FAILURES, + response: { + httpStatus: 503, + }, + }); + failoverComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: DEFAULT_MAX_CONSECUTIVE_FAILURES, + response: { + httpStatus: 503, + }, + }); + + const messenger = buildRootMessenger(); + const rpcEndpointChainUnavailableEventHandler = jest.fn(); + messenger.subscribe( + 'NetworkController:rpcEndpointChainUnavailable', + rpcEndpointChainUnavailableEventHandler, + ); + + await withNetworkClient( + { networkClientId: 'AAAA-AAAA-AAAA-AAAA', - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event each time the max number of consecutive request failures is reached for any of the endpoints in a chain of endpoints', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; - const request = { - method: 'eth_gasPrice', - params: [], - }; - const expectedError = createResourceUnavailableError(503); - const expectedUnavailableError = new HttpError(503); - - await withMockedCommunications( - { providerType: networkClientType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, - }, - async (failoverComms) => { - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: DEFAULT_MAX_CONSECUTIVE_FAILURES, - response: { - httpStatus: 503, - }, - }); - failoverComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: DEFAULT_MAX_CONSECUTIVE_FAILURES, - response: { - httpStatus: 503, - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType: networkClientType, - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - isOffline: (): boolean => false, - }), - }, - async ({ makeRpcCall, chainId, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - jest.advanceTimersByTime(backoffDuration); - }, - ); - - // Hit the primary and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the primary and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the primary and exceed the max number of retries, - // breaking the circuit; then hit the failover and exceed - // the max of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the failover and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the failover and exceed the max number of retries, - // breaking the circuit - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledTimes(2); - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - error: expectedUnavailableError, + providerType: networkClientType, + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + isOffline: (): boolean => false, + }), + }, + async ({ makeRpcCall, chainId }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + jest.advanceTimersByTime(backoffDuration); + }, + ); + + // Hit the primary and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the primary and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the primary and exceed the max number of retries, + // breaking the circuit; then hit the failover and exceed + // the max of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the failover and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the failover and exceed the max number of retries, + // breaking the circuit + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + + expect( + rpcEndpointChainUnavailableEventHandler, + ).toHaveBeenCalledTimes(1); + expect( + rpcEndpointChainUnavailableEventHandler, + ).toHaveBeenCalledWith({ + chainId, + error: expectedUnavailableError, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + }); + }, + ); + }, + ); + }, + ); + }); + + it('publishes the NetworkController:rpcEndpointUnavailable event each time the max number of consecutive request failures is reached for any of the endpoints in a chain of endpoints', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + const request = { + method: 'eth_gasPrice', + params: [], + }; + const expectedError = createResourceUnavailableError(503); + const expectedUnavailableError = new HttpError(503); + + await withMockedCommunications( + { providerType: networkClientType }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async (failoverComms) => { + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: DEFAULT_MAX_CONSECUTIVE_FAILURES, + response: { + httpStatus: 503, + }, + }); + failoverComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: DEFAULT_MAX_CONSECUTIVE_FAILURES, + response: { + httpStatus: 503, + }, + }); + + const messenger = buildRootMessenger(); + const rpcEndpointUnavailableEventHandler = jest.fn(); + messenger.subscribe( + 'NetworkController:rpcEndpointUnavailable', + rpcEndpointUnavailableEventHandler, + ); + + await withNetworkClient( + { + providerType: networkClientType, networkClientId: 'AAAA-AAAA-AAAA-AAAA', - primaryEndpointUrl: rpcUrl, - }); - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: failoverEndpointUrl, - error: expectedUnavailableError, + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + isOffline: (): boolean => false, + }), + }, + async ({ makeRpcCall, chainId, rpcUrl }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + jest.advanceTimersByTime(backoffDuration); + }, + ); + + // Hit the primary and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the primary and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the primary and exceed the max number of retries, + // breaking the circuit; then hit the failover and exceed + // the max of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the failover and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the failover and exceed the max number of retries, + // breaking the circuit + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + + expect( + rpcEndpointUnavailableEventHandler, + ).toHaveBeenCalledTimes(2); + expect( + rpcEndpointUnavailableEventHandler, + ).toHaveBeenCalledWith({ + chainId, + endpointUrl: rpcUrl, + error: expectedUnavailableError, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + primaryEndpointUrl: rpcUrl, + }); + expect( + rpcEndpointUnavailableEventHandler, + ).toHaveBeenCalledWith({ + chainId, + endpointUrl: failoverEndpointUrl, + error: expectedUnavailableError, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + primaryEndpointUrl: rpcUrl, + }); + }, + ); + }, + ); + }, + ); + }); + + it('does not retry requests when user is offline', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + const request = { + method: 'eth_gasPrice', + params: [], + }; + const expectedError = createResourceUnavailableError(503); + + await withMockedCommunications( + { providerType: networkClientType }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async () => { + // Mock only one failure - if retries were happening, we'd need more + primaryComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: 1, + response: { + httpStatus: 503, + }, + }); + + const rootMessenger = buildRootMessenger({ + connectivityStatus: CONNECTIVITY_STATUSES.Offline, + }); + + const rpcEndpointRetriedEventHandler = jest.fn(); + rootMessenger.subscribe( + 'NetworkController:rpcEndpointRetried', + rpcEndpointRetriedEventHandler, + ); + + await withNetworkClient( + { + providerType: networkClientType, networkClientId: 'AAAA-AAAA-AAAA-AAAA', - primaryEndpointUrl: rpcUrl, - }); - }, - ); - }, - ); - }, - ); - }); - - it('does not retry requests when user is offline', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; - const request = { - method: 'eth_gasPrice', - params: [], - }; - const expectedError = createResourceUnavailableError(503); - - await withMockedCommunications( - { providerType: networkClientType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, - }, - async () => { - // Mock only one failure - if retries were happening, we'd need more - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: 1, - response: { - httpStatus: 503, - }, - }); - - const rootMessenger = buildRootMessenger({ - connectivityStatus: CONNECTIVITY_STATUSES.Offline, - }); - - const rpcEndpointRetriedEventHandler = jest.fn(); - rootMessenger.subscribe( - 'NetworkController:rpcEndpointRetried', - rpcEndpointRetriedEventHandler, - ); - - await withNetworkClient( - { - providerType: networkClientType, - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger: rootMessenger, - getRpcServiceOptions: () => ({ - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall }) => { - // When offline, errors are not retried, so the request - // should fail immediately without retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - - // Verify that retry event was not published - expect( - rpcEndpointRetriedEventHandler, - ).not.toHaveBeenCalled(); - }, - ); - }, - ); - }, - ); - }); - - it('suppresses the NetworkController:rpcEndpointUnavailable event when user is offline', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; - const request = { - method: 'eth_gasPrice', - params: [], - }; - const expectedError = createResourceUnavailableError(503); - - await withMockedCommunications( - { providerType: networkClientType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, - }, - async (failoverComms) => { - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: DEFAULT_MAX_CONSECUTIVE_FAILURES, - response: { - httpStatus: 503, - }, - }); - failoverComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: DEFAULT_MAX_CONSECUTIVE_FAILURES, - response: { - httpStatus: 503, - }, - }); - - const rootMessenger = buildRootMessenger({ - connectivityStatus: CONNECTIVITY_STATUSES.Offline, - }); - - const rpcEndpointUnavailableEventHandler = jest.fn(); - rootMessenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType: networkClientType, - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger: rootMessenger, - getRpcServiceOptions: () => ({ - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall }) => { - rootMessenger.subscribe( - 'NetworkController:rpcEndpointRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - jest.advanceTimersByTime(backoffDuration); - }, - ); - - // When offline, errors are not retried, so the circuit - // won't break and onServiceBreak won't be called - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - - // Event should be suppressed when offline because retries - // are prevented, so onServiceBreak is never called - expect( - rpcEndpointUnavailableEventHandler, - ).not.toHaveBeenCalled(); - }, - ); - }, - ); - }, - ); - }); - - it('does not publish the NetworkController:rpcEndpointChainDegraded event again if the max number of retries is reached in making requests to a failover endpoint', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; - const request = { - method: 'eth_gasPrice', - params: [], - }; - const expectedError = createResourceUnavailableError(503); - const expectedDegradedError = new HttpError(503); - - await withMockedCommunications( - { providerType: networkClientType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, - }, - async (failoverComms) => { - const messenger = buildRootMessenger(); - const rpcEndpointChainDegradedEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointChainDegraded', - rpcEndpointChainDegradedEventHandler, - ); - - await withNetworkClient( - { - providerType: networkClientType, - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - isOffline: (): boolean => false, - }), - }, - async ({ makeRpcCall, chainId }) => { - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: DEFAULT_MAX_CONSECUTIVE_FAILURES, - response: { - httpStatus: 503, - }, - }); - failoverComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: 5, - response: { - httpStatus: 503, - }, - }); - - messenger.subscribe( - 'NetworkController:rpcEndpointRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - jest.advanceTimersByTime(backoffDuration); - }, - ); - - // Hit the primary and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the primary and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the primary and exceed the max number of retries, - // break the circuit; hit the failover and exceed the max - // number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - - expect( - rpcEndpointChainDegradedEventHandler, - ).toHaveBeenCalledTimes(1); - expect( - rpcEndpointChainDegradedEventHandler, - ).toHaveBeenCalledWith({ - chainId, - type: 'retries_exhausted', - error: expectedDegradedError, + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger: rootMessenger, + getRpcServiceOptions: () => ({ + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + }), + }, + async ({ makeRpcCall }) => { + // When offline, errors are not retried, so the request + // should fail immediately without retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + + // Verify that retry event was not published + expect( + rpcEndpointRetriedEventHandler, + ).not.toHaveBeenCalled(); + }, + ); + }, + ); + }, + ); + }); + + it('suppresses the NetworkController:rpcEndpointUnavailable event when user is offline', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + const request = { + method: 'eth_gasPrice', + params: [], + }; + const expectedError = createResourceUnavailableError(503); + + await withMockedCommunications( + { providerType: networkClientType }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async (failoverComms) => { + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: DEFAULT_MAX_CONSECUTIVE_FAILURES, + response: { + httpStatus: 503, + }, + }); + failoverComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: DEFAULT_MAX_CONSECUTIVE_FAILURES, + response: { + httpStatus: 503, + }, + }); + + const rootMessenger = buildRootMessenger({ + connectivityStatus: CONNECTIVITY_STATUSES.Offline, + }); + + const rpcEndpointUnavailableEventHandler = jest.fn(); + rootMessenger.subscribe( + 'NetworkController:rpcEndpointUnavailable', + rpcEndpointUnavailableEventHandler, + ); + + await withNetworkClient( + { + providerType: networkClientType, networkClientId: 'AAAA-AAAA-AAAA-AAAA', - retryReason: 'non_successful_http_status', - rpcMethodName: 'eth_blockNumber', - duration: undefined, - traceId: undefined, - }); - }, - ); - }, - ); - }, - ); - }); - - it('does not publish the NetworkController:rpcEndpointChainDegraded event again when the time to complete a request to a failover endpoint is too long', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; - const request = { - method: 'eth_gasPrice', - params: [], - }; - const expectedError = createResourceUnavailableError(503); - const expectedDegradedError = new HttpError(503); - - await withMockedCommunications( - { providerType: networkClientType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, - }, - async (failoverComms) => { - const messenger = buildRootMessenger(); - const rpcEndpointChainDegradedEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointChainDegraded', - rpcEndpointChainDegradedEventHandler, - ); - - await withNetworkClient( - { - providerType: networkClientType, - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - isOffline: (): boolean => false, - }), - }, - async ({ makeRpcCall, chainId }) => { - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: DEFAULT_MAX_CONSECUTIVE_FAILURES, - response: { - httpStatus: 503, - }, - }); - failoverComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: () => { - jest.advanceTimersByTime( - DEFAULT_DEGRADED_THRESHOLD + 1, - ); - return { - result: '0x1', - }; - }, - }); - failoverComms.mockRpcCall({ - request, - response: () => { - jest.advanceTimersByTime( - DEFAULT_DEGRADED_THRESHOLD + 1, - ); - return { - result: 'ok', - }; - }, - }); - - messenger.subscribe( - 'NetworkController:rpcEndpointRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - jest.advanceTimersByTime(backoffDuration); - }, - ); - - // Hit the primary and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the primary and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the primary and exceed the max number of retries, - // break the circuit; hit the failover - await makeRpcCall(request); - - expect( - rpcEndpointChainDegradedEventHandler, - ).toHaveBeenCalledTimes(1); - expect( - rpcEndpointChainDegradedEventHandler, - ).toHaveBeenCalledWith({ - chainId, - type: 'retries_exhausted', - error: expectedDegradedError, + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger: rootMessenger, + getRpcServiceOptions: () => ({ + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + }), + }, + async ({ makeRpcCall }) => { + rootMessenger.subscribe( + 'NetworkController:rpcEndpointRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + jest.advanceTimersByTime(backoffDuration); + }, + ); + + // When offline, errors are not retried, so the circuit + // won't break and onServiceBreak won't be called + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + + // Event should be suppressed when offline because retries + // are prevented, so onServiceBreak is never called + expect( + rpcEndpointUnavailableEventHandler, + ).not.toHaveBeenCalled(); + }, + ); + }, + ); + }, + ); + }); + + it('does not publish the NetworkController:rpcEndpointChainDegraded event again if the max number of retries is reached in making requests to a failover endpoint', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + const request = { + method: 'eth_gasPrice', + params: [], + }; + const expectedError = createResourceUnavailableError(503); + const expectedDegradedError = new HttpError(503); + + await withMockedCommunications( + { providerType: networkClientType }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async (failoverComms) => { + const messenger = buildRootMessenger(); + const rpcEndpointChainDegradedEventHandler = jest.fn(); + messenger.subscribe( + 'NetworkController:rpcEndpointChainDegraded', + rpcEndpointChainDegradedEventHandler, + ); + + await withNetworkClient( + { + providerType: networkClientType, networkClientId: 'AAAA-AAAA-AAAA-AAAA', - retryReason: 'non_successful_http_status', - rpcMethodName: 'eth_blockNumber', - duration: undefined, - traceId: undefined, - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointDegraded event again if the max number of retries is reached in making requests to a failover endpoint', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; - const request = { - method: 'eth_gasPrice', - params: [], - }; - const expectedError = createResourceUnavailableError(503); - const expectedDegradedError = new HttpError(503); - - await withMockedCommunications( - { providerType: networkClientType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, - }, - async (failoverComms) => { - const messenger = buildRootMessenger(); - const rpcEndpointDegradedEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointDegraded', - rpcEndpointDegradedEventHandler, - ); - - await withNetworkClient( - { - providerType: networkClientType, - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - isOffline: (): boolean => false, - }), - }, - async ({ makeRpcCall, chainId, rpcUrl }) => { - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: DEFAULT_MAX_CONSECUTIVE_FAILURES, - response: { - httpStatus: 503, - }, - }); - failoverComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: 5, - response: { - httpStatus: 503, - }, - }); - - messenger.subscribe( - 'NetworkController:rpcEndpointRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - jest.advanceTimersByTime(backoffDuration); - }, - ); - - // Hit the primary and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the primary and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the primary and exceed the max number of retries, - // break the circuit; hit the failover and exceed the max - // number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - - expect( - rpcEndpointDegradedEventHandler, - ).toHaveBeenCalledTimes(3); - expect( - rpcEndpointDegradedEventHandler, - ).toHaveBeenNthCalledWith(1, { - chainId, - type: 'retries_exhausted', - endpointUrl: rpcUrl, - error: expectedDegradedError, + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + isOffline: (): boolean => false, + }), + }, + async ({ makeRpcCall, chainId }) => { + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: DEFAULT_MAX_CONSECUTIVE_FAILURES, + response: { + httpStatus: 503, + }, + }); + failoverComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: 5, + response: { + httpStatus: 503, + }, + }); + + messenger.subscribe( + 'NetworkController:rpcEndpointRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + jest.advanceTimersByTime(backoffDuration); + }, + ); + + // Hit the primary and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the primary and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the primary and exceed the max number of retries, + // break the circuit; hit the failover and exceed the max + // number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + + expect( + rpcEndpointChainDegradedEventHandler, + ).toHaveBeenCalledTimes(1); + expect( + rpcEndpointChainDegradedEventHandler, + ).toHaveBeenCalledWith({ + chainId, + type: 'retries_exhausted', + error: expectedDegradedError, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + retryReason: 'non_successful_http_status', + rpcMethodName: 'eth_blockNumber', + duration: undefined, + traceId: undefined, + }); + }, + ); + }, + ); + }, + ); + }); + + it('does not publish the NetworkController:rpcEndpointChainDegraded event again when the time to complete a request to a failover endpoint is too long', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + const request = { + method: 'eth_gasPrice', + params: [], + }; + const expectedError = createResourceUnavailableError(503); + const expectedDegradedError = new HttpError(503); + + await withMockedCommunications( + { providerType: networkClientType }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async (failoverComms) => { + const messenger = buildRootMessenger(); + const rpcEndpointChainDegradedEventHandler = jest.fn(); + messenger.subscribe( + 'NetworkController:rpcEndpointChainDegraded', + rpcEndpointChainDegradedEventHandler, + ); + + await withNetworkClient( + { + providerType: networkClientType, networkClientId: 'AAAA-AAAA-AAAA-AAAA', - primaryEndpointUrl: rpcUrl, - retryReason: 'non_successful_http_status', - rpcMethodName: 'eth_blockNumber', - duration: undefined, - traceId: undefined, - }); - expect( - rpcEndpointDegradedEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - type: 'retries_exhausted', - endpointUrl: rpcUrl, - error: expectedDegradedError, + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + isOffline: (): boolean => false, + }), + }, + async ({ makeRpcCall, chainId }) => { + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: DEFAULT_MAX_CONSECUTIVE_FAILURES, + response: { + httpStatus: 503, + }, + }); + failoverComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + response: () => { + jest.advanceTimersByTime( + DEFAULT_DEGRADED_THRESHOLD + 1, + ); + return { + result: '0x1', + }; + }, + }); + failoverComms.mockRpcCall({ + request, + response: () => { + jest.advanceTimersByTime( + DEFAULT_DEGRADED_THRESHOLD + 1, + ); + return { + result: 'ok', + }; + }, + }); + + messenger.subscribe( + 'NetworkController:rpcEndpointRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + jest.advanceTimersByTime(backoffDuration); + }, + ); + + // Hit the primary and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the primary and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the primary and exceed the max number of retries, + // break the circuit; hit the failover + await makeRpcCall(request); + + expect( + rpcEndpointChainDegradedEventHandler, + ).toHaveBeenCalledTimes(1); + expect( + rpcEndpointChainDegradedEventHandler, + ).toHaveBeenCalledWith({ + chainId, + type: 'retries_exhausted', + error: expectedDegradedError, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + retryReason: 'non_successful_http_status', + rpcMethodName: 'eth_blockNumber', + duration: undefined, + traceId: undefined, + }); + }, + ); + }, + ); + }, + ); + }); + + it('publishes the NetworkController:rpcEndpointDegraded event again if the max number of retries is reached in making requests to a failover endpoint', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + const request = { + method: 'eth_gasPrice', + params: [], + }; + const expectedError = createResourceUnavailableError(503); + const expectedDegradedError = new HttpError(503); + + await withMockedCommunications( + { providerType: networkClientType }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async (failoverComms) => { + const messenger = buildRootMessenger(); + const rpcEndpointDegradedEventHandler = jest.fn(); + messenger.subscribe( + 'NetworkController:rpcEndpointDegraded', + rpcEndpointDegradedEventHandler, + ); + + await withNetworkClient( + { + providerType: networkClientType, networkClientId: 'AAAA-AAAA-AAAA-AAAA', - primaryEndpointUrl: rpcUrl, - retryReason: 'non_successful_http_status', - rpcMethodName: 'eth_blockNumber', - duration: undefined, - traceId: undefined, - }); - expect( - rpcEndpointDegradedEventHandler, - ).toHaveBeenNthCalledWith(3, { - chainId, - type: 'retries_exhausted', - endpointUrl: failoverEndpointUrl, - error: expectedDegradedError, + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + isOffline: (): boolean => false, + }), + }, + async ({ makeRpcCall, chainId, rpcUrl }) => { + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: DEFAULT_MAX_CONSECUTIVE_FAILURES, + response: { + httpStatus: 503, + }, + }); + failoverComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: 5, + response: { + httpStatus: 503, + }, + }); + + messenger.subscribe( + 'NetworkController:rpcEndpointRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + jest.advanceTimersByTime(backoffDuration); + }, + ); + + // Hit the primary and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the primary and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the primary and exceed the max number of retries, + // break the circuit; hit the failover and exceed the max + // number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + + expect( + rpcEndpointDegradedEventHandler, + ).toHaveBeenCalledTimes(3); + expect( + rpcEndpointDegradedEventHandler, + ).toHaveBeenNthCalledWith(1, { + chainId, + type: 'retries_exhausted', + endpointUrl: rpcUrl, + error: expectedDegradedError, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + primaryEndpointUrl: rpcUrl, + retryReason: 'non_successful_http_status', + rpcMethodName: 'eth_blockNumber', + duration: undefined, + traceId: undefined, + }); + expect( + rpcEndpointDegradedEventHandler, + ).toHaveBeenNthCalledWith(2, { + chainId, + type: 'retries_exhausted', + endpointUrl: rpcUrl, + error: expectedDegradedError, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + primaryEndpointUrl: rpcUrl, + retryReason: 'non_successful_http_status', + rpcMethodName: 'eth_blockNumber', + duration: undefined, + traceId: undefined, + }); + expect( + rpcEndpointDegradedEventHandler, + ).toHaveBeenNthCalledWith(3, { + chainId, + type: 'retries_exhausted', + endpointUrl: failoverEndpointUrl, + error: expectedDegradedError, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + primaryEndpointUrl: rpcUrl, + retryReason: 'non_successful_http_status', + rpcMethodName: 'eth_blockNumber', + duration: undefined, + traceId: undefined, + }); + }, + ); + }, + ); + }, + ); + }); + + it('publishes the NetworkController:rpcEndpointDegraded event again when the time to complete a request to a failover endpoint is too long', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + const request = { + method: 'eth_gasPrice', + params: [], + }; + const expectedError = createResourceUnavailableError(503); + const expectedDegradedError = new HttpError(503); + + await withMockedCommunications( + { providerType: networkClientType }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async (failoverComms) => { + const messenger = buildRootMessenger(); + const rpcEndpointDegradedEventHandler = jest.fn(); + messenger.subscribe( + 'NetworkController:rpcEndpointDegraded', + rpcEndpointDegradedEventHandler, + ); + + await withNetworkClient( + { + providerType: networkClientType, networkClientId: 'AAAA-AAAA-AAAA-AAAA', - primaryEndpointUrl: rpcUrl, - retryReason: 'non_successful_http_status', - rpcMethodName: 'eth_blockNumber', - duration: undefined, - traceId: undefined, - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointDegraded event again when the time to complete a request to a failover endpoint is too long', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; - const request = { - method: 'eth_gasPrice', - params: [], - }; - const expectedError = createResourceUnavailableError(503); - const expectedDegradedError = new HttpError(503); - - await withMockedCommunications( - { providerType: networkClientType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, - }, - async (failoverComms) => { - const messenger = buildRootMessenger(); - const rpcEndpointDegradedEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointDegraded', - rpcEndpointDegradedEventHandler, - ); - - await withNetworkClient( - { - providerType: networkClientType, - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - isOffline: (): boolean => false, - }), - }, - async ({ makeRpcCall, chainId, rpcUrl }) => { - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: DEFAULT_MAX_CONSECUTIVE_FAILURES, - response: { - httpStatus: 503, - }, - }); - failoverComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: () => { - jest.advanceTimersByTime( - DEFAULT_DEGRADED_THRESHOLD + 1, - ); - return { - result: '0x1', - }; - }, - }); - failoverComms.mockRpcCall({ - request, - response: () => { - jest.advanceTimersByTime( - DEFAULT_DEGRADED_THRESHOLD + 1, - ); - return { - result: 'ok', - }; - }, - }); - - messenger.subscribe( - 'NetworkController:rpcEndpointRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - jest.advanceTimersByTime(backoffDuration); - }, - ); - - // Hit the primary and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the primary and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the primary and exceed the max number of retries, - // break the circuit; hit the failover - await makeRpcCall(request); - - expect( - rpcEndpointDegradedEventHandler, - ).toHaveBeenCalledTimes(4); - expect( - rpcEndpointDegradedEventHandler, - ).toHaveBeenNthCalledWith(1, { - chainId, - type: 'retries_exhausted', - endpointUrl: rpcUrl, - error: expectedDegradedError, - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - primaryEndpointUrl: rpcUrl, - retryReason: 'non_successful_http_status', - rpcMethodName: 'eth_blockNumber', - duration: undefined, - traceId: undefined, - }); - expect( - rpcEndpointDegradedEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - type: 'retries_exhausted', - endpointUrl: rpcUrl, - error: expectedDegradedError, - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - primaryEndpointUrl: rpcUrl, - retryReason: 'non_successful_http_status', - rpcMethodName: 'eth_blockNumber', - duration: undefined, - traceId: undefined, - }); - expect( - rpcEndpointDegradedEventHandler, - ).toHaveBeenNthCalledWith(3, { - chainId, - type: 'slow_success', - endpointUrl: failoverEndpointUrl, - error: undefined, - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - primaryEndpointUrl: rpcUrl, - rpcMethodName: 'eth_blockNumber', - duration: expect.any(Number), - traceId: undefined, - }); - expect( - rpcEndpointDegradedEventHandler, - ).toHaveBeenNthCalledWith(4, { - chainId, - type: 'slow_success', - endpointUrl: failoverEndpointUrl, - error: undefined, - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - primaryEndpointUrl: rpcUrl, - rpcMethodName: 'eth_gasPrice', - duration: expect.any(Number), - traceId: undefined, - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointChainAvailable event the first time a successful request to a failover endpoint is made', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; - const request = { - method: 'eth_gasPrice', - params: [], - }; - const expectedError = createResourceUnavailableError(503); - - await withMockedCommunications( - { providerType: networkClientType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, - }, - async (failoverComms) => { - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - times: DEFAULT_MAX_CONSECUTIVE_FAILURES, - response: { - httpStatus: 503, - }, - }); - failoverComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - result: '0x1', - }, - }); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointChainAvailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointChainAvailable', - rpcEndpointChainAvailableEventHandler, - ); - - await withNetworkClient( - { - providerType: networkClientType, - networkClientId: 'AAAA-AAAA-AAAA-AAAA', - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - isOffline: (): boolean => false, - }), - }, - async ({ makeRpcCall, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - jest.advanceTimersByTime(backoffDuration); - }, - ); - - // Hit the endpoint and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the endpoint and exceed the max number of retries - await expect(makeRpcCall(request)).rejects.toThrow( - expectedError, - ); - // Hit the endpoint and exceed the max number of retries, - // breaking the circuit; hit the failover - await makeRpcCall(request); - - expect( - rpcEndpointChainAvailableEventHandler, - ).toHaveBeenCalledTimes(1); - expect( - rpcEndpointChainAvailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + isOffline: (): boolean => false, + }), + }, + async ({ makeRpcCall, chainId, rpcUrl }) => { + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: DEFAULT_MAX_CONSECUTIVE_FAILURES, + response: { + httpStatus: 503, + }, + }); + failoverComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + response: () => { + jest.advanceTimersByTime( + DEFAULT_DEGRADED_THRESHOLD + 1, + ); + return { + result: '0x1', + }; + }, + }); + failoverComms.mockRpcCall({ + request, + response: () => { + jest.advanceTimersByTime( + DEFAULT_DEGRADED_THRESHOLD + 1, + ); + return { + result: 'ok', + }; + }, + }); + + messenger.subscribe( + 'NetworkController:rpcEndpointRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + jest.advanceTimersByTime(backoffDuration); + }, + ); + + // Hit the primary and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the primary and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the primary and exceed the max number of retries, + // break the circuit; hit the failover + await makeRpcCall(request); + + expect( + rpcEndpointDegradedEventHandler, + ).toHaveBeenCalledTimes(4); + expect( + rpcEndpointDegradedEventHandler, + ).toHaveBeenNthCalledWith(1, { + chainId, + type: 'retries_exhausted', + endpointUrl: rpcUrl, + error: expectedDegradedError, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + primaryEndpointUrl: rpcUrl, + retryReason: 'non_successful_http_status', + rpcMethodName: 'eth_blockNumber', + duration: undefined, + traceId: undefined, + }); + expect( + rpcEndpointDegradedEventHandler, + ).toHaveBeenNthCalledWith(2, { + chainId, + type: 'retries_exhausted', + endpointUrl: rpcUrl, + error: expectedDegradedError, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + primaryEndpointUrl: rpcUrl, + retryReason: 'non_successful_http_status', + rpcMethodName: 'eth_blockNumber', + duration: undefined, + traceId: undefined, + }); + expect( + rpcEndpointDegradedEventHandler, + ).toHaveBeenNthCalledWith(3, { + chainId, + type: 'slow_success', + endpointUrl: failoverEndpointUrl, + error: undefined, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + primaryEndpointUrl: rpcUrl, + rpcMethodName: 'eth_blockNumber', + duration: expect.any(Number), + traceId: undefined, + }); + expect( + rpcEndpointDegradedEventHandler, + ).toHaveBeenNthCalledWith(4, { + chainId, + type: 'slow_success', + endpointUrl: failoverEndpointUrl, + error: undefined, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + primaryEndpointUrl: rpcUrl, + rpcMethodName: 'eth_gasPrice', + duration: expect.any(Number), + traceId: undefined, + }); + }, + ); + }, + ); + }, + ); + }); + + it('publishes the NetworkController:rpcEndpointChainAvailable event the first time a successful request to a failover endpoint is made', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + const request = { + method: 'eth_gasPrice', + params: [], + }; + const expectedError = createResourceUnavailableError(503); + + await withMockedCommunications( + { providerType: networkClientType }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async (failoverComms) => { + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + times: DEFAULT_MAX_CONSECUTIVE_FAILURES, + response: { + httpStatus: 503, + }, + }); + failoverComms.mockRpcCall({ + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x1', + }, + }); + failoverComms.mockRpcCall({ + request, + response: { + result: 'ok', + }, + }); + + const messenger = buildRootMessenger(); + const rpcEndpointChainAvailableEventHandler = jest.fn(); + messenger.subscribe( + 'NetworkController:rpcEndpointChainAvailable', + rpcEndpointChainAvailableEventHandler, + ); + + await withNetworkClient( + { + providerType: networkClientType, networkClientId: 'AAAA-AAAA-AAAA-AAAA', - }); - }, - ); - }, - ); - }, - ); + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + isOffline: (): boolean => false, + }), + }, + async ({ makeRpcCall, chainId }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + jest.advanceTimersByTime(backoffDuration); + }, + ); + + // Hit the endpoint and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the endpoint and exceed the max number of retries + await expect(makeRpcCall(request)).rejects.toThrow( + expectedError, + ); + // Hit the endpoint and exceed the max number of retries, + // breaking the circuit; hit the failover + await makeRpcCall(request); + + expect( + rpcEndpointChainAvailableEventHandler, + ).toHaveBeenCalledTimes(1); + expect( + rpcEndpointChainAvailableEventHandler, + ).toHaveBeenCalledWith({ + chainId, + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + }); + }, + ); + }, + ); + }, + ); + }); }); - }); + } describe('without RPC failover', () => { it('publishes the NetworkController:rpcEndpointChainDegraded event only once, even if the max number of retries is continually reached in making requests to a primary endpoint', async () => { diff --git a/packages/network-controller/src/create-network-client.ts b/packages/network-controller/src/create-network-client.ts index cc8967cf36..514d82d4a1 100644 --- a/packages/network-controller/src/create-network-client.ts +++ b/packages/network-controller/src/create-network-client.ts @@ -264,15 +264,16 @@ function createRpcServiceChain({ isRpcFailoverEnabled: boolean; logger?: Logger; }): RpcServiceChain { - const availableEndpoints = isRpcFailoverEnabled - ? [ - { url: primaryEndpointUrl, isFailover: false }, - ...(configuration.failoverRpcUrls ?? []).map((url) => ({ - url, - isFailover: true, - })), - ] - : [{ url: primaryEndpointUrl, isFailover: false }]; + const availableEndpoints = + isRpcFailoverEnabled && configuration.type === NetworkClientType.Infura + ? [ + { url: primaryEndpointUrl, isFailover: false }, + ...(configuration.failoverRpcUrls ?? []).map((url) => ({ + url, + isFailover: true, + })), + ] + : [{ url: primaryEndpointUrl, isFailover: false }]; const isOffline = (): boolean => { const connectivityState = messenger.call('ConnectivityController:getState'); diff --git a/packages/network-controller/tests/NetworkController.provider.test.ts b/packages/network-controller/tests/NetworkController.provider.test.ts index 1588aaaf7d..2819f0de7f 100644 --- a/packages/network-controller/tests/NetworkController.provider.test.ts +++ b/packages/network-controller/tests/NetworkController.provider.test.ts @@ -1,4 +1,7 @@ -import { DEFAULT_DEGRADED_THRESHOLD } from '@metamask/controller-utils'; +import { + DEFAULT_DEGRADED_THRESHOLD, + InfuraNetworkType, +} from '@metamask/controller-utils'; import { Duration, inMilliseconds } from '@metamask/utils'; import nock from 'nock'; @@ -6,6 +9,8 @@ import { NetworkStatus } from '../src/constants'; import { buildCustomNetworkConfiguration, buildCustomRpcEndpoint, + buildInfuraNetworkConfiguration, + buildInfuraRpcEndpoint, withController, } from './helpers'; @@ -235,13 +240,14 @@ describe('NetworkController provider tests', () => { }); it('transitions the status of a network client from "degraded" to "available" the first time a failover is activated and returns a 2xx response', async () => { - const primaryEndpointUrl = 'https://first.endpoint'; + const primaryEndpointUrl = 'https://mainnet.infura.io'; + const primaryEndpointPath = '/v3/infura-project-id'; const secondaryEndpointUrl = 'https://second.endpoint'; - const networkClientId = 'AAAA-AAAA-AAAA-AAAA'; + const networkClientId = InfuraNetworkType.mainnet; const rpcMethod = 'eth_gasPrice'; nock(primaryEndpointUrl) - .post('/', { + .post(primaryEndpointPath, { id: /^\d+$/u, jsonrpc: '2.0', method: 'eth_blockNumber', @@ -278,13 +284,9 @@ describe('NetworkController provider tests', () => { isRpcFailoverEnabled: true, state: { networkConfigurationsByChainId: { - '0x1337': buildCustomNetworkConfiguration({ - chainId: '0x1337', - name: 'Test Network', + '0x1': buildInfuraNetworkConfiguration(InfuraNetworkType.mainnet, { rpcEndpoints: [ - buildCustomRpcEndpoint({ - networkClientId, - url: primaryEndpointUrl, + buildInfuraRpcEndpoint(InfuraNetworkType.mainnet, { failoverUrls: [secondaryEndpointUrl], }), ], @@ -331,7 +333,7 @@ describe('NetworkController provider tests', () => { [ { op: 'replace', - path: ['networksMetadata', 'AAAA-AAAA-AAAA-AAAA', 'status'], + path: ['networksMetadata', networkClientId, 'status'], value: 'degraded', }, ], @@ -342,7 +344,7 @@ describe('NetworkController provider tests', () => { [ { op: 'replace', - path: ['networksMetadata', 'AAAA-AAAA-AAAA-AAAA', 'status'], + path: ['networksMetadata', networkClientId, 'status'], value: 'available', }, ], @@ -352,13 +354,14 @@ describe('NetworkController provider tests', () => { }); it('does not transition the status of a network client from "degraded" the first time a failover is activated if it returns a non-2xx response', async () => { - const primaryEndpointUrl = 'https://first.endpoint'; + const primaryEndpointUrl = 'https://mainnet.infura.io'; + const primaryEndpointPath = '/v3/infura-project-id'; const secondaryEndpointUrl = 'https://second.endpoint'; - const networkClientId = 'AAAA-AAAA-AAAA-AAAA'; + const networkClientId = InfuraNetworkType.mainnet; const rpcMethod = 'eth_gasPrice'; nock(primaryEndpointUrl) - .post('/', { + .post(primaryEndpointPath, { id: /^\d+$/u, jsonrpc: '2.0', method: 'eth_blockNumber', @@ -381,13 +384,9 @@ describe('NetworkController provider tests', () => { isRpcFailoverEnabled: true, state: { networkConfigurationsByChainId: { - '0x1337': buildCustomNetworkConfiguration({ - chainId: '0x1337', - name: 'Test Network', + '0x1': buildInfuraNetworkConfiguration(InfuraNetworkType.mainnet, { rpcEndpoints: [ - buildCustomRpcEndpoint({ - networkClientId, - url: primaryEndpointUrl, + buildInfuraRpcEndpoint(InfuraNetworkType.mainnet, { failoverUrls: [secondaryEndpointUrl], }), ], @@ -436,13 +435,14 @@ describe('NetworkController provider tests', () => { }); it('does not transition the status of a network client from "degraded" the first time a failover is activated if requests are slow to complete', async () => { - const primaryEndpointUrl = 'https://first.endpoint'; + const primaryEndpointUrl = 'https://mainnet.infura.io'; + const primaryEndpointPath = '/v3/infura-project-id'; const secondaryEndpointUrl = 'https://second.endpoint'; - const networkClientId = 'AAAA-AAAA-AAAA-AAAA'; + const networkClientId = InfuraNetworkType.mainnet; const rpcMethod = 'eth_gasPrice'; nock(primaryEndpointUrl) - .post('/', { + .post(primaryEndpointPath, { id: /^\d+$/u, jsonrpc: '2.0', method: 'eth_blockNumber', @@ -491,13 +491,9 @@ describe('NetworkController provider tests', () => { isRpcFailoverEnabled: true, state: { networkConfigurationsByChainId: { - '0x1337': buildCustomNetworkConfiguration({ - chainId: '0x1337', - name: 'Test Network', + '0x1': buildInfuraNetworkConfiguration(InfuraNetworkType.mainnet, { rpcEndpoints: [ - buildCustomRpcEndpoint({ - networkClientId, - url: primaryEndpointUrl, + buildInfuraRpcEndpoint(InfuraNetworkType.mainnet, { failoverUrls: [secondaryEndpointUrl], }), ], @@ -545,13 +541,14 @@ describe('NetworkController provider tests', () => { }); it('sets the status of a network client to "unavailable" when all of its RPC endpoints consistently return 5xx errors, reaching the max consecutive number of failures', async () => { - const primaryEndpointUrl = 'https://first.endpoint'; + const primaryEndpointUrl = 'https://mainnet.infura.io'; + const primaryEndpointPath = '/v3/infura-project-id'; const secondaryEndpointUrl = 'https://second.endpoint'; - const networkClientId = 'AAAA-AAAA-AAAA-AAAA'; + const networkClientId = InfuraNetworkType.mainnet; const rpcMethod = 'eth_gasPrice'; nock(primaryEndpointUrl) - .post('/', { + .post(primaryEndpointPath, { id: /^\d+$/u, jsonrpc: '2.0', method: 'eth_blockNumber', @@ -574,13 +571,9 @@ describe('NetworkController provider tests', () => { isRpcFailoverEnabled: true, state: { networkConfigurationsByChainId: { - '0x1337': buildCustomNetworkConfiguration({ - chainId: '0x1337', - name: 'Test Network', + '0x1': buildInfuraNetworkConfiguration(InfuraNetworkType.mainnet, { rpcEndpoints: [ - buildCustomRpcEndpoint({ - networkClientId, - url: primaryEndpointUrl, + buildInfuraRpcEndpoint(InfuraNetworkType.mainnet, { failoverUrls: [secondaryEndpointUrl], }), ], diff --git a/packages/network-controller/tests/network-client/rpc-failover.ts b/packages/network-controller/tests/network-client/rpc-failover.ts index 220cf3d7c1..a587b1e24f 100644 --- a/packages/network-controller/tests/network-client/rpc-failover.ts +++ b/packages/network-controller/tests/network-client/rpc-failover.ts @@ -35,6 +35,10 @@ export function testsForRpcFailoverBehavior({ getExpectedError: (url: string) => Error | jest.Constructable; getExpectedBreakError?: (url: string) => Error | jest.Constructable; }): void { + if (providerType === 'custom') { + return; + } + const blockNumber = '0x100'; const backoffDuration = 100; const maxConsecutiveFailures = 15;