diff --git a/tap_github/client.py b/tap_github/client.py index 77f07c39..749963a6 100644 --- a/tap_github/client.py +++ b/tap_github/client.py @@ -334,6 +334,9 @@ def get_next_page_token( Warning - we recommend to avoid using deep (nested) pagination. """ + if hasattr(self, "do_not_paginate") and self.do_not_paginate: + return None + resp_json = response.json() # Find if results contains "hasNextPage_X" flags and if any are True. diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 2e03497c..7a522017 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2263,7 +2263,6 @@ def query(self) -> str: cost } } - """ schema = th.PropertiesList( @@ -2296,6 +2295,20 @@ def query(self) -> str: ).to_dict() +class DependenciesStreamIncomplete(DependenciesStream): + """Defines 'DependenciesStreamDirty' stream to limit pagination.""" + + do_not_paginate = True + + @property + def query(self) -> str: + """Return altered query to limit pagination.""" + initial_query = super().query + no_pagination_query = initial_query.replace("first: 1", "first: 20") + no_pagination_query = no_pagination_query.replace("first: 50", "first: 100") + return no_pagination_query + + class TrafficRestStream(GitHubRestStream): """Base class for Traffic Streams""" diff --git a/tap_github/streams.py b/tap_github/streams.py index e1b05e58..5f71d809 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -18,6 +18,7 @@ CommunityProfileStream, ContributorsStream, DependenciesStream, + DependenciesStreamIncomplete, DependentsStream, EventsStream, ExtraMetricsStream, @@ -75,6 +76,7 @@ def __init__(self, valid_queries: Set[str], streams: List[Type[Stream]]): CommunityProfileStream, ContributorsStream, DependenciesStream, + DependenciesStreamIncomplete, DependentsStream, EventsStream, IssueCommentsStream,