From c8cbef3e287d6e5ef49fc2335ac6624aa9dfa111 Mon Sep 17 00:00:00 2001 From: Mozzam Shahid Date: Wed, 15 Jul 2026 21:04:37 +0500 Subject: [PATCH] feat(cli): add --fail-on-error for batch place scraping --- src/gmaps_scraper/cli.py | 12 +++++ tests/test_cli.py | 101 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/src/gmaps_scraper/cli.py b/src/gmaps_scraper/cli.py index 9484a60..8e0bab9 100644 --- a/src/gmaps_scraper/cli.py +++ b/src/gmaps_scraper/cli.py @@ -181,6 +181,14 @@ def build_parser() -> argparse.ArgumentParser: default=0, help="For batch place scraping, delay starts between URLs or workers.", ) + parser.add_argument( + "--fail-on-error", + action="store_true", + help=( + "For batch place scraping, return a non-zero exit code when any " + "place fails to scrape." + ), + ) parser.add_argument( "--llm-repair", action="store_true", @@ -348,6 +356,10 @@ def main() -> int: args.output.write_text(f"{payload}\n", encoding="utf-8") else: print(payload) + if args.fail_on_error and any( + result.error is not None for result in batch_results + ): + return 1 return 0 if ( args.download_photo is not None diff --git a/tests/test_cli.py b/tests/test_cli.py index 4e94680..e9cbb7f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -988,6 +988,107 @@ def test_place_kind_can_scrape_batch_from_stdin(self) -> None: ], ) + def test_place_kind_batch_fail_on_error_returns_one_on_error(self) -> None: + stdout = io.StringIO() + details = PlaceDetails( + source_url="https://www.google.com/maps/place/Den", + resolved_url="https://www.google.com/maps/place/Den", + name="Den", + category="Japanese restaurant", + rating=4.4, + review_count=324, + address="Tokyo, Japan", + ) + with ( + patch( + "sys.argv", + [ + "gmaps-scraper", + "--kind", + "place", + "https://www.google.com/maps/place/Den", + "https://www.google.com/maps/place/Narisawa", + "--fail-on-error", + ], + ), + patch( + "gmaps_scraper.cli.scrape_places", + return_value=[ + PlaceScrapeResult( + source_url="https://www.google.com/maps/place/Den", + place=details, + attempts=1, + ), + PlaceScrapeResult( + source_url="https://www.google.com/maps/place/Narisawa", + error="blocked", + attempts=3, + ), + ], + ) as scrape_places, + redirect_stdout(stdout), + ): + exit_code = main() + + self.assertEqual(exit_code, 1) + scrape_places.assert_called_once_with( + [ + "https://www.google.com/maps/place/Den", + "https://www.google.com/maps/place/Narisawa", + ], + headless=True, + timeout_ms=30_000, + settle_time_ms=3_000, + browser_session=None, + http_session=None, + llm_fallback=None, + llm_policy="on_quality_failure", + llm_tasks=("dom_repair", "display_translation"), + collect_reviews=True, + collect_about=True, + max_concurrency=1, + max_retries=1, + retry_backoff_ms=2_000, + stagger_ms=0, + ) + payload = json.loads(stdout.getvalue()) + self.assertEqual(payload["results"][0]["place"]["name"], "Den") + self.assertEqual(payload["results"][1]["error"], "blocked") + + def test_place_kind_batch_fail_on_error_returns_zero_when_all_succeed(self) -> None: + stdout = io.StringIO() + with ( + patch( + "sys.argv", + [ + "gmaps-scraper", + "--kind", + "place", + "https://www.google.com/maps/place/Den", + "https://www.google.com/maps/place/Narisawa", + "--fail-on-error", + ], + ), + patch( + "gmaps_scraper.cli.scrape_places", + return_value=[ + PlaceScrapeResult( + source_url="https://www.google.com/maps/place/Den", + attempts=1, + ), + PlaceScrapeResult( + source_url="https://www.google.com/maps/place/Narisawa", + attempts=1, + ), + ], + ) as scrape_places, + redirect_stdout(stdout), + ): + exit_code = main() + + self.assertEqual(exit_code, 0) + scrape_places.assert_called_once() + def test_download_place_photo_writes_bytes(self) -> None: details = PlaceDetails( source_url="https://www.google.com/maps/place/Den",