diff --git a/mozregression/fetch_build_info.py b/mozregression/fetch_build_info.py index 01e105927..e28bd2141 100644 --- a/mozregression/fetch_build_info.py +++ b/mozregression/fetch_build_info.py @@ -10,6 +10,7 @@ import os import re +from dataclasses import dataclass from datetime import datetime from threading import Lock, Thread @@ -19,7 +20,7 @@ from taskcluster.exceptions import TaskclusterFailure from mozregression.build_info import IntegrationBuildInfo, NightlyBuildInfo -from mozregression.errors import BuildInfoNotFound, MozRegressionError +from mozregression.errors import BuildInfoNotFound, EmptyPushlogError, MozRegressionError from mozregression.json_pushes import JsonPushes, Push from mozregression.network import retry_get, url_links @@ -32,37 +33,7 @@ def __init__(self, fetch_config): self.build_regex = re.compile(fetch_config.build_regex()) self.build_info_regex = re.compile(fetch_config.build_info_regex()) - def _update_build_info_from_txt(self, build_info): - LOG.debug("Update build info from {}".format(build_info)) - if "build_txt_url" in build_info: - build_info.update(self._fetch_txt_info(build_info["build_txt_url"])) - - def _fetch_txt_info(self, url): - """ - Retrieve information from a build information txt file. - - Returns a dict with keys repository and changeset if information - is found. - """ - LOG.debug("Fetching txt info from {}".format(url)) - data = {} - response = retry_get(url) - for line in response.text.splitlines(): - if "/rev/" in line: - repository, changeset = line.split("/rev/") - data["repository"] = repository - data["changeset"] = changeset - break - if not data: - # the txt file could be in an old format: - # DATE CHANGESET - # we can try to extract that to get the changeset at least. - matched = re.match(r"^\d+ (\w+)$", response.text.strip()) - if matched: - data["changeset"] = matched.group(1) - return data - - def find_build_info(self, changeset_or_date, fetch_txt_info=True): + def find_build_info(self, changeset_or_date): """ Abstract method to retrieve build information over the internet for one build. @@ -172,9 +143,80 @@ def find_build_info(self, push): ) +@dataclass +class ChangesetInfo: + """ + Result of resolving changeset info for a nightly build. + + These are generated during the fetch process and then turned into a + `BuildInfo` before being consumed elsewhere in the system. Some sample + providers are included, but the fetch config's `get_nightly_changeset` + can provide other behaviours. + """ + + changeset: str = None + repo_url: str = None + + @staticmethod + def from_nightly_txt(archive_urls): + """ + Lookup changeset info from .txt file on archive server if exists. + """ + if archive_urls.build_txt_url: + LOG.debug("Fetching txt info from {}".format(archive_urls.build_txt_url)) + + response = retry_get(archive_urls.build_txt_url) + for line in response.text.splitlines(): + if "/rev/" in line: + repo_url, changeset = line.split("/rev/") + return ChangesetInfo(changeset, repo_url) + + # the txt file could be in an old format: + # DATE CHANGESET + # we can try to extract that to get the changeset at least. + matched = re.match(r"^\d+ (\w+)$", response.text.strip()) + if matched: + changeset = matched.group(1) + return ChangesetInfo(changeset) + + return ChangesetInfo() + + @staticmethod + def from_pushlog(archive_urls, fetch_config): + """ + Use the json-pushes API of the source control server to lookup the + changeset using the build timestamp. + """ + build_dt = fetch_config.get_nightly_timestamp_from_url(archive_urls.build_url) + branch = fetch_config.get_nightly_repo(build_dt.date()) + + try: + jpushes = JsonPushes(branch=branch) + except MozRegressionError: + LOG.info(f"Repo {branch} does not support json-pushes queries") + return ChangesetInfo() + + try: + push = jpushes.push_by_timestamp(build_dt)[-1] + except EmptyPushlogError: + LOG.info(f"Unable to fetch push by timestamp for {build_dt}") + return ChangesetInfo() + + return ChangesetInfo(push.changeset, jpushes.repo_url) + + +@dataclass +class ArchiveBuildUrls: + """Result of scraping build folder on archive server.""" + + build_url: str + build_txt_url: str + + class NightlyInfoFetcher(InfoFetcher): def __init__(self, fetch_config): InfoFetcher.__init__(self, fetch_config) + self._cache_months = {} self._lock = Lock() self._fetch_lock = Lock() @@ -183,34 +225,38 @@ def _fetch_build_info_from_url(self, url, index, lst): """ Retrieve information from a build folder url. - Stores in a list the url index and a dict instance with keys + Stores in a list the url index and a ArchiveBuildUrls instance with build_url and build_txt_url if respectively a build file and a build info file are found for the url. """ LOG.debug("Fetching build info from {}".format(url)) - data = {} + if not url.endswith("/"): url += "/" - links = url_links(url) - if not self.fetch_config.has_build_info: - links += url_links(self.fetch_config.get_nightly_info_url(url)) - for link in links: - name = os.path.basename(link) - if "build_url" not in data and self.build_regex.match(name): - data["build_url"] = link - elif "build_txt_url" not in data and self.build_info_regex.match(name): - data["build_txt_url"] = link - if data: - # Check that we found all required data. The URL in build_url is - # required. build_txt_url is optional. - if "build_url" not in data: - raise BuildInfoNotFound( - "Failed to find a build file in directory {} that " - "matches regex '{}'".format(url, self.build_regex.pattern) - ) + info_url = self.fetch_config.get_nightly_info_url(url) + + build_urls = [] + build_txt_urls = [] + for folder in {url, info_url}: + for link in url_links(folder): + name = os.path.basename(link) + if self.build_regex.match(name): + build_urls.append(link) + if self.build_info_regex.match(name): + build_txt_urls.append(link) + + if build_urls: + build = build_urls[0] + build_txt = build_txt_urls[0] if build_txt_urls else None with self._fetch_lock: - lst.append((index, data)) + lst.append((index, ArchiveBuildUrls(build, build_txt))) + elif build_txt_urls: + raise BuildInfoNotFound( + "Failed to find a build file in directory {} that matches regex '{}'".format( + url, self.build_regex.pattern + ) + ) def _get_month_links(self, url): with self._lock: @@ -223,7 +269,7 @@ def _get_urls(self, date): Get the url list of the build folder for a given date. This methods needs to be thread-safe as it is used in - :meth:`NightlyBuildData.get_build_url`. + :meth:`find_build_info`. """ LOG.debug("Get URLs for {}".format(date)) url = self.fetch_config.get_nightly_base_url(date) @@ -240,7 +286,7 @@ def _get_urls(self, date): matches.reverse() return matches - def find_build_info(self, date, fetch_txt_info=True, max_workers=2): + def find_build_info(self, date, max_workers=2): """ Find build info for a nightly build, given a date. @@ -274,16 +320,14 @@ def find_build_info(self, date, fetch_txt_info=True, max_workers=2): thread.join(0.1) LOG.debug("got valid_builds %s" % valid_builds) if valid_builds: - infos = sorted(valid_builds, key=lambda b: b[0])[0][1] - if fetch_txt_info: - self._update_build_info_from_txt(infos) - + selection = sorted(valid_builds, key=lambda b: b[0])[0][1] + changeset = self.fetch_config.get_nightly_changeset(selection) build_info = NightlyBuildInfo( self.fetch_config, - build_url=infos["build_url"], + build_url=selection.build_url, build_date=date, - changeset=infos.get("changeset"), - repo_url=infos.get("repository"), + changeset=changeset.changeset, + repo_url=changeset.repo_url, ) break build_urls = build_urls[max_workers:] diff --git a/mozregression/fetch_configs.py b/mozregression/fetch_configs.py index 9e1bb16fb..805d58225 100644 --- a/mozregression/fetch_configs.py +++ b/mozregression/fetch_configs.py @@ -32,6 +32,7 @@ from mozregression.class_registry import ClassRegistry from mozregression.config import ARCHIVE_BASE_URL from mozregression.dates import to_utc_timestamp +from mozregression.fetch_build_info import ChangesetInfo LOG = get_proxy_logger(__name__) @@ -130,7 +131,7 @@ def __init__(self, os, bits, processor, arch): self.processor = processor self.set_arch(arch) self.repo = None - self.set_build_type("opt") + self.set_build_type(self.BUILD_TYPES[0]) self._used_build_index = 0 @property @@ -270,7 +271,7 @@ class NightlyConfigMixin(metaclass=ABCMeta): A nightly build url is divided in 2 parts here: 1. the base part as returned by :meth:`get_nightly_base_url` - 2. the final part, which can be found using :meth:`get_nighly_repo_regex` + 2. the final part, which can be found using :meth:`get_nightly_repo_regex` The final part contains a repo name, which is returned by :meth:`get_nightly_repo`. @@ -281,8 +282,6 @@ class NightlyConfigMixin(metaclass=ABCMeta): archive_base_url = ARCHIVE_BASE_URL nightly_base_repo_name = "firefox" - nightly_repo = None - has_build_info = True def set_base_url(self, url): self.archive_base_url = url.rstrip("/") @@ -304,6 +303,16 @@ def get_nightly_info_url(self, url): """ return url + def get_nightly_changeset(self, archive_urls): + """ + Retrieve changeset info for a nightly build based on url. + + There is no universal way to determine changeset of a binary build for + all the supported products. Subclasses should override with their specific + approach if possible. + """ + return ChangesetInfo() + def get_nightly_repo(self, date): """ Returns the repo name for a given date. @@ -339,6 +348,13 @@ def _get_nightly_repo_regex(self, date, repo): ) return r"/%04d-%02d-%02d-[\d-]+%s/$" % (date.year, date.month, date.day, repo) + def get_nightly_timestamp_from_url(self, url): + """ + Extract the build timestamp from a build url. + """ + matches = re.search(r"/(\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2})-(.*)/", url) + return datetime.datetime.strptime(matches.group(1), "%Y-%m-%d-%H-%M-%S") + def can_go_integration(self): """ Indicate if we can bisect integration from this nightly config. @@ -346,7 +362,31 @@ def can_go_integration(self): return self.is_integration() -class FirefoxNightlyConfigMixin(NightlyConfigMixin): +class NightlyTxtConfigMixin(NightlyConfigMixin): + """ + Config mixin to use when a .txt file with changeset/repo info should exist. + + The meth:`get_nightly_info_url` and meth:`build_info_regex` methods of the + config are used to locate the appropriate build info .txt file. + """ + + def get_nightly_changeset(self, archive_urls): + return ChangesetInfo.from_nightly_txt(archive_urls) + + +class NightlyPushlogConfigMixin(NightlyConfigMixin): + """ + Config mixin when the source of changeset data is the pushlog. + + The meth:`get_nightly_timestamp_from_url` and meth:`get_nightly_repo` + methods of the config help generate the query. + """ + + def get_nightly_changeset(self, archive_urls): + return ChangesetInfo.from_pushlog(archive_urls, self) + + +class FirefoxNightlyConfigMixin(NightlyTxtConfigMixin): def _get_nightly_repo(self, date): if date < datetime.date(2008, 6, 17): return "trunk" @@ -354,8 +394,7 @@ def _get_nightly_repo(self, date): return "mozilla-central" -class FirefoxL10nNightlyConfigMixin(NightlyConfigMixin): - has_build_info = False +class FirefoxL10nNightlyConfigMixin(NightlyTxtConfigMixin): oldest_builds = datetime.date(2015, 10, 19) def _get_nightly_repo(self, date): @@ -370,7 +409,7 @@ def get_nightly_info_url(self, url): return url.replace("-l10n/", "/") -class ThunderbirdNightlyConfigMixin(NightlyConfigMixin): +class ThunderbirdNightlyConfigMixin(NightlyTxtConfigMixin): nightly_base_repo_name = "thunderbird" def _get_nightly_repo(self, date): @@ -390,7 +429,6 @@ def _get_nightly_repo(self, date): class ThunderbirdL10nNightlyConfigMixin(ThunderbirdNightlyConfigMixin): - has_build_info = False oldest_builds = datetime.date(2015, 10, 8) def _get_nightly_repo(self, date): @@ -400,8 +438,11 @@ def _get_nightly_repo(self, date): ) return "comm-central-l10n" + def get_nightly_info_url(self, url): + return url.replace("-l10n/", "/") -class FennecNightlyConfigMixin(NightlyConfigMixin): + +class FennecNightlyConfigMixin(NightlyTxtConfigMixin): nightly_base_repo_name = "mobile" def _get_nightly_repo(self, date): @@ -423,26 +464,35 @@ def get_nightly_repo_regex(self, date): return self._get_nightly_repo_regex(date, repo) -class FenixNightlyConfigMixin(NightlyConfigMixin): +class FenixNightlyConfigMixin(NightlyPushlogConfigMixin): + # https://archive.mozilla.org/pub/fenix/ nightly_base_repo_name = "fenix" - arch_regex_bits = "" def _get_nightly_repo(self, date): - return "fenix" + if date < datetime.date(2023, 2, 13): + # https://github.com/mozilla-mobile/fenix + return "fenix" + if date < datetime.date(2024, 3, 18): + # https://github.com/mozilla-mobile/firefox-android + return "firefox-android" + # https://hg.mozilla.org/mozilla-central/ + return "mozilla-central" def get_nightly_repo_regex(self, date): - repo = self.get_nightly_repo(date) - repo += self.arch_regex_bits # e.g., ".*arm64.*". + # Generated regex matches paths such as: + # 2022-06-10-17-01-58-fenix-103.0.0-android-x86_64 + # 2025-07-01-09-15-43-fenix-142.0a1-android + # + # Note: This scheme was the same regardless of which + # repo Fenix source code was in. + product = self.nightly_base_repo_name + version = r"[^-]+" + repo = f"{product}-{version}-android" + if self.arch: + repo += f"-{self.arch}" return self._get_nightly_repo_regex(date, repo) -class FocusNightlyConfigMixin(FenixNightlyConfigMixin): - nightly_base_repo_name = "focus" - - def _get_nightly_repo(self, date): - return "focus" - - class IntegrationConfigMixin(metaclass=ABCMeta): """ Define the integration-related required configuration. @@ -554,6 +604,21 @@ def tk_routes(self, push): return +class FenixIntegrationConfigMixin(IntegrationConfigMixin): + tk_name = "fenix" + + def tk_routes(self, push): + for build_type in self.build_types: + yield "gecko.v2.{}.revision.{}.mobile.{}-{}".format( + self.integration_branch, + push.changeset, + self.tk_name, + build_type, + ) + self._inc_used_build() + return + + class ThunderbirdIntegrationConfigMixin(IntegrationConfigMixin): default_integration_branch = "comm-central" @@ -626,10 +691,6 @@ class FirefoxConfig(CommonConfig, FirefoxNightlyConfigMixin, FirefoxIntegrationC "opt": ("shippable", "pgo"), } - def __init__(self, os, bits, processor, arch): - super(FirefoxConfig, self).__init__(os, bits, processor, arch) - self.set_build_type("shippable") - def build_regex(self): return ( get_build_regex( @@ -702,9 +763,17 @@ def available_bits(self): @REGISTRY.register("fenix") -class FenixConfig(CommonConfig, FenixNightlyConfigMixin): +class FenixConfig(CommonConfig, FenixNightlyConfigMixin, FenixIntegrationConfigMixin): + BUILD_TYPES = ("shippable",) + BUILD_TYPE_FALLBACKS = { + "shippable": ("nightly", "nightly-simulation"), + } + def build_regex(self): - return r"fenix-.+\.apk" + return r"(target.{}|fenix-.*)\.apk".format(self.arch) + + def build_info_regex(self): + return r"(?!)" # Match nothing def available_bits(self): return () @@ -717,25 +786,34 @@ def available_archs(self): "x86_64", ] - def set_arch(self, arch): - CommonConfig.set_arch(self, arch) - # Map "arch" value to one that can be used in the nightly repo regex lookup. - mapping = { - "arm64-v8a": "-.+-android-arm64-v8a", - "armeabi-v7a": "-.+-android-armeabi-v7a", - "x86": "-.+-android-x86", - "x86_64": "-.+-android-x86_64", - } - self.arch_regex_bits = mapping.get(self.arch, "") - def should_use_archive(self): return True @REGISTRY.register("focus") -class FocusConfig(FenixConfig, FocusNightlyConfigMixin): +class FocusConfig(FenixConfig): + BUILD_TYPE_FALLBACKS = { + "shippable": ("nightly",), + } + def build_regex(self): - return r"focus-.+\.apk" + return r"(target.{}|focus-.*)\.apk".format(self.arch) + + # https://archive.mozilla.org/pub/focus/ + nightly_base_repo_name = "focus" + + def _get_nightly_repo(self, date): + if date < datetime.date(2022, 12, 12): + # https://github.com/mozilla-mobile/focus-android + return "focus-android" + if date < datetime.date(2024, 3, 18): + # https://github.com/mozilla-mobile/firefox-android + return "firefox-android" + # https://hg.mozilla.org/mozilla-central/ + return "mozilla-central" + + # Index: gecko.v2.{repo}.revision.{rev}.mobile.focus-{build_type} + tk_name = "focus" @REGISTRY.register("gve") diff --git a/mozregression/json_pushes.py b/mozregression/json_pushes.py index 7e452b027..72cea5f87 100644 --- a/mozregression/json_pushes.py +++ b/mozregression/json_pushes.py @@ -155,3 +155,24 @@ def push(self, changeset, **kwargs): "No pushes available for the date %s on %s." % (changeset, self.branch) ) return self.pushes(changeset=changeset, **kwargs)[0] + + def push_by_timestamp(self, timestamp): + """ + Returns a list of Push objects for a timestamp. + + This will return at least one Push. In case of error it will raise + a MozRegressionError. + """ + + if not isinstance(timestamp, datetime.datetime): + raise TypeError() + + # The server interprets these as exclusive ranges, so shift + # them each by the minimum time unit. + dt = datetime.timedelta(seconds=1) + kwargs = { + "startdate": str(timestamp - dt), + "enddate": str(timestamp + dt), + } + + return self.pushes(**kwargs) diff --git a/tests/unit/test_fetch_build_info.py b/tests/unit/test_fetch_build_info.py index ebf71443a..0b9aaca8f 100644 --- a/tests/unit/test_fetch_build_info.py +++ b/tests/unit/test_fetch_build_info.py @@ -4,45 +4,42 @@ import re import unittest +import pytest from mock import Mock, patch from mozregression import errors, fetch_build_info, fetch_configs +from mozregression.fetch_build_info import ArchiveBuildUrls, ChangesetInfo from .test_fetch_configs import create_push -class TestInfoFetcher(unittest.TestCase): +class TestNightlyInfoFetcher(unittest.TestCase): def setUp(self): - fetch_config = fetch_configs.create_config("firefox", "linux", 64, "x86_64") - self.info_fetcher = fetch_build_info.InfoFetcher(fetch_config) + self.fetch_config = fetch_configs.create_config("firefox", "linux", 64, "x86_64") + self.info_fetcher = fetch_build_info.NightlyInfoFetcher(self.fetch_config) @patch("requests.get") def test__fetch_txt_info(self, get): response = Mock( - text="20141101030205\nhttps://hg.mozilla.org/\ -mozilla-central/rev/b695d9575654\n" + text="20141101030205\nhttps://hg.mozilla.org/mozilla-central/rev/b695d9575654\n" ) get.return_value = response - expected = { - "repository": "https://hg.mozilla.org/mozilla-central", - "changeset": "b695d9575654", - } - self.assertEqual(self.info_fetcher._fetch_txt_info("http://foo.txt"), expected) + + expected = ChangesetInfo( + "b695d9575654", + "https://hg.mozilla.org/mozilla-central", + ) + urls = ArchiveBuildUrls("http://build.tar.bz2", "http://foo.txt") + self.assertEqual(ChangesetInfo.from_nightly_txt(urls), expected) @patch("requests.get") def test__fetch_txt_info_old_format(self, get): response = Mock(text="20110126030333 e0fc18b3bc41\n") get.return_value = response - expected = { - "changeset": "e0fc18b3bc41", - } - self.assertEqual(self.info_fetcher._fetch_txt_info("http://foo.txt"), expected) - -class TestNightlyInfoFetcher(unittest.TestCase): - def setUp(self): - fetch_config = fetch_configs.create_config("firefox", "linux", 64, "x86_64") - self.info_fetcher = fetch_build_info.NightlyInfoFetcher(fetch_config) + urls = ArchiveBuildUrls("http://build.tar.bz2", "http://foo.txt") + expected = ChangesetInfo("e0fc18b3bc41") + self.assertEqual(ChangesetInfo.from_nightly_txt(urls), expected) @patch("mozregression.fetch_build_info.url_links") def test__find_build_info_from_url(self, url_links): @@ -52,10 +49,10 @@ def test__find_build_info_from_url(self, url_links): "http://foo/firefox01linux-x86_64.txt", "http://foo/firefox01linux-x86_64.tar.bz2", ] - expected = { - "build_txt_url": "http://foo/firefox01linux-x86_64.txt", - "build_url": "http://foo/firefox01linux-x86_64.tar.bz2", - } + expected = ArchiveBuildUrls( + "http://foo/firefox01linux-x86_64.tar.bz2", + "http://foo/firefox01linux-x86_64.txt", + ) builds = [] self.info_fetcher._fetch_build_info_from_url("http://foo", 0, builds) self.assertEqual(builds, [(0, expected)]) @@ -117,10 +114,11 @@ def my_find_build_info(url, index, lst): # say only the last build url is invalid if url in get_urls.return_value[:-1]: return - lst.append((index, {"build_txt_url": url, "build_url": url})) + fetch_info = ArchiveBuildUrls(url, url) + lst.append((index, fetch_info)) self.info_fetcher._fetch_build_info_from_url = Mock(side_effect=my_find_build_info) - self.info_fetcher._fetch_txt_info = Mock(return_value={}) + self.info_fetcher.fetch_config.get_nightly_changeset = Mock(return_value=ChangesetInfo()) result = self.info_fetcher.find_build_info(datetime.date(2014, 11, 15)) # we must have found the last build url valid self.assertEqual(result.build_url, get_urls.return_value[-1]) @@ -131,7 +129,7 @@ def test_find_build_info_no_data(self): self.info_fetcher.find_build_info(datetime.date(2014, 11, 15)) -class TestNightlyInfoFetcher2(unittest.TestCase): +class TestNightlyInfoFetcherWin(unittest.TestCase): def setUp(self): fetch_config = fetch_configs.create_config("firefox", "win", 64, "x86_64") self.info_fetcher = fetch_build_info.NightlyInfoFetcher(fetch_config) @@ -147,15 +145,101 @@ def test__find_build_info_from_url(self, url_links): "http://foo/firefox01win64.txt", "http://foo/firefox01win64.zip", ] - expected = { - "build_txt_url": "http://foo/firefox01win64.txt", - "build_url": "http://foo/firefox01win64.zip", - } + expected = ArchiveBuildUrls( + "http://foo/firefox01win64.zip", + "http://foo/firefox01win64.txt", + ) builds = [] self.info_fetcher._fetch_build_info_from_url("http://foo", 0, builds) self.assertEqual(builds, [(0, expected)]) +@pytest.mark.parametrize("app_name", ["firefox-l10n", "thunderbird-l10n"]) +class TestNightlyInfoFetcherL10N: + @patch("mozregression.fetch_build_info.retry_get") + @patch("mozregression.fetch_build_info.url_links") + def test_find_build_info(self, url_links, retry_get, app_name): + lang = "ar" + fetch_config = fetch_configs.create_config(app_name, "linux", 64, "x86_64") + fetch_config.set_lang(lang) + info_fetcher = fetch_build_info.NightlyInfoFetcher(fetch_config) + + if app_name == "firefox-l10n": + base_app = "firefox" + repo_name = "mozilla-central" + elif app_name == "thunderbird-l10n": + base_app = "thunderbird" + repo_name = "comm-central" + + version = "100.0a1" + l10n_suffix = "-l10n" + repo_url = f"https://hg.mozilla.org/{repo_name}" + + build_file = f"{base_app}-{version}.{lang}.linux-x86_64.tar.bz2" + txt_file = f"{base_app}-{version}.en-US.linux-x86_64.txt" + + l10n_url = ( + f"https://archive.mozilla.org/pub/{app_name}/nightly/" + f"2016/01/2016-01-01-10-02-05-{repo_name}{l10n_suffix}/" + ) + info_url = ( + f"https://archive.mozilla.org/pub/{base_app}/nightly/" + f"2016/01/2016-01-01-10-02-05-{repo_name}/" + ) + + def mock_url_links(url): + if "-l10n/" in url: + return [url + build_file] + else: + return [url + txt_file] + + info_fetcher._get_urls = Mock(return_value=[l10n_url]) + url_links.side_effect = mock_url_links + + txt_response = Mock(text=f"20160101100205\n{repo_url}/rev/abc123def456\n") + retry_get.return_value = txt_response + + result = info_fetcher.find_build_info(datetime.date(2016, 1, 1)) + + assert url_links.call_count == 2 + url_links.assert_any_call(l10n_url) + url_links.assert_any_call(info_url) + + retry_get.assert_called_once_with(info_url + txt_file) + + assert result.build_url == l10n_url + build_file + assert result.changeset == "abc123def456" + assert result.repo_url == repo_url + + +class TestFenixNightlyInfoFetch(unittest.TestCase): + def setUp(self): + self.fetch_config = fetch_configs.create_config("fenix", None, None, None, "arm64-v8a") + self.info_fetcher = fetch_build_info.NightlyInfoFetcher(self.fetch_config) + + @patch("mozregression.fetch_build_info.url_links") + def test__find_build_info_from_url(self, url_links): + """Test that _fetch_build_info_from_url returns ArchiveBuildUrls for Fenix builds.""" + build_folder = "2026-01-01-09-03-33-fenix-147.0a1-android-arm64-v8a" + build_file = "fenix-147.0a1.multi.android-arm64-v8a.apk" + build_url = f"http://foo/{build_folder}/" + + url_links.return_value = [build_url + build_file] + + builds = [] + self.info_fetcher._fetch_build_info_from_url(build_url, 0, builds) + + # Verify we got one build + self.assertEqual(len(builds), 1) + + # Verify the structure: (index, ArchiveBuildUrls) + index, archive_urls = builds[0] + self.assertEqual(index, 0) + self.assertIsInstance(archive_urls, ArchiveBuildUrls) + self.assertEqual(archive_urls.build_url, build_url + build_file) + self.assertIsNone(archive_urls.build_txt_url) # Fenix doesn't have .txt files + + class TestIntegrationInfoFetcher(unittest.TestCase): def setUp(self): self.fetch_config = fetch_configs.create_config("firefox", "linux", 64, "x86_64") @@ -180,7 +264,6 @@ def test_find_build_info(self, Queue, Index): "http://firefox-42.0a1.en-US.linux-x86_64.tar.bz2" ) self.info_fetcher = fetch_build_info.IntegrationInfoFetcher(self.fetch_config) - self.info_fetcher._fetch_txt_info = Mock(return_value={"changeset": "123456789"}) result = self.info_fetcher.find_build_info(create_push("123456789", 1)) self.assertEqual(result.build_url, "http://firefox-42.0a1.en-US.linux-x86_64.tar.bz2") @@ -255,7 +338,6 @@ def test_find_build_info(self, Queue, Index, requests_head): requests_head().status_code = 200 self.info_fetcher = fetch_build_info.IntegrationInfoFetcher(self.fetch_config) - self.info_fetcher._fetch_txt_info = Mock(return_value={"changeset": "123456789"}) result = self.info_fetcher.find_build_info(create_push("123456789", 1)) self.assertEqual(result.build_url, "http://geckoview_example.apk") @@ -282,7 +364,6 @@ def test_find_build_info_artifact_unavailable(self, Queue, Index, requests_head) requests_head().status_code = 403 self.info_fetcher = fetch_build_info.IntegrationInfoFetcher(self.fetch_config) - self.info_fetcher._fetch_txt_info = Mock(return_value={"changeset": "123456789"}) with self.assertRaises(errors.BuildInfoNotFound): self.info_fetcher.find_build_info(create_push("123456789", 1)) diff --git a/tests/unit/test_fetch_configs.py b/tests/unit/test_fetch_configs.py index bb0c50b14..21ef33c8b 100644 --- a/tests/unit/test_fetch_configs.py +++ b/tests/unit/test_fetch_configs.py @@ -278,6 +278,11 @@ def test_get_nightly_repo_regex(self, app_name): assert "mozilla-central-android-api-15" in regex regex = conf.get_nightly_repo_regex(datetime.date(2017, 8, 30)) assert "mozilla-central-android-api-16" in regex + elif app_name in ["fenix", "focus"]: + conf = create_config(app_name, None, None, None, "arm64-v8a") + date = datetime.date(2022, 1, 1) + regex = conf.get_nightly_repo_regex(date) + assert regex == f"/{date.isoformat()}-[\\d-]+{app_name}-[^-]+-android-arm64-v8a/$" else: conf = create_config(app_name, "linux", 64, None) date = datetime.date(2023, 1, 1) @@ -303,12 +308,58 @@ def setUp(self): self.conf = create_config("gve", "linux", 64, None) def test_fallbacking(self): - assert self.conf.build_type == "opt" - self.conf._inc_used_build() assert self.conf.build_type == "shippable" - # Check we wrap self.conf._inc_used_build() assert self.conf.build_type == "opt" + # Check we wrap + self.conf._inc_used_build() + assert self.conf.build_type == "shippable" + + +class TestFenixConfig(unittest.TestCase): + def setUp(self): + self.conf = create_config("fenix", None, None, None, "arm64-v8a") + + def test_get_nightly_changeset_uses_pushlog(self): + """Test that get_nightly_changeset for Fenix uses JsonPushes API.""" + from unittest.mock import Mock, patch + + from mozregression.fetch_build_info import ArchiveBuildUrls, ChangesetInfo + + # Create a sample build URL with timestamp + build_url = ( + "https://archive.mozilla.org/pub/fenix/nightly/2025/12/" + "2025-12-01-10-27-59-fenix-147.0a1-android-arm64-v8a/" + "fenix-147.0a1.multi.android-arm64-v8a.apk" + ) + archive_urls = ArchiveBuildUrls(build_url, None) + + # Mock JsonPushes to avoid network calls + with patch("mozregression.fetch_build_info.JsonPushes") as MockJsonPushes: + mock_jpushes_instance = Mock() + mock_jpushes_instance.repo_url = "https://hg.mozilla.org/mozilla-central" + mock_push = Mock() + mock_push.changeset = "abc123def456" + mock_jpushes_instance.push_by_timestamp.return_value = [mock_push] + MockJsonPushes.return_value = mock_jpushes_instance + + # Call get_nightly_changeset + result = self.conf.get_nightly_changeset(archive_urls) + + # Verify the result + assert isinstance(result, ChangesetInfo) + assert result.changeset == "abc123def456" + assert result.repo_url == "https://hg.mozilla.org/mozilla-central" + + # Verify JsonPushes was called + MockJsonPushes.assert_called_once_with(branch="mozilla-central") + mock_jpushes_instance.push_by_timestamp.assert_called_once() + + def test_build_regex(self): + assert re.match(self.conf.build_regex(), "fenix-148.0a1.multi.android-arm64-v8a.apk") + assert not re.match( + self.conf.build_info_regex(), "fenix-148.0a1.multi.android-arm64-v8a.txt" + ) class TestGetBuildUrl(unittest.TestCase): @@ -481,7 +532,7 @@ def test_aarch64_build_types(self): None, None, TIMESTAMP_FENNEC_API_15 - 1, - "gecko.v2.mozilla-central.revision.%s.mobile.android-api-11-opt" % CHSET, + "gecko.v2.mozilla-central.shippable.revision.%s.mobile.android-api-11-opt" % CHSET, ), ( "fennec", @@ -490,7 +541,7 @@ def test_aarch64_build_types(self): None, None, TIMESTAMP_FENNEC_API_15, - "gecko.v2.mozilla-central.revision.%s.mobile.android-api-15-opt" % CHSET, + "gecko.v2.mozilla-central.shippable.revision.%s.mobile.android-api-15-opt" % CHSET, ), ( "fennec", @@ -499,7 +550,7 @@ def test_aarch64_build_types(self): None, None, TIMESTAMP_FENNEC_API_16, - "gecko.v2.mozilla-central.revision.%s.mobile.android-api-16-opt" % CHSET, + "gecko.v2.mozilla-central.shippable.revision.%s.mobile.android-api-16-opt" % CHSET, ), # thunderbird ( @@ -627,6 +678,18 @@ def test_jsshell_aarch64_build_regex(): assert re.match(conf.build_regex(), "jsshell-win64-x86_64.zip") +def test_nightly_timestamp_parse(): + conf = create_config("fenix", None, None, None, "arm64-v8a") + build_url = ( + "https://archive.mozilla.org/pub/fenix/nightly/2025/12/" + "2025-12-01-10-27-59-fenix-147.0a1-android-arm64-v8a/" + "fenix-147.0a1.multi.android-arm64-v8a.apk" + ) + expected_dt = datetime.datetime(2025, 12, 1, 10, 27, 59) + + assert conf.get_nightly_timestamp_from_url(build_url) == expected_dt + + @pytest.mark.parametrize( "os,bits,processor,tc_suffix", [ diff --git a/tests/unit/test_json_pushes.py b/tests/unit/test_json_pushes.py index b8bc11ffe..bbbde7675 100644 --- a/tests/unit/test_json_pushes.py +++ b/tests/unit/test_json_pushes.py @@ -106,3 +106,22 @@ def test_push_with_date_raise_appropriate_error(): jpushes.push(date(2015, 1, 1)) assert str(ctx.value) == "No pushes available for the date 2015-01-01 on inbound." + + +def test_push_by_timestamp_builds_url(mocker): + timestamp = datetime(2025, 1, 2, 3, 4, 5) + pushlog = {"1": {"changesets": ["abc"], "date": 12345}} + + retry_get = mocker.patch("mozregression.json_pushes.retry_get") + retry_get.return_value = Mock(json=Mock(return_value=pushlog)) + + jpushes = JsonPushes(branch="mozilla-central") + pushes = jpushes.push_by_timestamp(timestamp) + + assert pushes[0].push_id == "1" + assert pushes[0].changeset == "abc" + expected_url = ( + "https://hg.mozilla.org/mozilla-central/json-pushes?" + "enddate=2025-01-02 03:04:06&startdate=2025-01-02 03:04:04" + ) + retry_get.assert_called_once_with(expected_url)