From bd4ce64cc51da9ac4bf8d7fb6b7ad59fe0b62e14 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 6 Jun 2018 00:18:25 +0100 Subject: [PATCH 01/14] lib/repo-pull: Retry pulls without static deltas if they fail If pulling a static delta fails (due to one part of it missing from a mirror, for example), retry the entire pull operation with disable-static-deltas=true. This is a functional, but ugly approach to fixing the problem, and a neater solution may be adopted upstream in future. If so, this commit should be dropped and replaced with the upstream solution: https://github.com/ostreedev/ostree/pull/1612 Signed-off-by: Philip Withnall https://phabricator.endlessm.com/T22873 Rebase 2018.6 (T23138): Fix a minor merge conflict due to "#ifdef OSTREE_ENABLE_EXPERIMENTAL_API" being gone now. --- src/libostree/ostree-repo-pull.c | 58 ++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 65b567891..e865973eb 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3706,13 +3706,13 @@ all_requested_refs_have_commit (GHashTable *requested_refs /* (element-type Ostr * * `disable-verify-bindings` (`b`): Disable verification of commit bindings. * Since: 2020.9 */ -gboolean -ostree_repo_pull_with_options (OstreeRepo *self, - const char *remote_name_or_baseurl, - GVariant *options, - OstreeAsyncProgress *progress, - GCancellable *cancellable, - GError **error) +static gboolean +ostree_repo_pull_with_options_internal (OstreeRepo *self, + const char *remote_name_or_baseurl, + GVariant *options, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; g_autoptr(GBytes) bytes_summary = NULL; @@ -5193,6 +5193,50 @@ ostree_repo_pull_with_options (OstreeRepo *self, return ret; } +gboolean +ostree_repo_pull_with_options (OstreeRepo *self, + const char *remote_name_or_baseurl, + GVariant *options, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GError) local_error = NULL; + + if (!ostree_repo_pull_with_options_internal (self, remote_name_or_baseurl, + options, progress, cancellable, + &local_error)) + { + gboolean disable_static_deltas = FALSE, require_static_deltas = FALSE; + + if (options != NULL) + { + g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas); + g_variant_lookup (options, "require-static-deltas", "b", &require_static_deltas); + } + + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND) && + !disable_static_deltas && !require_static_deltas) + { + g_autoptr(GVariant) modified_options = NULL; + g_auto(GVariantDict) modified_options_dict; + g_variant_dict_init (&modified_options_dict, options); + g_variant_dict_insert (&modified_options_dict, "disable-static-deltas", + "b", TRUE); + modified_options = g_variant_dict_end (&modified_options_dict); + + return ostree_repo_pull_with_options_internal (self, remote_name_or_baseurl, + modified_options, progress, + cancellable, error); + } + + g_propagate_error (error, g_steal_pointer (&local_error)); + return FALSE; + } + + return TRUE; +} + /* Structure used in ostree_repo_find_remotes_async() which stores metadata * about a given OSTree commit. This includes the metadata from the commit * #GVariant, plus some working state which is used to work out which remotes From 344474a916afda24a84932faf95a20578631d36c Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Tue, 29 Jan 2019 13:07:53 -0800 Subject: [PATCH 02/14] grub2: Disable use of grub-mkconfig on Endless OSTree recently gained support[1] for recognizing the Debian-style /boot/grub/grub.cfg path in addition to the existing support for the Red Hat-style /boot/grub2/grub.cfg. With this, it decides to run grub-mkconfig in ostree_sysroot_write_deployment_with_options() by calling ostree_bootloader_write_config(). However grub-mkconfig is disabled on Endless OS since the grub.cfg is hand-written and doesn't need updates[2] and therefore returns a non-zero exit status. This causes operations such as "ostree admin unlock --hotfix" and "ostree admin upgrade" to fail. So this commit effectively reverts the Debian grub support, by making _ostree_bootloader_grub2_query() always return TRUE with out_is_active set to FALSE, so that grub-mkconfig is not called and the operations above succeed. In the longer term, Endless should migrate to letting OSTree manage the grub config and drop this patch[3][4]. https://phabricator.endlessm.com/T25195 [1] https://github.com/ostreedev/ostree/commit/74bdf7e17 [2] https://phabricator.endlessm.com/T19614 [3] https://phabricator.endlessm.com/T14870 [4] https://phabricator.endlessm.com/T18848 --- src/libostree/ostree-bootloader-grub2.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libostree/ostree-bootloader-grub2.c b/src/libostree/ostree-bootloader-grub2.c index fff3b95c1..830c7a394 100644 --- a/src/libostree/ostree-bootloader-grub2.c +++ b/src/libostree/ostree-bootloader-grub2.c @@ -76,6 +76,16 @@ _ostree_bootloader_grub2_query (OstreeBootloader *bootloader, { OstreeBootloaderGrub2 *self = OSTREE_BOOTLOADER_GRUB2 (bootloader); + /* FIXME: Endless specific patch: don't let libostree find our + * /boot/grub/grub.cfg because we manage that separately and disable + * grub-mkconfig. See: + * - https://phabricator.endlessm.com/T19614 + * - https://phabricator.endlessm.com/T18848 */ + { + *out_is_active = FALSE; + return TRUE; + } + /* Look for the BIOS path first */ if (g_file_query_exists (self->config_path_bios_1, NULL) || g_file_query_exists (self->config_path_bios_2, NULL)) From e178b1c228c0a9dd1669f1b9f61ecc3afe233b62 Mon Sep 17 00:00:00 2001 From: Derek Foreman Date: Thu, 20 Jun 2019 17:28:00 -0500 Subject: [PATCH 03/14] deploy: Create fake symlinks when on FAT filesystems When trying to deploy on a vfat partitions, such as the EFI ESP, we fail to create the loader symlink, and deployment fails. We'd like to use sd-boot, which only works with the ESP, so we need a way to deploy without using symlinks. For a quick and dirty minimally invasive fix, I've made deployment create a bogus symlink - a text file named loader.sln that contains the name of the real loader dir. I've modified systemd-boot to read these bogus symlinks as well. https://phabricator.endlessm.com/T27040 Rebase 2022.1 (T32964): Handle conflicts with 02b61979 and 9a526bba. --- src/libostree/ostree-sysroot-deploy.c | 78 +++++++++++++++++++++++++-- src/libostree/ostree-sysroot.c | 27 ++++++---- 2 files changed, 89 insertions(+), 16 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index c4ae86d54..8f5ee5035 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -29,6 +29,8 @@ #include #include #include +#include +#include #ifdef HAVE_LIBMOUNT #include @@ -54,6 +56,35 @@ #define OSTREE_DEPLOYMENT_FINALIZING_ID SD_ID128_MAKE(e8,64,6c,d6,3d,ff,46,25,b7,79,09,a8,e7,a4,09,94) #endif +static gboolean +fs_is_fat (int parent_dfd) +{ + struct statfs buf; + + if (TEMP_FAILURE_RETRY (fstatfs (parent_dfd, &buf)) < 0) + return FALSE; + + if (buf.f_type == MSDOS_SUPER_MAGIC) + return TRUE; + + return FALSE; +} + +static gboolean +symlink_fat (const char *target, + int parent_dfd, + const char *linkpath, + GCancellable *cancellable, + GError **error) +{ + if (!glnx_file_replace_contents_at(parent_dfd, linkpath, (guint8*)target, + -1, GLNX_FILE_REPLACE_DATASYNC_NEW, + cancellable, error)) + return FALSE; + + return TRUE; +} + /* * Like symlinkat() but overwrites (atomically) an existing * symlink. @@ -65,6 +96,9 @@ symlink_at_replace (const char *oldpath, GCancellable *cancellable, GError **error) { + g_autofree char *dest_dir = g_path_get_dirname(newpath); + glnx_autofd int dest_dir_dfd = -1; + gboolean fat; /* Possibly in the future generate a temporary random name here, * would need to move "generate a temporary name" code into * libglnx or glib? @@ -74,12 +108,36 @@ symlink_at_replace (const char *oldpath, /* Clean up any stale temporary links */ (void) unlinkat (parent_dfd, temppath, 0); + /* Clean up any stale FAT "symlinks" */ + g_autofree char *temp_fat_path = g_strconcat (temppath, ".sln", NULL); + (void) unlinkat (parent_dfd, temp_fat_path, 0); + + if (!glnx_opendirat (parent_dfd, dest_dir, TRUE, &dest_dir_dfd, error)) + return FALSE; + + fat = fs_is_fat (dest_dir_dfd); + /* Create the temp link */ - if (TEMP_FAILURE_RETRY (symlinkat (oldpath, parent_dfd, temppath)) < 0) - return glnx_throw_errno_prefix (error, "symlinkat"); + if (!fat) + { + if (TEMP_FAILURE_RETRY (symlinkat (oldpath, parent_dfd, temppath)) < 0) + return glnx_throw_errno_prefix (error, "symlinkat"); + } + else + { + if (!symlink_fat (oldpath, parent_dfd, temp_fat_path, cancellable, error)) + return glnx_throw_errno_prefix (error, "symlinkat"); + } /* Rename it into place */ - if (!glnx_renameat (parent_dfd, temppath, parent_dfd, newpath, error)) + if (fat) + { + g_autofree char *new_fat_path = g_strconcat (newpath, ".sln", NULL); + if (!glnx_renameat (parent_dfd, temp_fat_path, parent_dfd, new_fat_path, error)) + return FALSE; + return TRUE; + } + else if (!glnx_renameat (parent_dfd, temppath, parent_dfd, newpath, error)) return FALSE; return TRUE; @@ -2073,12 +2131,22 @@ swap_bootloader (OstreeSysroot *sysroot, if (!_ostree_sysroot_ensure_boot_fd (sysroot, error)) return FALSE; + gboolean fat = fs_is_fat (sysroot->boot_fd); + /* The symlink was already written, and we used syncfs() to ensure * its data is in place. Renaming now should give us atomic semantics; * see https://bugzilla.gnome.org/show_bug.cgi?id=755595 */ - if (!glnx_renameat (sysroot->boot_fd, "loader.tmp", sysroot->boot_fd, "loader", error)) - return FALSE; + if (!fat) + { + if (!glnx_renameat (sysroot->boot_fd, "loader.tmp", sysroot->boot_fd, "loader", error)) + return FALSE; + } + else + { + if (!glnx_renameat (sysroot->boot_fd, "loader.tmp.sln", sysroot->boot_fd, "loader.sln", error)) + return FALSE; + } /* Now we explicitly fsync this directory, even though it * isn't required for atomicity, for two reasons: diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index eccf9375a..34b9e1680 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -621,33 +621,38 @@ read_current_bootversion (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - int ret_bootversion; + int ret_bootversion = 0; struct stat stbuf; + g_autofree char *target = NULL; if (!glnx_fstatat_allow_noent (self->sysroot_fd, "boot/loader", &stbuf, AT_SYMLINK_NOFOLLOW, error)) return FALSE; if (errno == ENOENT) { - g_debug ("Didn't find $sysroot/boot/loader symlink; assuming bootversion 0"); - ret_bootversion = 0; + /* Don't share the error because we want error handling to be the same as before we + * added fake symlinks */ + g_debug ("Didn't find $sysroot/boot/loader symlink; trying fake symlink"); + target = glnx_file_get_contents_utf8_at(self->sysroot_fd, "boot/loader.sln", NULL, cancellable, NULL); + if (!target) goto not_found; } else { if (!S_ISLNK (stbuf.st_mode)) return glnx_throw (error, "Not a symbolic link: boot/loader"); - - g_autofree char *target = + target = glnx_readlinkat_malloc (self->sysroot_fd, "boot/loader", cancellable, error); if (!target) return FALSE; - if (g_strcmp0 (target, "loader.0") == 0) - ret_bootversion = 0; - else if (g_strcmp0 (target, "loader.1") == 0) - ret_bootversion = 1; - else - return glnx_throw (error, "Invalid target '%s' in boot/loader", target); } + if (g_strcmp0 (target, "loader.0") == 0) + ret_bootversion = 0; + else if (g_strcmp0 (target, "loader.1") == 0) + ret_bootversion = 1; + else + return glnx_throw (error, "Invalid target '%s' in boot/loader", target); + +not_found: *out_bootversion = ret_bootversion; return TRUE; } From 60f4a586d44faab4550237fa4b474b928c923a54 Mon Sep 17 00:00:00 2001 From: Derek Foreman Date: Thu, 22 Aug 2019 15:04:19 -0500 Subject: [PATCH 04/14] deploy: Handle efi blobs We can now deploy our custom initramfs + kernel efi blobs. This will only happen when we're running in a PAYG system (ie: the kernel cmdline contains eospayg) or when we set the env var OSTREE_DEPLOY_PAYG to force PAYG deployment during image build. https://phabricator.endlessm.com/T27521 Rebase 2020.8 (T31593): Fix conflicts from upstream overlay_initrds feature and dropping flags argument to install_into_boot(). Added a couple more `!payg` to ensure other boot artifacts aren't installed. --- src/libostree/ostree-sysroot-deploy.c | 129 ++++++++++++++++++++------ 1 file changed, 99 insertions(+), 30 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 8f5ee5035..643e86948 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1049,6 +1049,8 @@ typedef struct { char *initramfs_namever; char *devicetree_srcpath; char *devicetree_namever; + char *efi_blob_srcpath; + char *efi_blob_namever; char *bootcsum; } OstreeKernelLayout; static void @@ -1063,6 +1065,8 @@ _ostree_kernel_layout_free (OstreeKernelLayout *layout) g_free (layout->initramfs_namever); g_free (layout->devicetree_srcpath); g_free (layout->devicetree_namever); + g_free (layout->efi_blob_srcpath); + g_free (layout->efi_blob_namever); g_free (layout->bootcsum); g_free (layout); } @@ -1226,6 +1230,15 @@ get_kernel_from_tree_usrlib_modules (OstreeSysroot *sysroot, g_clear_object (&in); glnx_close_fd (&fd); + if (!ot_openat_ignore_enoent (ret_layout->boot_dfd, "payg-image.efi", &fd, error)) + return FALSE; + if (fd != -1) + { + ret_layout->efi_blob_srcpath = g_strdup ("payg-image.efi"); + ret_layout->efi_blob_namever = g_strdup_printf ("payg-image-%s.efi", kver); + } + glnx_close_fd (&fd); + /* And finally, look for any HMAC file. This is needed for FIPS mode on some distros. */ if (!glnx_fstatat_allow_noent (ret_layout->boot_dfd, ".vmlinuz.hmac", NULL, 0, error)) return FALSE; @@ -1789,6 +1802,26 @@ parse_os_release (const char *contents, return ret; } +static gboolean +is_payg_deployment(GCancellable *cancellable) +{ + g_autoptr(GFile) cmdline_file = g_file_new_for_path ("/proc/cmdline"); + g_autofree char *cmdline = NULL; + gsize cmdline_len; + + if (g_getenv ("OSTREE_DEPLOY_PAYG")) + return TRUE; + + if (!g_file_load_contents (cmdline_file, cancellable, + &cmdline, &cmdline_len, NULL, NULL)) + return FALSE; + + if (g_strstr_len (cmdline, cmdline_len, "eospayg")) + return TRUE; + + return FALSE; +} + /* Given @deployment, prepare it to be booted; basically copying its * kernel/initramfs into /boot/ostree (if needed) and writing out an entry in * /boot/loader/entries. @@ -1804,6 +1837,8 @@ install_deployment_kernel (OstreeSysroot *sysroot, GError **error) { + gboolean payg; + GLNX_AUTO_PREFIX_ERROR ("Installing kernel", error); OstreeBootconfigParser *bootconfig = ostree_deployment_get_bootconfig (deployment); g_autofree char *deployment_dirpath = ostree_sysroot_get_deployment_dirpath (sysroot, deployment); @@ -1843,24 +1878,46 @@ install_deployment_kernel (OstreeSysroot *sysroot, if (!glnx_shutil_mkdir_p_at (sysroot->boot_fd, bootconfdir, 0775, cancellable, error)) return FALSE; + struct stat stbuf; + /* If this is a payg deployment, we want the efi blob and nothing else */ + payg = kernel_layout->efi_blob_srcpath && is_payg_deployment(cancellable); + /* If we're updating an old loader entry that doesn't use efi blobs + * keep it the way it was. */ + payg = payg && !ostree_bootconfig_parser_get (bootconfig, "linux"); + if (payg) + { + g_assert (kernel_layout->efi_blob_namever); + if (!glnx_fstatat_allow_noent (bootcsum_dfd, kernel_layout->efi_blob_namever, &stbuf, 0, error)) + return FALSE; + if (errno == ENOENT) + { + if (!install_into_boot (repo, sepolicy, kernel_layout->boot_dfd, kernel_layout->efi_blob_srcpath, + bootcsum_dfd, kernel_layout->efi_blob_namever, + cancellable, error)) + return FALSE; + } + } + /* Install (hardlink/copy) the kernel into /boot/ostree/osname-${bootcsum} if * it doesn't exist already. */ - struct stat stbuf; - if (!glnx_fstatat_allow_noent (bootcsum_dfd, kernel_layout->kernel_namever, &stbuf, 0, error)) - return FALSE; - if (errno == ENOENT) + if (!payg) { - if (!install_into_boot (repo, sepolicy, kernel_layout->boot_dfd, kernel_layout->kernel_srcpath, - bootcsum_dfd, kernel_layout->kernel_namever, - cancellable, error)) + if (!glnx_fstatat_allow_noent (bootcsum_dfd, kernel_layout->kernel_namever, &stbuf, 0, error)) return FALSE; + if (errno == ENOENT) + { + if (!install_into_boot (repo, sepolicy, kernel_layout->boot_dfd, kernel_layout->kernel_srcpath, + bootcsum_dfd, kernel_layout->kernel_namever, + cancellable, error)) + return FALSE; + } } /* If we have an initramfs, then install it into * /boot/ostree/osname-${bootcsum} if it doesn't exist already. */ - if (kernel_layout->initramfs_srcpath) + if (kernel_layout->initramfs_srcpath && !payg) { g_assert (kernel_layout->initramfs_namever); if (!glnx_fstatat_allow_noent (bootcsum_dfd, kernel_layout->initramfs_namever, &stbuf, 0, error)) @@ -1874,7 +1931,7 @@ install_deployment_kernel (OstreeSysroot *sysroot, } } - if (kernel_layout->devicetree_srcpath) + if (kernel_layout->devicetree_srcpath && !payg) { /* If devicetree_namever is set a single device tree is deployed */ if (kernel_layout->devicetree_namever) @@ -1897,7 +1954,7 @@ install_deployment_kernel (OstreeSysroot *sysroot, } } - if (kernel_layout->kernel_hmac_srcpath) + if (kernel_layout->kernel_hmac_srcpath && !payg) { if (!glnx_fstatat_allow_noent (bootcsum_dfd, kernel_layout->kernel_hmac_namever, &stbuf, 0, error)) return FALSE; @@ -1911,7 +1968,7 @@ install_deployment_kernel (OstreeSysroot *sysroot, } g_autoptr(GPtrArray) overlay_initrds = NULL; - for (char **it = _ostree_deployment_get_overlay_initrds (deployment); it && *it; it++) + for (char **it = _ostree_deployment_get_overlay_initrds (deployment); !payg && it && *it; it++) { char *checksum = *it; @@ -2019,39 +2076,45 @@ install_deployment_kernel (OstreeSysroot *sysroot, g_autofree char *version_key = g_strdup_printf ("%d", n_deployments - ostree_deployment_get_index (deployment)); ostree_bootconfig_parser_set (bootconfig, OSTREE_COMMIT_META_KEY_VERSION, version_key); - g_autofree char * boot_relpath = g_strconcat ("/", bootcsumdir, "/", kernel_layout->kernel_namever, NULL); - ostree_bootconfig_parser_set (bootconfig, "linux", boot_relpath); + + if (!payg) + { + g_autofree char * boot_relpath = g_strconcat ("/", bootcsumdir, "/", kernel_layout->kernel_namever, NULL); + ostree_bootconfig_parser_set (bootconfig, "linux", boot_relpath); + } val = ostree_bootconfig_parser_get (bootconfig, "options"); g_autoptr(OstreeKernelArgs) kargs = ostree_kernel_args_from_string (val); - if (kernel_layout->initramfs_namever) + if (!payg) { - g_autofree char * initrd_boot_relpath = - g_strconcat ("/", bootcsumdir, "/", kernel_layout->initramfs_namever, NULL); - ostree_bootconfig_parser_set (bootconfig, "initrd", initrd_boot_relpath); + if (kernel_layout->initramfs_namever) + { + g_autofree char * boot_relpath = g_strconcat ("/", bootcsumdir, "/", kernel_layout->initramfs_namever, NULL); + ostree_bootconfig_parser_set (bootconfig, "initrd", boot_relpath); - if (overlay_initrds) + if (overlay_initrds) + { + g_ptr_array_add (overlay_initrds, NULL); + ostree_bootconfig_parser_set_overlay_initrds (bootconfig, (char**)overlay_initrds->pdata); + } + } + else { - g_ptr_array_add (overlay_initrds, NULL); - ostree_bootconfig_parser_set_overlay_initrds (bootconfig, (char**)overlay_initrds->pdata); + g_autofree char *prepare_root_arg = NULL; + prepare_root_arg = g_strdup_printf ("init=/ostree/boot.%d/%s/%s/%d/usr/lib/ostree/ostree-prepare-root", + new_bootversion, osname, bootcsum, + ostree_deployment_get_bootserial (deployment)); + ostree_kernel_args_replace_take (kargs, g_steal_pointer (&prepare_root_arg)); } } - else - { - g_autofree char *prepare_root_arg = NULL; - prepare_root_arg = g_strdup_printf ("init=/ostree/boot.%d/%s/%s/%d/usr/lib/ostree/ostree-prepare-root", - new_bootversion, osname, bootcsum, - ostree_deployment_get_bootserial (deployment)); - ostree_kernel_args_replace_take (kargs, g_steal_pointer (&prepare_root_arg)); - } - if (kernel_layout->devicetree_namever) + if (kernel_layout->devicetree_namever && !payg) { g_autofree char * dt_boot_relpath = g_strconcat ("/", bootcsumdir, "/", kernel_layout->devicetree_namever, NULL); ostree_bootconfig_parser_set (bootconfig, "devicetree", dt_boot_relpath); } - else if (kernel_layout->devicetree_srcpath) + else if (kernel_layout->devicetree_srcpath && !payg) { /* If devicetree_srcpath is set but devicetree_namever is NULL, then we * want to point to a whole directory of device trees. @@ -2061,6 +2124,12 @@ install_deployment_kernel (OstreeSysroot *sysroot, ostree_bootconfig_parser_set (bootconfig, "fdtdir", dt_boot_relpath); } + if (payg) + { + g_autofree char * boot_relpath = g_strconcat ("/", bootcsumdir, "/", kernel_layout->efi_blob_namever, NULL); + ostree_bootconfig_parser_set (bootconfig, "efi", boot_relpath); + } + /* Note this is parsed in ostree-impl-system-generator.c */ g_autofree char *ostree_kernel_arg = g_strdup_printf ("ostree=/ostree/boot.%d/%s/%s/%d", new_bootversion, osname, bootcsum, From a8433f1204381f9b4a1b1f09ea17b35e4265f20c Mon Sep 17 00:00:00 2001 From: Derek Foreman Date: Mon, 22 Mar 2021 13:31:14 -0500 Subject: [PATCH 05/14] deploy: Don't try to create symlinks on FAT /boot Code was introduced to make a symlink from /boot/boot to sysroot, which breaks for us when /boot is FAT (like on PAYG systems) Instead of using our mock-symlink code here, just avoid writing the symlink at all, as it's only for u-boot and syslinux' benefit, neither of which are used in PAYG. https://phabricator.endlessm.com/T31593 --- src/libostree/ostree-sysroot-deploy.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 643e86948..e7f23cfdb 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -2166,11 +2166,18 @@ prepare_new_bootloader_link (OstreeSysroot *sysroot, g_assert ((current_bootversion == 0 && new_bootversion == 1) || (current_bootversion == 1 && new_bootversion == 0)); + /* Check if we're on a FAT FS, and unable to symlink */ + gboolean fat = FALSE; + glnx_autofd int dest_dir_dfd = -1; + if (glnx_opendirat (sysroot->sysroot_fd, "boot", TRUE, &dest_dir_dfd, error)) + fat = fs_is_fat (dest_dir_dfd); + /* This allows us to support both /boot on a seperate filesystem to / as well * as on the same filesystem. */ - if (TEMP_FAILURE_RETRY (symlinkat (".", sysroot->sysroot_fd, "boot/boot")) < 0) - if (errno != EEXIST) - return glnx_throw_errno_prefix (error, "symlinkat"); + if (!fat) + if (TEMP_FAILURE_RETRY (symlinkat (".", sysroot->sysroot_fd, "boot/boot")) < 0) + if (errno != EEXIST) + return glnx_throw_errno_prefix (error, "symlinkat"); g_autofree char *new_target = g_strdup_printf ("loader.%d", new_bootversion); From e20b372a3c18dea316d5d2573dec4b71e79da983 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Mon, 30 Jul 2018 16:51:01 +0100 Subject: [PATCH 06/14] Skip test-pull-repeated during CI This test is expected to fail a small proportion of the time. During the build of ostree 2018.7-1 in Debian, it seems we were unlucky on s390x. Non-deterministic tests are also problematic for autopkgtest, where they can gate migration of our dependencies like GLib, so skip this test unless the caller has opted-in to non-deterministic tests. It would be appropriate to enable this test in environments where failures can easily be retried and are not disruptive to other packages. Signed-off-by: Simon McVittie --- tests/test-pull-repeated.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test-pull-repeated.sh b/tests/test-pull-repeated.sh index 4c321618b..3b7d10b18 100755 --- a/tests/test-pull-repeated.sh +++ b/tests/test-pull-repeated.sh @@ -21,6 +21,10 @@ set -euo pipefail . $(dirname $0)/libtest.sh +if [ "${OSTREE_TEST_ALLOW_RANDOM:-}" != 1 ]; then + skip "Non-deterministic test will fail if we are unlucky" +fi + COMMIT_SIGN="" if has_gpgme; then COMMIT_SIGN="--gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1}" From f05a2607c83d890232b225d05a3e475639b29e01 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 4 Feb 2022 14:11:06 -0700 Subject: [PATCH 07/14] lib/repo: Add commit version metadata to summary metadata The commit metadata `version` key is well established but getting it for a remote commit is cumbersome since the commit object needs to be fetched and loaded. Including it in the summary additional metadata allows a much more convenient view of what each of the remote refs represents. (cherry picked from commit 6fbf759279ca9908e0c7ceb2a3af75b1180fa8a1) https://phabricator.endlessm.com/T33099 --- man/ostree-summary.xml | 1 + src/libostree/ostree-core.h | 2 ++ src/libostree/ostree-repo-private.h | 1 + src/libostree/ostree-repo.c | 7 ++++++- src/ostree/ot-dump.c | 5 +++++ tests/test-summary-view.sh | 1 + 6 files changed, 16 insertions(+), 1 deletion(-) diff --git a/man/ostree-summary.xml b/man/ostree-summary.xml index 4c9652ccb..e853e8cdf 100644 --- a/man/ostree-summary.xml +++ b/man/ostree-summary.xml @@ -183,6 +183,7 @@ License along with this library. If not, see . Latest Commit (4.2 MB): 9828ab80f357459b4ab50f0629beab2ae3b67318fc3d161d10a89fae353afa90 Timestamp (ostree.commit.timestamp): 2017-11-21T01:41:10-08 + Version (ostree.commit.version): 1.2.3 Last-Modified (ostree.summary.last-modified): 2018-01-12T22:06:38-08 diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 36e612909..48a75f92a 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -164,6 +164,8 @@ typedef enum { * The currently defined keys for the `a{sv}` of additional metadata for each commit are: * - key: `ostree.commit.timestamp`, value: `t`, timestamp (seconds since the * Unix epoch in UTC, big-endian) when the commit was committed + * - key: `ostree.commit.version`, value: `s`, the `version` value from the + * commit's metadata if it was defined. Since: 2022.2 */ #define OSTREE_SUMMARY_GVARIANT_STRING "(a(s(taya{sv}))a{sv})" #define OSTREE_SUMMARY_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_SUMMARY_GVARIANT_STRING) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 6d8f0193b..988c2179a 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -63,6 +63,7 @@ G_BEGIN_DECLS /* Well-known keys for the additional metadata field in a commit in a ref entry * in a summary file. */ #define OSTREE_COMMIT_TIMESTAMP "ostree.commit.timestamp" +#define OSTREE_COMMIT_VERSION "ostree.commit.version" typedef enum { OSTREE_REPO_TEST_ERROR_PRE_COMMIT = (1 << 0), diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 86948ee26..891e113df 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -6064,10 +6064,11 @@ summary_add_ref_entry (OstreeRepo *self, g_autoptr(GVariant) commit_obj = NULL; if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, checksum, &commit_obj, error)) return FALSE; + g_autoptr(GVariant) orig_metadata = g_variant_get_child_value (commit_obj, 0); g_variant_dict_init (&commit_metadata_builder, NULL); - /* Forward the commit’s timestamp if it’s valid. */ + /* Forward the commit’s timestamp and version if they're valid. */ guint64 commit_timestamp = ostree_commit_get_timestamp (commit_obj); g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (commit_timestamp); @@ -6075,6 +6076,10 @@ summary_add_ref_entry (OstreeRepo *self, g_variant_dict_insert_value (&commit_metadata_builder, OSTREE_COMMIT_TIMESTAMP, g_variant_new_uint64 (GUINT64_TO_BE (commit_timestamp))); + const char *version = NULL; + if (g_variant_lookup (orig_metadata, OSTREE_COMMIT_META_KEY_VERSION, "&s", &version)) + g_variant_dict_insert (&commit_metadata_builder, OSTREE_COMMIT_VERSION, "s", version); + g_variant_builder_add_value (refs_builder, g_variant_new ("(s(t@ay@a{sv}))", ref, (guint64) g_variant_get_size (commit_obj), diff --git a/src/ostree/ot-dump.c b/src/ostree/ot-dump.c index b874db0fe..509eb792d 100644 --- a/src/ostree/ot-dump.c +++ b/src/ostree/ot-dump.c @@ -249,6 +249,11 @@ dump_summary_ref (const char *collection_id, pretty_key = "Timestamp"; value_str = uint64_secs_to_iso8601 (GUINT64_FROM_BE (g_variant_get_uint64 (value))); } + else if (g_strcmp0 (key, OSTREE_COMMIT_VERSION) == 0) + { + pretty_key = "Version"; + value_str = g_strdup (g_variant_get_string (value, NULL)); + } else { value_str = g_variant_print (value, FALSE); diff --git a/tests/test-summary-view.sh b/tests/test-summary-view.sh index 088628d8a..9dfc74f44 100755 --- a/tests/test-summary-view.sh +++ b/tests/test-summary-view.sh @@ -56,6 +56,7 @@ assert_file_has_content_literal summary.txt "* main" assert_file_has_content_literal summary.txt "* other" assert_file_has_content_literal summary.txt "ostree.summary.last-modified" assert_file_has_content_literal summary.txt "Timestamp (ostree.commit.timestamp): " +assert_file_has_content_literal summary.txt "Version (ostree.commit.version): 3.2" echo "ok view summary" # Check the summary can be viewed raw too. From 739fb24595be429bce1667a399c782d7a48ba335 Mon Sep 17 00:00:00 2001 From: Phaedrus Leeds Date: Sat, 19 Feb 2022 07:55:02 -0600 Subject: [PATCH 08/14] Fix marking static delta commits as partial This patch makes it so that we mark the .commit file from a static delta as partial before writing the commit to the staging directory. This exactly mirrors what we do in meta_fetch_on_complete() when writing the commit on that codepath, which should lend some credibility to the correctness of this patch. I have checked that this fixes an issue Flatpak users have been encountering (https://github.com/flatpak/flatpak/issues/3479) which results in error messages like "error: Failed to install org.freedesktop.Sdk.Extension.texlive: Failed to read commit c7958d966cfa8b80a42877d1d6124831d7807f93c89461a2a586956aa28d438a: No such metadata object 8bdaa943b957f3cf14d19301c59c7eec076e57389e0fbb3ef5d30082e47a178f.dirtree" Here's the sequence of events that lead to the error: 1. An install operation is started that fetches static deltas. 2. The fetch is interrupted for some reason such as network connectivity dropping. 3. The .commit and .commitmeta files for the commit being pulled are left in the staging dir, e.g. "~/.local/share/flatpak/repo/tmp/staging-dfe862b2-13fc-49a2-ac92-5a59cc0d8e18-RURckd" 4. There is no `.commitpartial` file for the commit in "~/.local/share/flatpak/repo/state/" 5. The next time the user attempts the install, libostree reuses the existing staging dir, pulls the commit and commitmeta objects into the repo from the staging dir on the assumption that it's a complete commit. 6. Flatpak then tries to deploy the commit but fails in ostree_repo_read_commit() in flatpak_dir_deploy(), leading to the error message "Failed to read commit ..." 7. This happens again any subsequent time the user attempts the install, until the incomplete commit is removed with "flatpak repair --user". I will try to also add a workaround in Flatpak so this is fixed even when Flatpak links against affected versions of libostree. (cherry picked from commit 5d3b1ca37a508e9f80702b7ef7383fe95253ec6a) --- src/libostree/ostree-repo-pull.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index e865973eb..6ee8426d4 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2244,6 +2244,9 @@ process_one_static_delta (OtPullData *pull_data, ref, cancellable, error)) return FALSE; + if (!ostree_repo_mark_commit_partial (pull_data->repo, to_revision, TRUE, error)) + return FALSE; + if (detached_data && !ostree_repo_write_commit_detached_metadata (pull_data->repo, to_revision, detached_data, From a1d9e56328bc50bb6da608f6f581abffcc78539b Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 28 Jan 2022 11:08:00 +0000 Subject: [PATCH 09/14] libotutil: Avoid infinite recursion during error unwinding When we clean up from an error, for example copy_file_range() failing while we generate a static delta (perhaps caused by https://gitlab.gnome.org/GNOME/libglnx/-/issues/3 or by a genuine write error), we might free a variant builder that has a non-null parent. Previously, this caused infinite recursion and a stack overflow, repeatedly freeing the same object, but Luca Bruno suggested that the intention here appears to have been to free the parent object. Partially resolves https://github.com/ostreedev/ostree/issues/2525 (the other bug reported in that issue needs to be resolved by updating libglnx to a version where libglnx#3 has been fixed). Signed-off-by: Simon McVittie (cherry picked from commit 920f85cabc656e4a7c07574aa9af211b6153756d) --- src/libotutil/ot-variant-builder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libotutil/ot-variant-builder.c b/src/libotutil/ot-variant-builder.c index 92ac8edea..b7a7ec759 100644 --- a/src/libotutil/ot-variant-builder.c +++ b/src/libotutil/ot-variant-builder.c @@ -830,7 +830,7 @@ static void ot_variant_builder_info_free (OtVariantBuilderInfo *info) { if (info->parent) - ot_variant_builder_info_free (info); + ot_variant_builder_info_free (info->parent); g_variant_type_free (info->type); g_array_unref (info->child_ends); From a379455e9015f6e73818152de96c5946f5ee5bac Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Tue, 30 Aug 2022 08:38:36 -0600 Subject: [PATCH 10/14] main: Factor out sysroot loading It can be useful to parse the options and initialize the sysroot without actually loading it until later. Factor out the sysroot loading to a new `ostree_admin_sysroot_load` and add a new `OSTREE_ADMIN_BUILTIN_FLAG_NO_LOAD` flag to accommodate this. (cherry picked from commit e30a3b6b17c89f55c33e7985d11ccae7eb173507) --- src/ostree/ot-main.c | 83 ++++++++++++++++++++++++++------------------ src/ostree/ot-main.h | 6 ++++ 2 files changed, 55 insertions(+), 34 deletions(-) diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index 8ee730382..9ad357666 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -564,41 +564,11 @@ on_sysroot_journal_msg (OstreeSysroot *sysroot, } gboolean -ostree_admin_option_context_parse (GOptionContext *context, - const GOptionEntry *main_entries, - int *argc, - char ***argv, - OstreeAdminBuiltinFlags flags, - OstreeCommandInvocation *invocation, - OstreeSysroot **out_sysroot, - GCancellable *cancellable, - GError **error) +ostree_admin_sysroot_load (OstreeSysroot *sysroot, + OstreeAdminBuiltinFlags flags, + GCancellable *cancellable, + GError **error) { - /* Entries are listed in --help output in the order added. We add the - * main entries ourselves so that we can add the --sysroot entry first. */ - - g_option_context_add_main_entries (context, global_admin_entries, NULL); - - if (!ostree_option_context_parse (context, main_entries, argc, argv, - invocation, NULL, cancellable, error)) - return FALSE; - - if (!opt_print_current_dir && (flags & OSTREE_ADMIN_BUILTIN_FLAG_NO_SYSROOT)) - { - g_assert_null (out_sysroot); - /* Early return if no sysroot is requested */ - return TRUE; - } - - g_autoptr(GFile) sysroot_path = NULL; - if (opt_sysroot != NULL) - sysroot_path = g_file_new_for_path (opt_sysroot); - - g_autoptr(OstreeSysroot) sysroot = ostree_sysroot_new (sysroot_path); - if (!ostree_sysroot_initialize (sysroot, error)) - return FALSE; - g_signal_connect (sysroot, "journal-msg", G_CALLBACK (on_sysroot_journal_msg), NULL); - if ((flags & OSTREE_ADMIN_BUILTIN_FLAG_UNLOCKED) == 0) { /* If we're requested to lock the sysroot, first check if we're operating @@ -642,6 +612,51 @@ ostree_admin_option_context_parse (GOptionContext *context, } } + return TRUE; +} + +gboolean +ostree_admin_option_context_parse (GOptionContext *context, + const GOptionEntry *main_entries, + int *argc, + char ***argv, + OstreeAdminBuiltinFlags flags, + OstreeCommandInvocation *invocation, + OstreeSysroot **out_sysroot, + GCancellable *cancellable, + GError **error) +{ + /* Entries are listed in --help output in the order added. We add the + * main entries ourselves so that we can add the --sysroot entry first. */ + + g_option_context_add_main_entries (context, global_admin_entries, NULL); + + if (!ostree_option_context_parse (context, main_entries, argc, argv, + invocation, NULL, cancellable, error)) + return FALSE; + + if (!opt_print_current_dir && (flags & OSTREE_ADMIN_BUILTIN_FLAG_NO_SYSROOT)) + { + g_assert_null (out_sysroot); + /* Early return if no sysroot is requested */ + return TRUE; + } + + g_autoptr(GFile) sysroot_path = NULL; + if (opt_sysroot != NULL) + sysroot_path = g_file_new_for_path (opt_sysroot); + + g_autoptr(OstreeSysroot) sysroot = ostree_sysroot_new (sysroot_path); + if (!ostree_sysroot_initialize (sysroot, error)) + return FALSE; + g_signal_connect (sysroot, "journal-msg", G_CALLBACK (on_sysroot_journal_msg), NULL); + + if (opt_print_current_dir || (flags & OSTREE_ADMIN_BUILTIN_FLAG_NO_LOAD) == 0) + { + if (!ostree_admin_sysroot_load (sysroot, flags, cancellable, error)) + return FALSE; + } + if (opt_print_current_dir) { g_autoptr(GPtrArray) deployments = NULL; diff --git a/src/ostree/ot-main.h b/src/ostree/ot-main.h index b369deb87..e296501ae 100644 --- a/src/ostree/ot-main.h +++ b/src/ostree/ot-main.h @@ -36,6 +36,7 @@ typedef enum { OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER = (1 << 0), OSTREE_ADMIN_BUILTIN_FLAG_UNLOCKED = (1 << 1), OSTREE_ADMIN_BUILTIN_FLAG_NO_SYSROOT = (1 << 2), + OSTREE_ADMIN_BUILTIN_FLAG_NO_LOAD = (1 << 3), } OstreeAdminBuiltinFlags; @@ -91,6 +92,11 @@ gboolean ostree_admin_option_context_parse (GOptionContext *context, OstreeSysroot **out_sysroot, GCancellable *cancellable, GError **error); +gboolean ostree_admin_sysroot_load (OstreeSysroot *sysroot, + OstreeAdminBuiltinFlags flags, + GCancellable *cancellable, + GError **error); + gboolean ostree_ensure_repo_writable (OstreeRepo *repo, GError **error); void ostree_print_gpg_verify_result (OstreeGpgVerifyResult *result); From 97de3c2928bd1f1bfb5c5772fac5197b57804232 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 16 Feb 2022 15:58:58 -0700 Subject: [PATCH 11/14] finalize-staged: Ensure /boot automount doesn't expire If `/boot` is an automount, then the unit will be stopped as soon as the automount expires. That's would defeat the purpose of using systemd to delay finalizing the deployment until shutdown. This is not uncommon as `systemd-gpt-auto-generator` will create an automount unit for `/boot` when it's the EFI System Partition and there's no fstab entry. To ensure that systemd doesn't stop the service early when the `/boot` automount expires, introduce a new unit that holds `/boot` open until it's sent `SIGTERM`. This uses a new `--hold` option for `finalize-staged` that loads but doesn't lock the sysroot. A separate unit is used since we want the process to remain active throughout the finalization run in `ExecStop`. That wouldn't work if it was specified in `ExecStart` in the same unit since it would be killed before the `ExecStop` action was run. Fixes: #2543 (cherry picked from commit f3db79e7fa8d469e539b60ceb7e3d790747e530f) https://phabricator.endlessm.com/T33775 --- Makefile-boot.am | 2 + src/boot/ostree-finalize-staged-hold.service | 35 ++++++++ src/boot/ostree-finalize-staged.service | 5 ++ src/ostree/ot-admin-builtin-finalize-staged.c | 68 +++++++++++++-- tests/inst/src/destructive.rs | 7 +- tests/kolainst/destructive/boot-automount.sh | 83 +++++++++++++++++++ 6 files changed, 193 insertions(+), 7 deletions(-) create mode 100644 src/boot/ostree-finalize-staged-hold.service create mode 100755 tests/kolainst/destructive/boot-automount.sh diff --git a/Makefile-boot.am b/Makefile-boot.am index ec10a0d64..79de63e24 100644 --- a/Makefile-boot.am +++ b/Makefile-boot.am @@ -40,6 +40,7 @@ systemdsystemunit_DATA = src/boot/ostree-prepare-root.service \ src/boot/ostree-remount.service \ src/boot/ostree-finalize-staged.service \ src/boot/ostree-finalize-staged.path \ + src/boot/ostree-finalize-staged-hold.service \ $(NULL) systemdtmpfilesdir = $(prefix)/lib/tmpfiles.d dist_systemdtmpfiles_DATA = src/boot/ostree-tmpfiles.conf @@ -68,6 +69,7 @@ EXTRA_DIST += src/boot/dracut/module-setup.sh \ src/boot/ostree-finalize-staged.path \ src/boot/ostree-remount.service \ src/boot/ostree-finalize-staged.service \ + src/boot/ostree-finalize-staged-hold.service \ src/boot/grub2/grub2-15_ostree \ src/boot/grub2/ostree-grub-generator \ $(NULL) diff --git a/src/boot/ostree-finalize-staged-hold.service b/src/boot/ostree-finalize-staged-hold.service new file mode 100644 index 000000000..85b5d543b --- /dev/null +++ b/src/boot/ostree-finalize-staged-hold.service @@ -0,0 +1,35 @@ +# Copyright (C) 2018 Red Hat, Inc. +# Copyright (C) 2022 Endless OS Foundation LLC +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +# See https://github.com/ostreedev/ostree/pull/2543 for background. +[Unit] +Description=Hold /boot Open for OSTree Finalize Staged Deployment +Documentation=man:ostree(1) +ConditionPathExists=/run/ostree-booted +DefaultDependencies=no + +RequiresMountsFor=/sysroot /boot +After=local-fs.target +Before=basic.target final.target + +[Service] +Type=exec + +# This is explicitly run in the root namespace to ensure an automounted +# /boot doesn't time out since autofs doesn't handle mount namespaces. +# +# https://bugzilla.redhat.com/show_bug.cgi?id=2056090 +ExecStart=+/usr/bin/ostree admin finalize-staged --hold diff --git a/src/boot/ostree-finalize-staged.service b/src/boot/ostree-finalize-staged.service index 2f28bbb70..63621ce19 100644 --- a/src/boot/ostree-finalize-staged.service +++ b/src/boot/ostree-finalize-staged.service @@ -29,6 +29,11 @@ Before=basic.target final.target After=systemd-journal-flush.service Conflicts=final.target +# Start the hold unit and ensure it stays active throughout this +# service. +Wants=ostree-finalize-staged-hold.service +After=ostree-finalize-staged-hold.service + [Service] Type=oneshot RemainAfterExit=yes diff --git a/src/ostree/ot-admin-builtin-finalize-staged.c b/src/ostree/ot-admin-builtin-finalize-staged.c index 80d8a9b79..1d6b53efc 100644 --- a/src/ostree/ot-admin-builtin-finalize-staged.c +++ b/src/ostree/ot-admin-builtin-finalize-staged.c @@ -1,5 +1,6 @@ /* * Copyright (C) 2018 Red Hat, Inc. + * Copyright (C) 2022 Endless OS Foundation LLC * * SPDX-License-Identifier: LGPL-2.0+ * @@ -19,8 +20,9 @@ #include "config.h" -#include "config.h" - +#include +#include +#include #include #include "ot-main.h" @@ -32,10 +34,23 @@ #include "ostree-cmdprivate.h" #include "ostree.h" +static gboolean opt_hold; + static GOptionEntry options[] = { + { "hold", 0, 0, G_OPTION_ARG_NONE, &opt_hold, "Hold /boot open during finalization", NULL }, { NULL } }; +static gboolean +sigterm_cb (gpointer user_data) +{ + gboolean *running = user_data; + g_print ("Received SIGTERM, exiting\n"); + *running = FALSE; + g_main_context_wakeup (NULL); + return G_SOURCE_REMOVE; +} + /* Called by ostree-finalize-staged.service, and in turn * invokes a cmdprivate function inside the shared library. */ @@ -50,13 +65,56 @@ ot_admin_builtin_finalize_staged (int argc, char **argv, OstreeCommandInvocation g_autoptr(GOptionContext) context = g_option_context_new (""); g_autoptr(OstreeSysroot) sysroot = NULL; + + /* First parse the args without loading the sysroot to see what options are + * set. */ if (!ostree_admin_option_context_parse (context, options, &argc, &argv, - OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER, + OSTREE_ADMIN_BUILTIN_FLAG_NO_LOAD, invocation, &sysroot, cancellable, error)) return FALSE; - if (!ostree_cmd__private__()->ostree_finalize_staged (sysroot, cancellable, error)) - return FALSE; + if (opt_hold) + { + /* Load the sysroot unlocked so that a separate namespace isn't + * created. */ + if (!ostree_admin_sysroot_load (sysroot, + OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER | OSTREE_ADMIN_BUILTIN_FLAG_UNLOCKED, + cancellable, error)) + return FALSE; + + /* In case it's an automount, open /boot so that the automount doesn't + * expire until before this process exits. If it did expire and got + * unmounted, the service would be stopped and the deployment would be + * finalized earlier than expected. + */ + int sysroot_fd = ostree_sysroot_get_fd (sysroot); + glnx_autofd int boot_fd = -1; + g_debug ("Opening /boot directory"); + if (!glnx_opendirat (sysroot_fd, "boot", TRUE, &boot_fd, error)) + return FALSE; + + /* We want to keep /boot open until the deployment is finalized during + * system shutdown, so block on SIGTERM under the assumption that it will + * be received when systemd stops the unit. + */ + gboolean running = TRUE; + g_unix_signal_add (SIGTERM, sigterm_cb, &running); + g_print ("Waiting for SIGTERM\n"); + while (running) + g_main_context_iteration (NULL, TRUE); + } + else + { + /* Load the sysroot with the normal flags and actually finalize the + * deployment. */ + if (!ostree_admin_sysroot_load (sysroot, + OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER, + cancellable, error)) + return FALSE; + + if (!ostree_cmd__private__()->ostree_finalize_staged (sysroot, cancellable, error)) + return FALSE; + } return TRUE; } diff --git a/tests/inst/src/destructive.rs b/tests/inst/src/destructive.rs index 2e2bd374b..41fc53326 100644 --- a/tests/inst/src/destructive.rs +++ b/tests/inst/src/destructive.rs @@ -463,6 +463,7 @@ fn impl_transaction_test>( " systemctl stop rpm-ostreed systemctl stop ostree-finalize-staged + systemctl stop ostree-finalize-staged-hold ostree reset testrepo:{testref} {booted_commit} rpm-ostree cleanup -pbrm ", @@ -504,7 +505,8 @@ fn impl_transaction_test>( InterruptStrategy::Force(ForceInterruptStrategy::Kill9) => { bash!( "systemctl kill -s KILL rpm-ostreed || true - systemctl kill -s KILL ostree-finalize-staged || true" + systemctl kill -s KILL ostree-finalize-staged || true + systemctl kill -s KILL ostree-finalize-staged-hold || true" )?; live_strategy = Some(strategy); } @@ -528,7 +530,8 @@ fn impl_transaction_test>( InterruptStrategy::Polite(PoliteInterruptStrategy::Stop) => { bash!( "systemctl stop rpm-ostreed || true - systemctl stop ostree-finalize-staged || true" + systemctl stop ostree-finalize-staged || true + systemctl stop ostree-finalize-staged-hold || true" )?; live_strategy = Some(strategy); } diff --git a/tests/kolainst/destructive/boot-automount.sh b/tests/kolainst/destructive/boot-automount.sh new file mode 100755 index 000000000..d6d1732e7 --- /dev/null +++ b/tests/kolainst/destructive/boot-automount.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# https://github.com/ostreedev/ostree/issues/2543 +set -xeuo pipefail + +. ${KOLA_EXT_DATA}/libinsttest.sh + +case "${AUTOPKGTEST_REBOOT_MARK:-}" in + "") + # Ensure boot is mount point + mountpoint /boot + + # Create an automount unit with an extremely short timeout + cat > /etc/systemd/system/boot.automount <<"EOF" +[Automount] +Where=/boot +TimeoutIdleSec=1 + +[Install] +WantedBy=local-fs.target +EOF + systemctl daemon-reload + systemctl enable boot.automount + + # Unmount /boot, start the automount unit, and ensure the units are + # in the correct states. + umount /boot + systemctl start boot.automount + boot_state=$(systemctl show -P ActiveState boot.mount) + boot_auto_state=$(systemctl show -P ActiveState boot.automount) + assert_streq "${boot_state}" inactive + assert_streq "${boot_auto_state}" active + + # Trigger a new staged deployment and check that the relevant units + # are enabled. + ostree admin deploy --stage --karg-append=somedummykarg=1 "${host_commit}" + rpm-ostree status --json + deployment_staged=$(rpmostree_query_json '.deployments[0].staged') + assert_streq "${deployment_staged}" true + test -f /run/ostree/staged-deployment + finalize_staged_state=$(systemctl show -P ActiveState ostree-finalize-staged.service) + finalize_staged_hold_state=$(systemctl show -P ActiveState ostree-finalize-staged-hold.service) + assert_streq "${finalize_staged_state}" active + assert_streq "${finalize_staged_hold_state}" active + + # Sleep 1 second to ensure the boot automount idle timeout has + # passed and then check that /boot is still mounted. + sleep 1 + boot_state=$(systemctl show -P ActiveState boot.mount) + assert_streq "${boot_state}" active + + /tmp/autopkgtest-reboot "2" + ;; + "2") + rpm-ostree status --json + deployment_staged=$(rpmostree_query_json '.deployments[0].staged') + assert_streq "${deployment_staged}" false + test ! -f /run/ostree/staged-deployment + assert_file_has_content_literal /proc/cmdline somedummykarg=1 + + # Check that the finalize and hold services succeeded in the + # previous boot. Dump them to the test log to help debugging. + prepare_tmpdir + journalctl -b -1 -o short-monotonic \ + -u ostree-finalize-staged.service \ + -u ostree-finalize-staged-hold.service \ + -u boot.mount \ + -u boot.automount \ + > logs.txt + cat logs.txt + assert_file_has_content logs.txt 'ostree-finalize-staged.service: \(Succeeded\|Deactivated successfully\)' + assert_file_has_content logs.txt 'ostree-finalize-staged-hold.service: \(Succeeded\|Deactivated successfully\)' + + # Check that the hold service remained active and kept /boot mounted until + # the finalize service completed. + finalize_stopped=$(journalctl -b -1 -o json -g Stopped -u ostree-finalize-staged.service | tail -n1 | jq -r .__MONOTONIC_TIMESTAMP) + hold_stopping=$(journalctl -b -1 -o json -g Stopping -u ostree-finalize-staged-hold.service | tail -n1 | jq -r .__MONOTONIC_TIMESTAMP) + hold_stopped=$(journalctl -b -1 -o json -g Stopped -u ostree-finalize-staged-hold.service | tail -n1 | jq -r .__MONOTONIC_TIMESTAMP) + boot_unmounting=$(journalctl -b -1 -o json -g Unmounting -u boot.mount | tail -n1 | jq -r .__MONOTONIC_TIMESTAMP) + test "${finalize_stopped}" -lt "${hold_stopping}" + test "${hold_stopped}" -lt "${boot_unmounting}" + ;; + *) fatal "Unexpected AUTOPKGTEST_REBOOT_MARK=${AUTOPKGTEST_REBOOT_MARK}" ;; +esac From b9bb127571dfaafd45df6e4db3c38e59742d43f8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 30 Aug 2022 16:23:38 -0400 Subject: [PATCH 12/14] finalize-staged: Don't listen to `SIGTERM`, just let kernel exit us Followup from discussion in https://github.com/ostreedev/ostree/pull/2544#discussion_r958840936 This is more efficient; no need to have the kernel context switch us in at shutdown time just so we can turn around and call `exit()`. (cherry picked from commit 683e4eff08d668f5a5c0cef1f38797ebfff36624) --- src/ostree/ot-admin-builtin-finalize-staged.c | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/src/ostree/ot-admin-builtin-finalize-staged.c b/src/ostree/ot-admin-builtin-finalize-staged.c index 1d6b53efc..ac08c21be 100644 --- a/src/ostree/ot-admin-builtin-finalize-staged.c +++ b/src/ostree/ot-admin-builtin-finalize-staged.c @@ -41,16 +41,6 @@ static GOptionEntry options[] = { { NULL } }; -static gboolean -sigterm_cb (gpointer user_data) -{ - gboolean *running = user_data; - g_print ("Received SIGTERM, exiting\n"); - *running = FALSE; - g_main_context_wakeup (NULL); - return G_SOURCE_REMOVE; -} - /* Called by ostree-finalize-staged.service, and in turn * invokes a cmdprivate function inside the shared library. */ @@ -94,14 +84,10 @@ ot_admin_builtin_finalize_staged (int argc, char **argv, OstreeCommandInvocation return FALSE; /* We want to keep /boot open until the deployment is finalized during - * system shutdown, so block on SIGTERM under the assumption that it will - * be received when systemd stops the unit. + * system shutdown, so block until we get SIGTERM which systemd will send + * when the unit is stopped. */ - gboolean running = TRUE; - g_unix_signal_add (SIGTERM, sigterm_cb, &running); - g_print ("Waiting for SIGTERM\n"); - while (running) - g_main_context_iteration (NULL, TRUE); + pause (); } else { From 6aa9cbc7d97929a41639973807030911574b30d6 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Mon, 5 Dec 2022 09:19:27 -0700 Subject: [PATCH 13/14] Revert "grub2: Disable use of grub-mkconfig on Endless" This reverts commit 344474a916afda24a84932faf95a20578631d36c. All systems have been converted to declare the desired bootloader in the repository configuration `sysroot.bootloader` option. On grub systems this is set to `none` so that only the boot loader spec entries are updated. This and the reverted commit should be dropped during the next upstream rebase. https://phabricator.endlessm.com/T34161 --- src/libostree/ostree-bootloader-grub2.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/libostree/ostree-bootloader-grub2.c b/src/libostree/ostree-bootloader-grub2.c index 830c7a394..fff3b95c1 100644 --- a/src/libostree/ostree-bootloader-grub2.c +++ b/src/libostree/ostree-bootloader-grub2.c @@ -76,16 +76,6 @@ _ostree_bootloader_grub2_query (OstreeBootloader *bootloader, { OstreeBootloaderGrub2 *self = OSTREE_BOOTLOADER_GRUB2 (bootloader); - /* FIXME: Endless specific patch: don't let libostree find our - * /boot/grub/grub.cfg because we manage that separately and disable - * grub-mkconfig. See: - * - https://phabricator.endlessm.com/T19614 - * - https://phabricator.endlessm.com/T18848 */ - { - *out_is_active = FALSE; - return TRUE; - } - /* Look for the BIOS path first */ if (g_file_query_exists (self->config_path_bios_1, NULL) || g_file_query_exists (self->config_path_bios_2, NULL)) From 95e8bf9309afee6a5a9c844f29c3c146c89935d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Feb 2023 13:06:59 +0000 Subject: [PATCH 14/14] build(deps): bump libglnx from `ef502aa` to `07e3e49` Bumps libglnx from `ef502aa` to `07e3e49`. --- updated-dependencies: - dependency-name: libglnx dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- libglnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libglnx b/libglnx index ef502aabf..07e3e49d3 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit ef502aabf7d3a0d37f9c4d228f870ac93404447b +Subproject commit 07e3e49d3e47dfd4265ffb5495111439131715ca