cros_sdk: Drop --use-image support
We've phased out all users (deprecation notice for a while), and
builders are all passing no_use_image=False into the API.
BUG=b:266878468
TEST=./run_tests \
service/sdk_unittest.py \
api/controller/sdk_unittest.py \
scripts/cros_sdk_unittest.py
Change-Id: I0920b0b5233a029dc6928ccafe9b488e76b9303a
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/4304878
Commit-Queue: Brian Norris <briannorris@chromium.org>
Reviewed-by: Mike Frysinger <vapier@chromium.org>
Tested-by: Brian Norris <briannorris@chromium.org>
diff --git a/scripts/cros_sdk.py b/scripts/cros_sdk.py
index edb87b4..ad4e223 100644
--- a/scripts/cros_sdk.py
+++ b/scripts/cros_sdk.py
@@ -17,10 +17,8 @@
import os
from pathlib import Path
import pwd
-import random
import re
import shlex
-import subprocess
import sys
from typing import List
import urllib.parse
@@ -66,30 +64,6 @@
# Tools needed for --proxy-sim only.
PROXY_NEEDED_TOOLS = ("ip",)
-# Tools needed when use_image is true.
-IMAGE_NEEDED_TOOLS = (
- "losetup",
- "lvchange",
- "lvcreate",
- "lvs",
- "mke2fs",
- "pvscan",
- "thin_check",
- "vgchange",
- "vgcreate",
- "vgs",
-)
-
-# As space is used inside the chroot, the empty space in chroot.img is
-# allocated. Deleting files inside the chroot doesn't automatically return the
-# used space to the OS. Over time, this tends to make the sparse chroot.img
-# less sparse even if the chroot contents don't currently need much space. We
-# can recover most of this unused space with fstrim, but that takes too much
-# time to run it every time. Instead, check the used space against the image
-# size after mounting the chroot and only call fstrim if it looks like we could
-# recover at least this many GiB.
-MAX_UNUSED_IMAGE_GBS = 20
-
def GetArchStageTarballs(version):
"""Returns the URL for a given arch/version"""
@@ -189,250 +163,6 @@
return tarball_dest
-def _ImageFileForChroot(chroot):
- """Find the image file that should be associated with |chroot|.
-
- This function does not check if the image exists; it simply returns the
- filename that would be used.
-
- Args:
- chroot: Path to the chroot.
-
- Returns:
- Path to an image file that would be associated with chroot.
- """
- return chroot.rstrip("/") + ".img"
-
-
-def CreateChrootSnapshot(snapshot_name, chroot_vg, chroot_lv):
- """Create a snapshot for the specified chroot VG/LV.
-
- Args:
- snapshot_name: The name of the new snapshot.
- chroot_vg: The name of the VG containing the origin LV.
- chroot_lv: The name of the origin LV.
-
- Returns:
- True if the snapshot was created, or False if a snapshot with the same
- name already exists.
-
- Raises:
- SystemExit: The lvcreate command failed.
- """
- if snapshot_name in ListChrootSnapshots(chroot_vg, chroot_lv):
- logging.error(
- "Cannot create snapshot %s: A volume with that name already "
- "exists.",
- snapshot_name,
- )
- return False
-
- cmd = [
- "lvcreate",
- "-s",
- "--name",
- snapshot_name,
- "%s/%s" % (chroot_vg, chroot_lv),
- ]
- try:
- logging.notice(
- "Creating snapshot %s from %s in VG %s.",
- snapshot_name,
- chroot_lv,
- chroot_vg,
- )
- cros_build_lib.dbg_run(cmd, capture_output=True)
- return True
- except cros_build_lib.RunCommandError as e:
- cros_build_lib.Die("Creating snapshot failed!\n%s", e)
-
-
-def DeleteChrootSnapshot(snapshot_name, chroot_vg, chroot_lv):
- """Delete the named snapshot from the specified chroot VG.
-
- If the requested snapshot is not found, nothing happens. The main chroot LV
- and internal thinpool LV cannot be deleted with this function.
-
- Args:
- snapshot_name: The name of the snapshot to delete.
- chroot_vg: The name of the VG containing the origin LV.
- chroot_lv: The name of the origin LV.
-
- Raises:
- SystemExit: The lvremove command failed.
- """
- if snapshot_name in (
- cros_sdk_lib.CHROOT_LV_NAME,
- cros_sdk_lib.CHROOT_THINPOOL_NAME,
- ):
- logging.error(
- "Cannot remove LV %s as a snapshot. Use cros_sdk --delete "
- "if you want to remove the whole chroot.",
- snapshot_name,
- )
- return
-
- if snapshot_name not in ListChrootSnapshots(chroot_vg, chroot_lv):
- return
-
- cmd = ["lvremove", "-f", "%s/%s" % (chroot_vg, snapshot_name)]
- try:
- logging.notice(
- "Deleting snapshot %s in VG %s.", snapshot_name, chroot_vg
- )
- cros_build_lib.dbg_run(cmd, capture_output=True)
- except cros_build_lib.RunCommandError as e:
- cros_build_lib.Die("Deleting snapshot failed!\n%s", e)
-
-
-def RestoreChrootSnapshot(snapshot_name, chroot_vg, chroot_lv):
- """Restore the chroot to an existing snapshot.
-
- This is done by renaming the original |chroot_lv| LV to a temporary name,
- renaming the snapshot named |snapshot_name| to |chroot_lv|, and deleting the
- now unused LV. If an error occurs, attempts to rename the original snapshot
- back to |chroot_lv| to leave the chroot unchanged.
-
- The chroot must be unmounted before calling this function, and will be left
- unmounted after this function returns.
-
- Args:
- snapshot_name: The name of the snapshot to restore. This snapshot will
- no longer be accessible at its original name after this function
- finishes.
- chroot_vg: The VG containing the chroot LV and snapshot LV.
- chroot_lv: The name of the original chroot LV.
-
- Returns:
- True if the chroot was restored to the requested snapshot, or False if
- the snapshot wasn't found or isn't valid.
-
- Raises:
- SystemExit: Any of the LVM commands failed.
- """
- valid_snapshots = ListChrootSnapshots(chroot_vg, chroot_lv)
- if (
- snapshot_name
- in (cros_sdk_lib.CHROOT_LV_NAME, cros_sdk_lib.CHROOT_THINPOOL_NAME)
- or snapshot_name not in valid_snapshots
- ):
- logging.error(
- "Chroot cannot be restored to %s. Valid snapshots: %s",
- snapshot_name,
- ", ".join(valid_snapshots),
- )
- return False
-
- backup_chroot_name = "chroot-bak-%d" % random.randint(0, 1000)
- cmd = ["lvrename", chroot_vg, chroot_lv, backup_chroot_name]
- try:
- cros_build_lib.dbg_run(cmd, capture_output=True)
- except cros_build_lib.RunCommandError as e:
- cros_build_lib.Die("Restoring snapshot failed!\n%s", e)
-
- cmd = ["lvrename", chroot_vg, snapshot_name, chroot_lv]
- try:
- cros_build_lib.dbg_run(cmd, capture_output=True)
- except cros_build_lib.RunCommandError as e:
- cmd = ["lvrename", chroot_vg, backup_chroot_name, chroot_lv]
- try:
- cros_build_lib.dbg_run(cmd, capture_output=True)
- except cros_build_lib.RunCommandError as e:
- cros_build_lib.Die(
- "Failed to rename %s to chroot and failed to restore %s back "
- "to chroot!\n%s",
- snapshot_name,
- backup_chroot_name,
- e,
- )
- cros_build_lib.Die(
- "Failed to rename %s to chroot! Original chroot LV has "
- "been restored.\n%s",
- snapshot_name,
- e,
- )
-
- # Some versions of LVM set snapshots to be skipped at auto-activate time.
- # Other versions don't have this flag at all. We run lvchange to try
- # disabling auto-skip and activating the volume, but ignore errors. Versions
- # that don't have the flag should be auto-activated.
- chroot_lv_path = "%s/%s" % (chroot_vg, chroot_lv)
- cmd = ["lvchange", "-kn", chroot_lv_path]
- cros_build_lib.run(cmd, print_cmd=False, capture_output=True, check=False)
-
- # Activate the LV in case the lvchange above was needed. Activating an LV
- # that is already active shouldn't do anything, so this is safe to run even
- # if the -kn wasn't needed.
- cmd = ["lvchange", "-ay", chroot_lv_path]
- cros_build_lib.dbg_run(cmd, capture_output=True)
-
- cmd = ["lvremove", "-f", "%s/%s" % (chroot_vg, backup_chroot_name)]
- try:
- cros_build_lib.dbg_run(cmd, capture_output=True)
- except cros_build_lib.RunCommandError as e:
- cros_build_lib.Die(
- "Failed to remove backup LV %s/%s!\n%s",
- chroot_vg,
- backup_chroot_name,
- e,
- )
-
- return True
-
-
-def ListChrootSnapshots(chroot_vg, chroot_lv):
- """Return all snapshots in |chroot_vg| regardless of origin volume.
-
- Args:
- chroot_vg: The name of the VG containing the chroot.
- chroot_lv: The name of the chroot LV.
-
- Returns:
- A (possibly-empty) list of snapshot LVs found in |chroot_vg|.
-
- Raises:
- SystemExit: The lvs command failed.
- """
- if not chroot_vg or not chroot_lv:
- return []
-
- cmd = [
- "lvs",
- "-o",
- "lv_name,pool_lv,lv_attr",
- "-O",
- "lv_name",
- "--noheadings",
- "--separator",
- "\t",
- chroot_vg,
- ]
- try:
- result = cros_build_lib.run(
- cmd, print_cmd=False, stdout=True, encoding="utf-8"
- )
- except cros_build_lib.RunCommandError:
- raise SystemExit("Running %r failed!" % cmd)
-
- # Once the thin origin volume has been deleted, there's no way to tell a
- # snapshot apart from any other volume. Since this VG is created and managed
- # by cros_sdk, we'll assume that all volumes that share the same thin pool
- # are valid snapshots.
- snapshots = []
- snapshot_attrs = re.compile(r"^V.....t.{2,}") # Matches a thin volume.
- for line in result.stdout.splitlines():
- lv_name, pool_lv, lv_attr = line.lstrip().split("\t")
- if (
- lv_name == chroot_lv
- or lv_name == cros_sdk_lib.CHROOT_THINPOOL_NAME
- or pool_lv != cros_sdk_lib.CHROOT_THINPOOL_NAME
- or not snapshot_attrs.match(lv_attr)
- ):
- continue
- snapshots.append(lv_name)
- return snapshots
-
-
def _SudoCommand():
"""Get the 'sudo' command, along with all needed environment variables."""
@@ -726,16 +456,16 @@
dest="use_image",
action="store_false",
default=False,
- help="Do not mount the chroot on a loopback image; "
- "instead, create it directly in a directory.",
+ deprecated="--[no]use-image is no longer supported (b/266878468).",
+ help=argparse.SUPPRESS,
)
parser.add_argument(
"--use-image",
dest="use_image",
action="store_true",
default=False,
- help="Mount the chroot on a loopback image "
- "instead of creating it directly in a directory.",
+ deprecated="--[no]use-image is no longer supported (b/266878468).",
+ help=argparse.SUPPRESS,
)
parser.add_argument(
@@ -861,7 +591,7 @@
default=False,
help="Unmount and clean up devices associated with the "
"SDK chroot if it exists. This does not delete the "
- "backing image file, so the same chroot can be later "
+ "chroot contents, so the same chroot can be later "
"re-mounted for reuse. To fully delete the chroot, use "
"--delete. This is primarily useful for working on "
"cros_sdk or the chroot setup; you should not need it "
@@ -873,29 +603,6 @@
default=False,
help="Download the sdk.",
)
- group.add_argument(
- "--snapshot-create",
- metavar="SNAPSHOT_NAME",
- help="Create a snapshot of the chroot. Requires that the chroot was "
- "created without the --nouse-image option.",
- )
- group.add_argument(
- "--snapshot-restore",
- metavar="SNAPSHOT_NAME",
- help="Restore the chroot to a previously created snapshot.",
- )
- group.add_argument(
- "--snapshot-delete",
- metavar="SNAPSHOT_NAME",
- help="Delete a previously created snapshot. Deleting a snapshot that "
- "does not exist is not an error.",
- )
- group.add_argument(
- "--snapshot-list",
- action="store_true",
- default=False,
- help="List existing snapshots of the chroot and exit.",
- )
commands = group
# Namespace options.
@@ -996,7 +703,6 @@
_ReportMissing(osutils.FindMissingBinaries(NEEDED_TOOLS))
if options.proxy_sim:
_ReportMissing(osutils.FindMissingBinaries(PROXY_NEEDED_TOOLS))
- missing_image_tools = osutils.FindMissingBinaries(IMAGE_NEEDED_TOOLS)
if (
sdk_latest_version == "<unknown>"
@@ -1010,22 +716,6 @@
" https://dev.chromium.org/chromium-os/developer-guide"
)
- any_snapshot_operation = (
- options.snapshot_create
- or options.snapshot_restore
- or options.snapshot_delete
- or options.snapshot_list
- )
-
- if (
- options.snapshot_delete
- and options.snapshot_delete == options.snapshot_restore
- ):
- parser.error(
- "Cannot --snapshot_delete the same snapshot you are "
- "restoring with --snapshot_restore."
- )
-
_ReExecuteIfNeeded([sys.argv[0]] + argv, options)
lock_path = os.path.dirname(options.chroot)
@@ -1050,79 +740,18 @@
# pylint: enable=protected-access
options.enter |= bool(options.commands)
- if (
- options.delete
- and not options.create
- and (options.enter or any_snapshot_operation)
- ):
+ if options.delete and not options.create and options.enter:
parser.error(
- "Trying to enter or snapshot the chroot when --delete "
+ "Trying to enter the chroot when --delete "
"was specified makes no sense."
)
- if options.unmount and (
- options.create or options.enter or any_snapshot_operation
- ):
+ if options.unmount and (options.create or options.enter):
parser.error("--unmount cannot be specified with other chroot actions.")
- # If there is an existing chroot image and we're not removing it then force
- # use_image on. This ensures that people don't have to remember to pass
- # --use-image after a reboot to avoid losing access to their existing
- # chroot.
chroot_exists = cros_sdk_lib.IsChrootReady(options.chroot)
- img_path = _ImageFileForChroot(options.chroot)
- if (
- not options.use_image
- and not options.delete
- and not options.unmount
- and os.path.exists(img_path)
- ):
- if chroot_exists:
- # If the chroot is already populated, make sure it has something
- # mounted on it before we assume it came from an image.
- cmd = ["mountpoint", "-q", options.chroot]
- if cros_build_lib.dbg_run(cmd, check=False).returncode == 0:
- options.use_image = True
-
- else:
- logging.notice(
- "Existing chroot image %s found. Forcing --use-image on.",
- img_path,
- )
- options.use_image = True
-
- if any_snapshot_operation and not options.use_image:
- if os.path.exists(img_path):
- options.use_image = True
- else:
- cros_build_lib.Die(
- "Snapshot operations are not compatible with " "--nouse-image."
- )
- if options.use_image:
- logging.warning("--use-image is deprecated and will be removed soon.")
- logging.warning("Please migrate, or create a new one with --delete.")
- logging.warning("See http://b/266878468 for details.")
-
- # Discern if we need to create the chroot.
- if (
- options.use_image
- and not chroot_exists
- and not options.delete
- and not options.unmount
- and not missing_image_tools
- and os.path.exists(img_path)
- ):
- # Try to re-mount an existing image in case the user has rebooted.
- with locking.FileLock(lock_path, "chroot lock") as lock:
- logging.debug("Checking if existing chroot image can be mounted.")
- lock.write_lock()
- cros_sdk_lib.MountChroot(options.chroot, create=False)
- chroot_exists = cros_sdk_lib.IsChrootReady(options.chroot)
- if chroot_exists:
- logging.notice("Mounted existing image %s on chroot", img_path)
-
# Finally, flip create if necessary.
- if options.enter or options.snapshot_create:
+ if options.enter:
options.create |= not chroot_exists
# Make sure we will download if we plan to create.
@@ -1156,8 +785,6 @@
# Anything that needs to manipulate the main chroot mount or communicate
# with LVM needs to be done here before we enter the new namespaces.
- # If deleting, do it regardless of the use_image flag so that a
- # previously-created loopback chroot can also be cleaned up.
if options.delete:
# Set a timeout of 300 seconds when getting the lock.
with locking.FileLock(
@@ -1207,134 +834,10 @@
cros_sdk_lib.CleanupChrootMount(chroot.path, delete=False)
sys.exit(0)
- # Make sure the main chroot mount is visible. Contents will be filled in
- # below if needed.
- if options.create and options.use_image:
- if missing_image_tools:
- raise SystemExit(
- """The tool(s) %s were not found.
-Please make sure the lvm2 and thin-provisioning-tools packages
-are installed on your host.
-Example(ubuntu):
- sudo apt-get install lvm2 thin-provisioning-tools
-
-If you want to run without lvm2, pass --nouse-image (chroot
-snapshots will be unavailable)."""
- % ", ".join(missing_image_tools)
- )
-
- logging.debug("Making sure chroot image is mounted.")
- with locking.FileLock(lock_path, "chroot lock") as lock:
- lock.write_lock()
- if not cros_sdk_lib.MountChroot(chroot.path, create=True):
- cros_build_lib.Die(
- "Unable to mount %s on chroot",
- _ImageFileForChroot(chroot.path),
- )
- logging.notice(
- "Mounted %s on chroot", _ImageFileForChroot(chroot.path)
- )
-
- # Snapshot operations will always need the VG/LV, but other actions won't.
- if any_snapshot_operation:
- with locking.FileLock(lock_path, "chroot lock") as lock:
- chroot_vg, chroot_lv = cros_sdk_lib.FindChrootMountSource(
- chroot.path
- )
- if not chroot_vg or not chroot_lv:
- cros_build_lib.Die(
- "Unable to find VG/LV for chroot %s", chroot.path
- )
-
- # Delete snapshot before creating a new one. This allows the user to
- # throw out old state, create a new snapshot, and enter the chroot
- # in a single call to cros_sdk. Since restore involves deleting,
- # also do it before creating.
- if options.snapshot_restore:
- lock.write_lock()
- valid_snapshots = ListChrootSnapshots(chroot_vg, chroot_lv)
- if options.snapshot_restore not in valid_snapshots:
- cros_build_lib.Die(
- "%s is not a valid snapshot to restore to. "
- "Valid snapshots: %s",
- options.snapshot_restore,
- ", ".join(valid_snapshots),
- )
- osutils.UmountTree(chroot.path)
- if not RestoreChrootSnapshot(
- options.snapshot_restore, chroot_vg, chroot_lv
- ):
- cros_build_lib.Die("Unable to restore chroot to snapshot.")
- if not cros_sdk_lib.MountChroot(chroot.path, create=False):
- cros_build_lib.Die(
- "Unable to mount restored snapshot onto chroot."
- )
-
- # Use a read lock for snapshot delete and create even though they
- # modify the filesystem, because they don't modify the mounted
- # chroot itself. The underlying LVM commands take their own locks,
- # so conflicting concurrent operations here may crash cros_sdk, but
- # won't corrupt the chroot image. This tradeoff seems worth it to
- # allow snapshot operations on chroots that have a process inside.
- if options.snapshot_delete:
- lock.read_lock()
- DeleteChrootSnapshot(
- options.snapshot_delete, chroot_vg, chroot_lv
- )
-
- if options.snapshot_create:
- lock.read_lock()
- if not CreateChrootSnapshot(
- options.snapshot_create, chroot_vg, chroot_lv
- ):
- cros_build_lib.Die("Unable to create snapshot.")
-
- img_path = _ImageFileForChroot(chroot.path)
- if (
- options.use_image
- and os.path.exists(chroot.path)
- and os.path.exists(img_path)
- ):
- img_stat = os.stat(img_path)
- img_used_bytes = img_stat.st_blocks * 512
-
- mount_stat = os.statvfs(chroot.path)
- mount_used_bytes = mount_stat.f_frsize * (
- mount_stat.f_blocks - mount_stat.f_bfree
- )
-
- extra_gbs = (img_used_bytes - mount_used_bytes) // 2**30
- if extra_gbs > MAX_UNUSED_IMAGE_GBS:
- logging.notice(
- "%s is using %s GiB more than needed. Running "
- "fstrim in background.",
- img_path,
- extra_gbs,
- )
- pid = os.fork()
- if pid == 0:
- try:
- # Directly call Popen to run fstrim concurrently.
- cmd = ["fstrim", chroot.path]
- subprocess.Popen(cmd, close_fds=True, shell=False)
- except subprocess.SubprocessError as e:
- logging.warning(
- "Running fstrim failed. Consider running fstrim on "
- "your chroot manually.\n%s",
- e,
- )
- os._exit(0) # pylint: disable=protected-access
- os.waitpid(pid, 0)
-
# Enter a new set of namespaces. Everything after here cannot directly
# affect the hosts's mounts or alter LVM volumes.
namespaces.SimpleUnshare(net=options.ns_net, pid=options.ns_pid)
- if options.snapshot_list:
- for snap in ListChrootSnapshots(chroot_vg, chroot_lv):
- print(snap)
- sys.exit(0)
-
if not options.sdk_version:
sdk_version = (
bootstrap_latest_version