bin: lib: script: Expose cros_generate_dlc_artifacts
.. to allow for CrOS DLC developers/clients to start invoking and
generating DLC artifacts.
Produced artifacts:
```
[4.0K] dlcart
├── [4.0M] dlc.img
└── [ 659] meta.tar.zst
[4.0K] dlcartmeta
└── [ 281] metadata
```
Inside basic mount:
```
[ 42] mnt/
├── [ 1472] LICENSE
└── [ 27] root
└── [ 4194304] file
```
Replace below <..> variables with specific DLC values.
[..] are optional.
Tests:
```
Official upload using --upload:
$> cros_sdk -- \
cros_generate_dlc_artifacts \
--src-dir <SOURCE_DIR> \
--license ../third_party/chromiumos-overlay/licenses/<LICENSE> \
--id <DLC_ID> \
--preallocated-blocks <PREALLOCATED_BLOCKS>\
--version <VERSION> \
[--output-dir <DLC_OUTPUT_DIR>/] \
[--output-metadata_dir <DLC_OUTPUT_METADATA_DIR>/] \
[--debug] \
[--upload] \
[--upload-dry-run] \
[--uri-path <URI_DIR>] \
[--disable-randomness]
* Dry run upload using --upload-dry-run
* Enable randomness by not using --disable-randomness
* Output artifacts and/or metadata using --output-* options
* Override URI output using --uri-path
```
BUG=b:286327155
TEST=comment above
TEST=./run_tests
Change-Id: I703d8988983b84202f352ce0367115f9ffe1cbf1
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/4707265
Commit-Queue: Jae Hoon Kim <kimjae@chromium.org>
Reviewed-by: Yuanpeng Ni <yuanpengni@chromium.org>
Reviewed-by: Mike Frysinger <vapier@chromium.org>
Auto-Submit: Jae Hoon Kim <kimjae@chromium.org>
Tested-by: Jae Hoon Kim <kimjae@chromium.org>
diff --git a/scripts/cros_generate_dlc_artifacts.py b/scripts/cros_generate_dlc_artifacts.py
new file mode 100644
index 0000000..b52d69d
--- /dev/null
+++ b/scripts/cros_generate_dlc_artifacts.py
@@ -0,0 +1,241 @@
+# Copyright 2023 The ChromiumOS Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Script to generate (+ upload) DLC artifacts."""
+
+import logging
+import os
+import shutil
+from typing import List
+
+from chromite.lib import commandline
+from chromite.lib import cros_build_lib
+from chromite.lib import dlc_lib
+from chromite.lib import osutils
+
+
+# Predefined salts.
+_SHORT_SALT = "1337D00D"
+
+# Tarball extension with correct compression.
+_TAR_COMP_EXT = ".tar.zst"
+_META_OUT_FILE = dlc_lib.DLC_TMP_META_DIR + _TAR_COMP_EXT
+
+# Filenames.
+_METADATA_FILE = "metadata"
+
+
+def ParseArguments(argv: List[str]) -> commandline.ArgumentNamespace:
+ """Returns a namespace for the CLI arguments."""
+ parser = commandline.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "--src-dir",
+ type="dir_exists",
+ required=True,
+ help="The directory to package as a DLC",
+ )
+ # Support license addition here. For now, have users explicitly pass in a
+ # stub license path.
+ parser.add_argument(
+ "--license",
+ type="file_exists",
+ required=True,
+ help="The path to license, this should be the same license as the one"
+ " used within the package",
+ )
+ parser.add_argument(
+ "--output-dir",
+ type="path",
+ help="The optional output directory to put artifacts into",
+ )
+ parser.add_argument(
+ "--output-metadata-dir",
+ type="path",
+ help="The optional output directory to put metadata into",
+ )
+
+ parser.add_argument(
+ "--upload",
+ action="store_true",
+ help="Upload the DLC artifacts to google buckets",
+ )
+ parser.add_argument(
+ "--uri-path",
+ type="gs_path",
+ help="The override for DLC image URI, check dlc_lib for default",
+ )
+ parser.add_argument(
+ "--upload-dry-run",
+ action="store_true",
+ help="Dry run without actual upload",
+ )
+
+ # Salt for randomness DLC image randomness.
+ parser.add_argument(
+ "--disable-randomness",
+ action="store_true",
+ help="To disable randomizing hashes when generating DLC images",
+ )
+
+ # DLC required fields.
+ parser.add_argument(
+ "--id",
+ type=str,
+ required=True,
+ help="The DLC ID",
+ )
+ parser.add_argument(
+ "--preallocated-blocks",
+ type=int,
+ required=True,
+ help="The preallocated number of blocks in 4KiB chunks",
+ )
+
+ # DLC optional fields.
+ parser.add_argument(
+ "--name",
+ type=str,
+ default="",
+ help="The name of the DLC in human friendly format",
+ )
+ parser.add_argument(
+ "--description",
+ type=str,
+ default="",
+ help="The description of the DLC in human friendly format",
+ )
+ parser.add_argument(
+ "--version",
+ type=str,
+ required=True,
+ help="The version of this DLC build",
+ )
+
+ opts = parser.parse_args(argv)
+
+ dlc_lib.ValidateDlcIdentifier(opts.id)
+
+ opts.Freeze()
+
+ return opts
+
+
+def GenerateDlcParams(
+ opts: commandline.ArgumentNamespace,
+) -> dlc_lib.EbuildParams:
+ """Generates and verifies DLC parameters based on options
+
+ Args:
+ opts: The command line arguments.
+
+ Returns:
+ The DLC ebuild parameters.
+ """
+ params = dlc_lib.EbuildParams(
+ dlc_id=opts.id,
+ dlc_package="package",
+ fs_type=dlc_lib.SQUASHFS_TYPE,
+ pre_allocated_blocks=opts.preallocated_blocks,
+ version=opts.version,
+ name=opts.name,
+ description=opts.description,
+ # Add preloading support.
+ preload=False,
+ used_by="",
+ mount_file_required=False,
+ fullnamerev="",
+ scaled=True,
+ loadpin_verity_digest=False,
+ )
+ params.VerifyDlcParameters()
+ return params
+
+
+def UploadDlcArtifacts(
+ dlcartifacts: dlc_lib.DlcArtifacts, dry_run: bool
+) -> None:
+ """Uploads the DLC artifacts based on `DlcArtifacts`
+
+ Args:
+ dlcartifacts: The DLC artifacts to upload.
+ dry_run: Dry run without actually uploading if true.
+ """
+ logging.info("Uploading DLC artifacts")
+ logging.debug(
+ "Uploading DLC image %s to %s",
+ dlcartifacts.image,
+ dlcartifacts.uri_path,
+ )
+ logging.debug(
+ "Uploading DLC meta %s to %s", dlcartifacts.meta, dlcartifacts.uri_path
+ )
+ dlcartifacts.Upload(dry_run=dry_run)
+
+
+def GenerateDlcArtifacts(opts: commandline.ArgumentNamespace) -> None:
+ """Generates the DLC artifacts
+
+ Args:
+ opts: The command line arguments.
+ """
+ params = GenerateDlcParams(opts)
+ uri_path = opts.uri_path or params.GetURIDir()
+
+ with osutils.TempDir(prefix="dlcartifacts", sudo_rm=True) as tmpdir:
+ output_dir = opts.output_dir or tmpdir
+ os.makedirs(output_dir, exist_ok=True)
+
+ logging.info("Generating DLC artifacts")
+ artifacts = dlc_lib.DlcGenerator(
+ src_dir=opts.src_dir,
+ sysroot="",
+ board=dlc_lib.MAGIC_BOARD,
+ ebuild_params=params,
+ reproducible=opts.disable_randomness,
+ license_file=opts.license,
+ ).ExternalGenerateDLC(
+ tmpdir, _SHORT_SALT if opts.disable_randomness else None
+ )
+ logging.debug("Generated DLC artifacts: %s", artifacts.StringJSON())
+
+ # Handle the meta.
+ meta_out = os.path.join(output_dir, _META_OUT_FILE)
+
+ logging.info("Emitting the metadata into %s", meta_out)
+ cros_build_lib.CreateTarball(
+ tarball_path=meta_out,
+ cwd=artifacts.meta,
+ compression=cros_build_lib.CompressionType.ZSTD,
+ extra_env={"ZSTD_CLEVEL": "9"},
+ )
+
+ # Handle the image.
+ image_out = os.path.join(output_dir, dlc_lib.DLC_IMAGE)
+ logging.info("Emitting the DLC image into %s", image_out)
+ shutil.move(artifacts.image, image_out)
+
+ # Handle the upload.
+ ret_artifacts = dlc_lib.DlcArtifacts(
+ uri_path=uri_path,
+ image=image_out,
+ meta=meta_out,
+ )
+ ret_artifacts_json = ret_artifacts.StringJSON()
+ logging.debug("The final DLC artifacts: %s", ret_artifacts_json)
+
+ if opts.output_metadata_dir:
+ osutils.WriteFile(
+ os.path.join(opts.output_metadata_dir, _METADATA_FILE),
+ ret_artifacts_json,
+ makedirs=True,
+ )
+
+ if opts.upload or opts.upload_dry_run:
+ UploadDlcArtifacts(ret_artifacts, opts.upload_dry_run)
+ else:
+ logging.debug("Skipping DLC artifacts upload")
+
+
+def main(argv):
+ GenerateDlcArtifacts(ParseArguments(argv))