Reland "cbuildbot_launch: Clean distfiles cache when too old"
This is a reland of d2cb40af59d51077daef3adca2692d3c00cdef45
diff: Changed to using unix time instead of stip time because
cbuildbot_launch runs in an environment that does not have pytz.
Original change's description:
> cbuildbot_launch: Clean distfiles cache when too old
>
> - Use the cbuildbot_launch to persist time since last cleanup of
> distfiles.
> - Do this in a way that old format state without the timestamp is
> simply updated with a timestamp.
> - Clean distfiles cache when it is more than 8 days old.
>
> + make BuildrootCleanup tests more behavioural.
>
> BUG=chromium:814989
> TEST=unittests.
>
> Change-Id: I0f1c74993f3dd59a16da333fb1ff49056c63086b
> Reviewed-on: https://chromium-review.googlesource.com/993759
> Commit-Ready: Prathmesh Prabhu <pprabhu@chromium.org>
> Tested-by: Prathmesh Prabhu <pprabhu@chromium.org>
> Reviewed-by: Don Garrett <dgarrett@chromium.org>
BUG=chromium:814989
TEST=unittests.
Change-Id: I9a784091684ea10e3316e543a5cb0121bc331490
Reviewed-on: https://chromium-review.googlesource.com/998392
Reviewed-by: Don Garrett <dgarrett@chromium.org>
Tested-by: Prathmesh Prabhu <pprabhu@chromium.org>
diff --git a/scripts/cbuildbot_launch.py b/scripts/cbuildbot_launch.py
index ff57c27..4393f9b 100644
--- a/scripts/cbuildbot_launch.py
+++ b/scripts/cbuildbot_launch.py
@@ -18,6 +18,7 @@
import functools
import os
+import time
from chromite.cbuildbot import repository
from chromite.cbuildbot.stages import sync_stages
@@ -30,10 +31,10 @@
from chromite.lib import ts_mon_config
from chromite.scripts import cbuildbot
-
# This number should be incremented when we change the layout of the buildroot
# in a non-backwards compatible way. This wipes all buildroots.
BUILDROOT_BUILDROOT_LAYOUT = 2
+_DISTFILES_CACHE_EXPIRY_HOURS = 8 * 24
# Metrics reported to Monarch.
METRIC_ACTIVE = 'chromeos/chromite/cbuildbot_launch/active'
@@ -45,6 +46,8 @@
METRIC_CBUILDBOT = 'chromeos/chromite/cbuildbot_launch/cbuildbot_durations'
METRIC_CLOBBER = 'chromeos/chromite/cbuildbot_launch/clobber'
METRIC_BRANCH_CLEANUP = 'chromeos/chromite/cbuildbot_launch/branch_cleanup'
+METRIC_DISTFILES_CLEANUP = (
+ 'chromeos/chromite/cbuildbot_launch/distfiles_cleanup')
METRIC_DEPOT_TOOLS = 'chromeos/chromite/cbuildbot_launch/depot_tools_prep'
@@ -128,32 +131,72 @@
Returns:
Layout version as an integer (0 for unknown).
Previous branch as a string ('' for unknown).
+ Last distfiles clearance time in POSIX time (None for unknown).
"""
state_file = os.path.join(root, '.cbuildbot_launch_state')
try:
state = osutils.ReadFile(state_file)
- buildroot_layout, branchname = state.split()
- buildroot_layout = int(buildroot_layout)
- return buildroot_layout, branchname
- except (IOError, ValueError):
+ parts = state.split()
+ if len(parts) >= 3:
+ return int(parts[0]), parts[1], float(parts[2])
+ else:
+ # TODO(pprabhu) delete this branch once most buildslaves have migrated to
+ # newer state with three parts.
+ return int(parts[0]), parts[1], None
+ except (IOError, ValueError, IndexError):
# If we are unable to either read or parse the state file, we get here.
- return 0, ''
+ return 0, '', None
-def SetState(branchname, root):
+def SetState(branchname, root, distfiles_ts=None):
"""Save the current state of our working directory.
Args:
branchname: Name of branch we prepped for as a string.
root: Root of the working directory tree as a string.
+ distfiles_ts: float unix timestamp to include as the distfiles timestamp. If
+ None, current time will be used.
"""
assert branchname
state_file = os.path.join(root, '.cbuildbot_launch_state')
- new_state = '%d %s' % (BUILDROOT_BUILDROOT_LAYOUT, branchname)
+ if distfiles_ts is None:
+ distfiles_ts = time.time()
+ new_state = '%d %s %f' % (
+ BUILDROOT_BUILDROOT_LAYOUT, branchname, distfiles_ts)
osutils.WriteFile(state_file, new_state)
+def _MaybeCleanDistfiles(repo, distfiles_ts, metrics_fields):
+ """Cleans the distfiles directory if too old.
+
+ Args:
+ repo: repository.RepoRepository instance.
+ distfiles_ts: A timestamp str for the last time distfiles was cleaned. May
+ be None.
+ metrics_fields: Dictionary of fields to include in metrics.
+
+ Returns:
+ The new distfiles_ts to persist in state.
+ """
+
+ if distfiles_ts is None:
+ return None
+
+ distfiles_age = (time.time() - distfiles_ts) / 3600.0
+ if distfiles_age < _DISTFILES_CACHE_EXPIRY_HOURS:
+ return distfiles_ts
+
+ logging.info('Remove old distfiles cache (cache expiry %d hours)',
+ _DISTFILES_CACHE_EXPIRY_HOURS)
+ osutils.RmDir(os.path.join(repo.directory, '.cache', 'distfiles'),
+ ignore_missing=True, sudo=True)
+ metrics.Counter(METRIC_DISTFILES_CLEANUP).increment(
+ field(metrics_fields, reason='cache_expired'))
+ # Cleaned cache, so reset distfiles_ts
+ return None
+
+
@StageDecorator
def CleanBuildRoot(root, repo, metrics_fields):
"""Some kinds of branch transitions break builds.
@@ -167,7 +210,8 @@
repo: repository.RepoRepository instance.
metrics_fields: Dictionary of fields to include in metrics.
"""
- old_buildroot_layout, old_branch = GetState(root)
+ old_buildroot_layout, old_branch, distfiles_ts = GetState(root)
+ distfiles_ts = _MaybeCleanDistfiles(repo, distfiles_ts, metrics_fields)
if old_buildroot_layout != BUILDROOT_BUILDROOT_LAYOUT:
logging.PrintBuildbotStepText('Unknown layout: Wiping buildroot.')
@@ -205,7 +249,7 @@
# Ensure buildroot exists. Save the state we are prepped for.
osutils.SafeMakedirs(repo.directory)
- SetState(repo.branch, root)
+ SetState(repo.branch, root, distfiles_ts)
@StageDecorator