Revert "cbuildbot_launch: Clean distfiles cache when too old"
This reverts commit d2cb40af59d51077daef3adca2692d3c00cdef45.
Reason for revert: pre-cq and other swarming bots are failing
Original change's description:
> cbuildbot_launch: Clean distfiles cache when too old
>
> - Use the cbuildbot_launch to persist time since last cleanup of
> distfiles.
> - Do this in a way that old format state without the timestamp is
> simply updated with a timestamp.
> - Clean distfiles cache when it is more than 8 days old.
>
> + make BuildrootCleanup tests more behavioural.
>
> BUG=chromium:814989
> TEST=unittests.
>
> Change-Id: I0f1c74993f3dd59a16da333fb1ff49056c63086b
> Reviewed-on: https://chromium-review.googlesource.com/993759
> Commit-Ready: Prathmesh Prabhu <pprabhu@chromium.org>
> Tested-by: Prathmesh Prabhu <pprabhu@chromium.org>
> Reviewed-by: Don Garrett <dgarrett@chromium.org>
Change-Id: I3bc3af85cfd058bc27d2a8417833693c184f2d8b
BUG:chromium:829230
Reviewed-on: https://chromium-review.googlesource.com/997273
Reviewed-by: Stephen Barber <smbarber@chromium.org>
Commit-Queue: Stephen Barber <smbarber@chromium.org>
Tested-by: Stephen Barber <smbarber@chromium.org>
diff --git a/scripts/cbuildbot_launch.py b/scripts/cbuildbot_launch.py
index 175335d..ff57c27 100644
--- a/scripts/cbuildbot_launch.py
+++ b/scripts/cbuildbot_launch.py
@@ -16,7 +16,6 @@
from __future__ import print_function
-import datetime
import functools
import os
@@ -31,14 +30,10 @@
from chromite.lib import ts_mon_config
from chromite.scripts import cbuildbot
-# Import setup by chromite import
-from infra_libs.time_functions import zulu
-
# This number should be incremented when we change the layout of the buildroot
# in a non-backwards compatible way. This wipes all buildroots.
BUILDROOT_BUILDROOT_LAYOUT = 2
-_DISTFILES_CACHE_EXPIRY_DAYS = 8
# Metrics reported to Monarch.
METRIC_ACTIVE = 'chromeos/chromite/cbuildbot_launch/active'
@@ -50,8 +45,6 @@
METRIC_CBUILDBOT = 'chromeos/chromite/cbuildbot_launch/cbuildbot_durations'
METRIC_CLOBBER = 'chromeos/chromite/cbuildbot_launch/clobber'
METRIC_BRANCH_CLEANUP = 'chromeos/chromite/cbuildbot_launch/branch_cleanup'
-METRIC_DISTFILES_CLEANUP = (
- 'chromeos/chromite/cbuildbot_launch/distfiles_cleanup')
METRIC_DEPOT_TOOLS = 'chromeos/chromite/cbuildbot_launch/depot_tools_prep'
@@ -135,93 +128,32 @@
Returns:
Layout version as an integer (0 for unknown).
Previous branch as a string ('' for unknown).
- Last distfiles clearance time as datetime.datetime (None for unknown).
"""
state_file = os.path.join(root, '.cbuildbot_launch_state')
try:
state = osutils.ReadFile(state_file)
- parts = state.split()
- if len(parts) >= 3:
- return int(parts[0]), parts[1], parts[2]
- else:
- # TODO(pprabhu) delete this branch once most buildslaves have migrated to
- # newer state with three parts.
- return int(parts[0]), parts[1], None
- except (IOError, ValueError, IndexError):
+ buildroot_layout, branchname = state.split()
+ buildroot_layout = int(buildroot_layout)
+ return buildroot_layout, branchname
+ except (IOError, ValueError):
# If we are unable to either read or parse the state file, we get here.
- return 0, '', None
+ return 0, ''
-def SetState(branchname, root, distfiles_ts=None):
+def SetState(branchname, root):
"""Save the current state of our working directory.
Args:
branchname: Name of branch we prepped for as a string.
root: Root of the working directory tree as a string.
- distfiles_ts: A timestamp str to include as the distfiles timestamp. If
- None, current time will be used.
"""
assert branchname
state_file = os.path.join(root, '.cbuildbot_launch_state')
- if distfiles_ts is None:
- now = datetime.datetime.utcnow()
- distfiles_ts = zulu.to_zulu_string(now)
- new_state = '%d %s %s' % (
- BUILDROOT_BUILDROOT_LAYOUT, branchname, distfiles_ts)
+ new_state = '%d %s' % (BUILDROOT_BUILDROOT_LAYOUT, branchname)
osutils.WriteFile(state_file, new_state)
-def _ComputeAge(timestamp_utc):
- """Compute timedelta since a given timestamp.
-
- Args:
- timestamp_utc: A string representation of a timestamp in utc.
-
- Returns:
- a datetime.timedelta object, None on errors.
- """
- try:
- last = zulu.parse_zulu_time(timestamp_utc)
- except (OverflowError, ValueError, OSError):
- return None
- now = datetime.datetime.utcnow()
- return now - last
-
-
-def _MaybeCleanDistfiles(repo, distfiles_ts, metrics_fields):
- """Cleans the distfiles directory if too old.
-
- Args:
- repo: repository.RepoRepository instance.
- distfiles_ts: A timestamp str for the last time distfiles was cleaned. May
- be None.
- metrics_fields: Dictionary of fields to include in metrics.
-
- Returns:
- The new distfiles_ts to persist in state.
- """
-
- if distfiles_ts is None:
- return None
-
- distfiles_age = _ComputeAge(distfiles_ts)
- if distfiles_age is None:
- # Corrupted distfiles_ts. Reset it
- return None
- if distfiles_age < datetime.timedelta(days=_DISTFILES_CACHE_EXPIRY_DAYS):
- return distfiles_age
-
- logging.info('Remove old distfiles cache (cache expiry %d days)',
- _DISTFILES_CACHE_EXPIRY_DAYS)
- osutils.RmDir(os.path.join(repo.directory, '.cache', 'distfiles'),
- ignore_missing=True, sudo=True)
- metrics.Counter(METRIC_DISTFILES_CLEANUP).increment(
- field(metrics_fields, reason='cache_expired'))
- # Cleaned cache, so reset distfiles_ts
- return None
-
-
@StageDecorator
def CleanBuildRoot(root, repo, metrics_fields):
"""Some kinds of branch transitions break builds.
@@ -235,8 +167,7 @@
repo: repository.RepoRepository instance.
metrics_fields: Dictionary of fields to include in metrics.
"""
- old_buildroot_layout, old_branch, distfiles_ts = GetState(root)
- distfiles_ts = _MaybeCleanDistfiles(repo, distfiles_ts, metrics_fields)
+ old_buildroot_layout, old_branch = GetState(root)
if old_buildroot_layout != BUILDROOT_BUILDROOT_LAYOUT:
logging.PrintBuildbotStepText('Unknown layout: Wiping buildroot.')
@@ -274,7 +205,7 @@
# Ensure buildroot exists. Save the state we are prepped for.
osutils.SafeMakedirs(repo.directory)
- SetState(repo.branch, root, distfiles_ts)
+ SetState(repo.branch, root)
@StageDecorator