blob: a641b6908d30b19d024bc03c6a12f5232b46809a [file] [log] [blame]
Derek Beckettf73baca2020-08-19 15:08:47 -07001# Lint as: python2, python3
Oleg Loskutoff75cc9b22019-10-16 17:28:12 -07002# Copyright (c) 2008 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Derek Beckettf73baca2020-08-19 15:08:47 -07006from __future__ import absolute_import
7from __future__ import division
8from __future__ import print_function
9
Dan Shid9be07a2017-08-18 09:51:45 -070010import os, time, socket, shutil, glob, logging, tempfile, re
xixuan6cf6d2f2016-01-29 15:29:00 -080011import shlex
Cheng-Yi Chianga155e7e2015-08-20 20:42:04 +080012import subprocess
13
Dan Shi4f8c0242017-07-07 15:34:49 -070014from autotest_lib.client.bin.result_tools import runner as result_tools_runner
Hidehiko Abe28422ed2017-06-21 10:50:44 +090015from autotest_lib.client.common_lib import error
Otabek Kasimovf5e0f102020-06-30 19:41:02 -070016from autotest_lib.client.common_lib import utils
Dan Shi9f92aa62017-07-27 17:07:05 -070017from autotest_lib.client.common_lib.cros.network import ping_runner
Hidehiko Abe28422ed2017-06-21 10:50:44 +090018from autotest_lib.client.common_lib.global_config import global_config
jadmanski31c49b72008-10-27 20:44:48 +000019from autotest_lib.server import utils, autotest
Prathmesh Prabhu8b5065d2017-01-10 17:13:01 -080020from autotest_lib.server.hosts import host_info
mblighe8b93af2009-01-30 00:45:53 +000021from autotest_lib.server.hosts import remote
Roshan Pius58e5dd32015-10-16 15:16:42 -070022from autotest_lib.server.hosts import rpc_server_tracker
Hidehiko Abe28422ed2017-06-21 10:50:44 +090023from autotest_lib.server.hosts import ssh_multiplex
Derek Beckettf73baca2020-08-19 15:08:47 -070024import six
25from six.moves import filter
jadmanskica7da372008-10-21 16:26:52 +000026
Otabek Kasimovf5e0f102020-06-30 19:41:02 -070027try:
28 from chromite.lib import metrics
29except ImportError:
30 metrics = utils.metrics_mock
31
Gwendal Grignou36b61702016-02-10 11:57:53 -080032# pylint: disable=C0111
jadmanskica7da372008-10-21 16:26:52 +000033
mblighb86bfa12010-02-12 20:22:21 +000034get_value = global_config.get_config_value
Derek Beckett189dfa42021-01-05 12:47:14 -080035enable_main_ssh = get_value('AUTOSERV',
36 'enable_main_ssh',
37 type=bool,
Derek Beckett4d102242020-12-01 14:24:37 -080038 default=False)
mblighefccc1b2010-01-11 19:08:42 +000039
Dan Shi9f92aa62017-07-27 17:07:05 -070040# Number of seconds to use the cached up status.
41_DEFAULT_UP_STATUS_EXPIRATION_SECONDS = 300
Dean Liaoe3e75f62017-11-14 10:36:43 +080042_DEFAULT_SSH_PORT = 22
mblighefccc1b2010-01-11 19:08:42 +000043
Lutz Justen043e9c12017-10-27 12:40:47 +020044# Number of seconds to wait for the host to shut down in wait_down().
45_DEFAULT_WAIT_DOWN_TIME_SECONDS = 120
46
Philip Chen7ce1e392018-12-09 23:53:32 -080047# Number of seconds to wait for the host to boot up in wait_up().
48_DEFAULT_WAIT_UP_TIME_SECONDS = 120
49
50# Timeout in seconds for a single call of get_boot_id() in wait_down()
51# and a single ssh ping in wait_up().
Lutz Justen043e9c12017-10-27 12:40:47 +020052_DEFAULT_MAX_PING_TIMEOUT = 10
53
Derek Beckettfc04fbc2020-12-14 16:37:31 -080054# The client symlink directory.
55AUTOTEST_CLIENT_SYMLINK_END = 'client/autotest_lib'
56
57
Fang Deng96667ca2013-08-01 17:46:18 -070058class AbstractSSHHost(remote.RemoteHost):
mblighbc9402b2009-12-29 01:15:34 +000059 """
60 This class represents a generic implementation of most of the
jadmanskica7da372008-10-21 16:26:52 +000061 framework necessary for controlling a host via ssh. It implements
62 almost all of the abstract Host methods, except for the core
mblighbc9402b2009-12-29 01:15:34 +000063 Host.run method.
64 """
Simran Basi5ace6f22016-01-06 17:30:44 -080065 VERSION_PREFIX = ''
Derek Beckett4d102242020-12-01 14:24:37 -080066 # Timeout for main ssh connection setup, in seconds.
67 DEFAULT_START_MAIN_SSH_TIMEOUT_S = 5
jadmanskica7da372008-10-21 16:26:52 +000068
Dean Liaoe3e75f62017-11-14 10:36:43 +080069 def _initialize(self, hostname, user="root", port=_DEFAULT_SSH_PORT,
70 password="", is_client_install_supported=True,
71 afe_host=None, host_info_store=None, connection_pool=None,
Hidehiko Abe06893302017-06-24 07:32:38 +090072 *args, **dargs):
jadmanskif6562912008-10-21 17:59:01 +000073 super(AbstractSSHHost, self)._initialize(hostname=hostname,
74 *args, **dargs)
Kevin Cheng05ae2a42016-06-06 10:12:48 -070075 """
76 @param hostname: The hostname of the host.
77 @param user: The username to use when ssh'ing into the host.
78 @param password: The password to use when ssh'ing into the host.
79 @param port: The port to use for ssh.
80 @param is_client_install_supported: Boolean to indicate if we can
81 install autotest on the host.
82 @param afe_host: The host object attained from the AFE (get_hosts).
Prathmesh Prabhu8b5065d2017-01-10 17:13:01 -080083 @param host_info_store: Optional host_info.CachingHostInfoStore object
84 to obtain / update host information.
Hidehiko Abe06893302017-06-24 07:32:38 +090085 @param connection_pool: ssh_multiplex.ConnectionPool instance to share
Derek Beckett4d102242020-12-01 14:24:37 -080086 the main ssh connection across control scripts.
Kevin Cheng05ae2a42016-06-06 10:12:48 -070087 """
Otabek Kasimovf5e0f102020-06-30 19:41:02 -070088 self._track_class_usage()
Dan Shic07b8932014-12-11 15:22:30 -080089 # IP address is retrieved only on demand. Otherwise the host
90 # initialization will fail for host is not online.
91 self._ip = None
jadmanskica7da372008-10-21 16:26:52 +000092 self.user = user
93 self.port = port
94 self.password = password
Roshan Piusa58163a2015-10-14 13:36:29 -070095 self._is_client_install_supported = is_client_install_supported
showard6eafb492010-01-15 20:29:06 +000096 self._use_rsync = None
Fang Deng3af66202013-08-16 15:19:25 -070097 self.known_hosts_file = tempfile.mkstemp()[1]
Roshan Pius58e5dd32015-10-16 15:16:42 -070098 self._rpc_server_tracker = rpc_server_tracker.RpcServerTracker(self);
jadmanskica7da372008-10-21 16:26:52 +000099
mblighefccc1b2010-01-11 19:08:42 +0000100 """
Derek Beckett4d102242020-12-01 14:24:37 -0800101 Main SSH connection background job, socket temp directory and socket
102 control path option. If main-SSH is enabled, these fields will be
103 initialized by start_main_ssh when a new SSH connection is initiated.
mblighefccc1b2010-01-11 19:08:42 +0000104 """
Hidehiko Abe06893302017-06-24 07:32:38 +0900105 self._connection_pool = connection_pool
106 if connection_pool:
Derek Beckett4d102242020-12-01 14:24:37 -0800107 self._main_ssh = connection_pool.get(hostname, user, port)
Hidehiko Abe06893302017-06-24 07:32:38 +0900108 else:
Derek Beckett4d102242020-12-01 14:24:37 -0800109 self._main_ssh = ssh_multiplex.MainSsh(hostname, user, port)
Simran Basi3b858a22015-03-17 16:23:24 -0700110
Kevin Cheng05ae2a42016-06-06 10:12:48 -0700111 self._afe_host = afe_host or utils.EmptyAFEHost()
Prathmesh Prabhu8b5065d2017-01-10 17:13:01 -0800112 self.host_info_store = (host_info_store or
113 host_info.InMemoryHostInfoStore())
showard6eafb492010-01-15 20:29:06 +0000114
Dan Shi9f92aa62017-07-27 17:07:05 -0700115 # The cached status of whether the DUT responded to ping.
116 self._cached_up_status = None
117 # The timestamp when the value of _cached_up_status is set.
118 self._cached_up_status_updated = None
119
120
Dan Shic07b8932014-12-11 15:22:30 -0800121 @property
122 def ip(self):
123 """@return IP address of the host.
124 """
125 if not self._ip:
126 self._ip = socket.getaddrinfo(self.hostname, None)[0][4][0]
127 return self._ip
128
129
Roshan Piusa58163a2015-10-14 13:36:29 -0700130 @property
131 def is_client_install_supported(self):
132 """"
133 Returns True if the host supports autotest client installs, False
134 otherwise.
135 """
136 return self._is_client_install_supported
137
138
Roshan Pius58e5dd32015-10-16 15:16:42 -0700139 @property
140 def rpc_server_tracker(self):
141 """"
142 @return The RPC server tracker associated with this host.
143 """
144 return self._rpc_server_tracker
145
146
Dean Liaoe3e75f62017-11-14 10:36:43 +0800147 @property
148 def is_default_port(self):
Garry Wang81bea592020-08-27 17:25:25 -0700149 """Returns True if its port is default SSH port."""
150 return self.port == _DEFAULT_SSH_PORT
Dean Liaoe3e75f62017-11-14 10:36:43 +0800151
152 @property
153 def host_port(self):
154 """Returns hostname if port is default. Otherwise, hostname:port.
155 """
156 if self.is_default_port:
157 return self.hostname
158 else:
159 return '%s:%d' % (self.hostname, self.port)
160
161
162 # Though it doesn't use self here, it is not declared as staticmethod
163 # because its subclass may use self to access member variables.
164 def make_ssh_command(self, user="root", port=_DEFAULT_SSH_PORT, opts='',
165 hosts_file='/dev/null', connect_timeout=30,
166 alive_interval=300, alive_count_max=3,
167 connection_attempts=1):
168 ssh_options = " ".join([
169 opts,
170 self.make_ssh_options(
171 hosts_file=hosts_file, connect_timeout=connect_timeout,
172 alive_interval=alive_interval, alive_count_max=alive_count_max,
173 connection_attempts=connection_attempts)])
174 return "/usr/bin/ssh -a -x %s -l %s -p %d" % (ssh_options, user, port)
175
176
177 @staticmethod
178 def make_ssh_options(hosts_file='/dev/null', connect_timeout=30,
179 alive_interval=300, alive_count_max=3,
180 connection_attempts=1):
181 """Composes SSH -o options."""
Derek Beckettf73baca2020-08-19 15:08:47 -0700182 assert isinstance(connect_timeout, six.integer_types)
Fang Deng96667ca2013-08-01 17:46:18 -0700183 assert connect_timeout > 0 # can't disable the timeout
Dean Liaoe3e75f62017-11-14 10:36:43 +0800184
185 options = [("StrictHostKeyChecking", "no"),
186 ("UserKnownHostsFile", hosts_file),
187 ("BatchMode", "yes"),
188 ("ConnectTimeout", str(connect_timeout)),
189 ("ServerAliveInterval", str(alive_interval)),
190 ("ServerAliveCountMax", str(alive_count_max)),
191 ("ConnectionAttempts", str(connection_attempts))]
192 return " ".join("-o %s=%s" % kv for kv in options)
Fang Deng96667ca2013-08-01 17:46:18 -0700193
194
showard6eafb492010-01-15 20:29:06 +0000195 def use_rsync(self):
196 if self._use_rsync is not None:
197 return self._use_rsync
198
mblighc9892c02010-01-06 19:02:16 +0000199 # Check if rsync is available on the remote host. If it's not,
200 # don't try to use it for any future file transfers.
Gwendal Grignou03286f02017-03-24 10:50:59 -0700201 self._use_rsync = self.check_rsync()
showard6eafb492010-01-15 20:29:06 +0000202 if not self._use_rsync:
Ilja H. Friedel04be2bd2014-05-07 21:29:59 -0700203 logging.warning("rsync not available on remote host %s -- disabled",
Dean Liaoe3e75f62017-11-14 10:36:43 +0800204 self.host_port)
Eric Lie0493a42010-11-15 13:05:43 -0800205 return self._use_rsync
mblighc9892c02010-01-06 19:02:16 +0000206
207
Gwendal Grignou03286f02017-03-24 10:50:59 -0700208 def check_rsync(self):
mblighc9892c02010-01-06 19:02:16 +0000209 """
210 Check if rsync is available on the remote host.
211 """
212 try:
Allen Liad719c12017-06-27 23:48:04 +0000213 self.run("rsync --version", stdout_tee=None, stderr_tee=None)
mblighc9892c02010-01-06 19:02:16 +0000214 except error.AutoservRunError:
215 return False
216 return True
217
jadmanskica7da372008-10-21 16:26:52 +0000218
Gwendal Grignou36b61702016-02-10 11:57:53 -0800219 def _encode_remote_paths(self, paths, escape=True, use_scp=False):
mblighbc9402b2009-12-29 01:15:34 +0000220 """
221 Given a list of file paths, encodes it as a single remote path, in
222 the style used by rsync and scp.
Gwendal Grignou36b61702016-02-10 11:57:53 -0800223 escape: add \\ to protect special characters.
224 use_scp: encode for scp if true, rsync if false.
mblighbc9402b2009-12-29 01:15:34 +0000225 """
showard56176ec2009-10-28 19:52:30 +0000226 if escape:
227 paths = [utils.scp_remote_escape(path) for path in paths]
Marc Herbert21eb6492015-11-13 15:48:53 -0800228
229 remote = self.hostname
230
231 # rsync and scp require IPv6 brackets, even when there isn't any
232 # trailing port number (ssh doesn't support IPv6 brackets).
233 # In the Python >= 3.3 future, 'import ipaddress' will parse addresses.
234 if re.search(r':.*:', remote):
235 remote = '[%s]' % remote
236
Gwendal Grignou36b61702016-02-10 11:57:53 -0800237 if use_scp:
238 return '%s@%s:"%s"' % (self.user, remote, " ".join(paths))
239 else:
240 return '%s@%s:%s' % (
241 self.user, remote,
242 " :".join('"%s"' % p for p in paths))
jadmanskica7da372008-10-21 16:26:52 +0000243
Gwendal Grignou36b61702016-02-10 11:57:53 -0800244 def _encode_local_paths(self, paths, escape=True):
245 """
246 Given a list of file paths, encodes it as a single local path.
247 escape: add \\ to protect special characters.
248 """
249 if escape:
250 paths = [utils.sh_escape(path) for path in paths]
251
252 return " ".join('"%s"' % p for p in paths)
jadmanskica7da372008-10-21 16:26:52 +0000253
Dean Liaoe3e75f62017-11-14 10:36:43 +0800254
255 def rsync_options(self, delete_dest=False, preserve_symlinks=False,
256 safe_symlinks=False, excludes=None):
257 """Obtains rsync options for the remote."""
Fang Deng96667ca2013-08-01 17:46:18 -0700258 ssh_cmd = self.make_ssh_command(user=self.user, port=self.port,
Derek Beckett4d102242020-12-01 14:24:37 -0800259 opts=self._main_ssh.ssh_option,
Fang Deng96667ca2013-08-01 17:46:18 -0700260 hosts_file=self.known_hosts_file)
jadmanskid7b79ed2009-01-07 17:19:48 +0000261 if delete_dest:
262 delete_flag = "--delete"
263 else:
264 delete_flag = ""
Luigi Semenzato9b083072016-12-19 16:50:40 -0800265 if safe_symlinks:
266 symlink_flag = "-l --safe-links"
267 elif preserve_symlinks:
268 symlink_flag = "-l"
mbligh45561782009-05-11 21:14:34 +0000269 else:
270 symlink_flag = "-L"
Dan Shi92c34c92017-07-14 15:28:56 -0700271 exclude_args = ''
272 if excludes:
273 exclude_args = ' '.join(
274 ["--exclude '%s'" % exclude for exclude in excludes])
Dean Liaoe3e75f62017-11-14 10:36:43 +0800275 return "%s %s --timeout=1800 --rsh='%s' -az --no-o --no-g %s" % (
276 symlink_flag, delete_flag, ssh_cmd, exclude_args)
277
278
279 def _make_rsync_cmd(self, sources, dest, delete_dest,
280 preserve_symlinks, safe_symlinks, excludes=None):
281 """
282 Given a string of source paths and a destination path, produces the
283 appropriate rsync command for copying them. Remote paths must be
284 pre-encoded.
285 """
286 rsync_options = self.rsync_options(
287 delete_dest=delete_dest, preserve_symlinks=preserve_symlinks,
288 safe_symlinks=safe_symlinks, excludes=excludes)
289 return 'rsync %s %s "%s"' % (rsync_options, sources, dest)
jadmanskid7b79ed2009-01-07 17:19:48 +0000290
291
Eric Li861b2d52011-02-04 14:50:35 -0800292 def _make_ssh_cmd(self, cmd):
293 """
294 Create a base ssh command string for the host which can be used
295 to run commands directly on the machine
296 """
Fang Deng96667ca2013-08-01 17:46:18 -0700297 base_cmd = self.make_ssh_command(user=self.user, port=self.port,
Derek Beckett4d102242020-12-01 14:24:37 -0800298 opts=self._main_ssh.ssh_option,
Fang Deng96667ca2013-08-01 17:46:18 -0700299 hosts_file=self.known_hosts_file)
Eric Li861b2d52011-02-04 14:50:35 -0800300
301 return '%s %s "%s"' % (base_cmd, self.hostname, utils.sh_escape(cmd))
302
jadmanskid7b79ed2009-01-07 17:19:48 +0000303 def _make_scp_cmd(self, sources, dest):
mblighbc9402b2009-12-29 01:15:34 +0000304 """
Gwendal Grignou36b61702016-02-10 11:57:53 -0800305 Given a string of source paths and a destination path, produces the
jadmanskid7b79ed2009-01-07 17:19:48 +0000306 appropriate scp command for encoding it. Remote paths must be
mblighbc9402b2009-12-29 01:15:34 +0000307 pre-encoded.
308 """
mblighc0649d62010-01-15 18:15:58 +0000309 command = ("scp -rq %s -o StrictHostKeyChecking=no "
lmraf676f32010-02-04 03:36:26 +0000310 "-o UserKnownHostsFile=%s -P %d %s '%s'")
Derek Beckett4d102242020-12-01 14:24:37 -0800311 return command % (self._main_ssh.ssh_option, self.known_hosts_file,
Gwendal Grignou36b61702016-02-10 11:57:53 -0800312 self.port, sources, dest)
jadmanskid7b79ed2009-01-07 17:19:48 +0000313
314
315 def _make_rsync_compatible_globs(self, path, is_local):
mblighbc9402b2009-12-29 01:15:34 +0000316 """
317 Given an rsync-style path, returns a list of globbed paths
jadmanskid7b79ed2009-01-07 17:19:48 +0000318 that will hopefully provide equivalent behaviour for scp. Does not
319 support the full range of rsync pattern matching behaviour, only that
320 exposed in the get/send_file interface (trailing slashes).
321
322 The is_local param is flag indicating if the paths should be
mblighbc9402b2009-12-29 01:15:34 +0000323 interpreted as local or remote paths.
324 """
jadmanskid7b79ed2009-01-07 17:19:48 +0000325
326 # non-trailing slash paths should just work
327 if len(path) == 0 or path[-1] != "/":
328 return [path]
329
330 # make a function to test if a pattern matches any files
331 if is_local:
showard56176ec2009-10-28 19:52:30 +0000332 def glob_matches_files(path, pattern):
333 return len(glob.glob(path + pattern)) > 0
jadmanskid7b79ed2009-01-07 17:19:48 +0000334 else:
showard56176ec2009-10-28 19:52:30 +0000335 def glob_matches_files(path, pattern):
336 result = self.run("ls \"%s\"%s" % (utils.sh_escape(path),
337 pattern),
338 stdout_tee=None, ignore_status=True)
jadmanskid7b79ed2009-01-07 17:19:48 +0000339 return result.exit_status == 0
340
341 # take a set of globs that cover all files, and see which are needed
342 patterns = ["*", ".[!.]*"]
showard56176ec2009-10-28 19:52:30 +0000343 patterns = [p for p in patterns if glob_matches_files(path, p)]
jadmanskid7b79ed2009-01-07 17:19:48 +0000344
345 # convert them into a set of paths suitable for the commandline
jadmanskid7b79ed2009-01-07 17:19:48 +0000346 if is_local:
showard56176ec2009-10-28 19:52:30 +0000347 return ["\"%s\"%s" % (utils.sh_escape(path), pattern)
348 for pattern in patterns]
jadmanskid7b79ed2009-01-07 17:19:48 +0000349 else:
showard56176ec2009-10-28 19:52:30 +0000350 return [utils.scp_remote_escape(path) + pattern
351 for pattern in patterns]
jadmanskid7b79ed2009-01-07 17:19:48 +0000352
353
354 def _make_rsync_compatible_source(self, source, is_local):
mblighbc9402b2009-12-29 01:15:34 +0000355 """
356 Applies the same logic as _make_rsync_compatible_globs, but
jadmanskid7b79ed2009-01-07 17:19:48 +0000357 applies it to an entire list of sources, producing a new list of
mblighbc9402b2009-12-29 01:15:34 +0000358 sources, properly quoted.
359 """
jadmanskid7b79ed2009-01-07 17:19:48 +0000360 return sum((self._make_rsync_compatible_globs(path, is_local)
361 for path in source), [])
jadmanskica7da372008-10-21 16:26:52 +0000362
363
mblighfeac0102009-04-28 18:31:12 +0000364 def _set_umask_perms(self, dest):
mblighbc9402b2009-12-29 01:15:34 +0000365 """
366 Given a destination file/dir (recursively) set the permissions on
367 all the files and directories to the max allowed by running umask.
368 """
mblighfeac0102009-04-28 18:31:12 +0000369
370 # now this looks strange but I haven't found a way in Python to _just_
371 # get the umask, apparently the only option is to try to set it
372 umask = os.umask(0)
373 os.umask(umask)
374
Derek Beckettf73baca2020-08-19 15:08:47 -0700375 max_privs = 0o777 & ~umask
mblighfeac0102009-04-28 18:31:12 +0000376
377 def set_file_privs(filename):
Chris Masone567d0d92011-12-19 09:38:30 -0800378 """Sets mode of |filename|. Assumes |filename| exists."""
379 file_stat = os.stat(filename)
mblighfeac0102009-04-28 18:31:12 +0000380
381 file_privs = max_privs
382 # if the original file permissions do not have at least one
383 # executable bit then do not set it anywhere
Derek Beckettf73baca2020-08-19 15:08:47 -0700384 if not file_stat.st_mode & 0o111:
385 file_privs &= ~0o111
mblighfeac0102009-04-28 18:31:12 +0000386
387 os.chmod(filename, file_privs)
388
389 # try a bottom-up walk so changes on directory permissions won't cut
390 # our access to the files/directories inside it
391 for root, dirs, files in os.walk(dest, topdown=False):
392 # when setting the privileges we emulate the chmod "X" behaviour
393 # that sets to execute only if it is a directory or any of the
394 # owner/group/other already has execute right
395 for dirname in dirs:
396 os.chmod(os.path.join(root, dirname), max_privs)
397
Chris Masone567d0d92011-12-19 09:38:30 -0800398 # Filter out broken symlinks as we go.
399 for filename in filter(os.path.exists, files):
mblighfeac0102009-04-28 18:31:12 +0000400 set_file_privs(os.path.join(root, filename))
401
402
403 # now set privs for the dest itself
404 if os.path.isdir(dest):
405 os.chmod(dest, max_privs)
406 else:
407 set_file_privs(dest)
408
409
mbligh45561782009-05-11 21:14:34 +0000410 def get_file(self, source, dest, delete_dest=False, preserve_perm=True,
Dana Goyette4d864e12019-09-19 11:05:44 -0700411 preserve_symlinks=False, retry=True, safe_symlinks=False,
412 try_rsync=True):
jadmanskica7da372008-10-21 16:26:52 +0000413 """
414 Copy files from the remote host to a local path.
415
416 Directories will be copied recursively.
417 If a source component is a directory with a trailing slash,
418 the content of the directory will be copied, otherwise, the
419 directory itself and its content will be copied. This
420 behavior is similar to that of the program 'rsync'.
421
422 Args:
423 source: either
424 1) a single file or directory, as a string
425 2) a list of one or more (possibly mixed)
426 files or directories
427 dest: a file or a directory (if source contains a
428 directory or more than one element, you must
429 supply a directory dest)
mbligh89e258d2008-10-24 13:58:08 +0000430 delete_dest: if this is true, the command will also clear
431 out any old files at dest that are not in the
432 source
mblighfeac0102009-04-28 18:31:12 +0000433 preserve_perm: tells get_file() to try to preserve the sources
434 permissions on files and dirs
mbligh45561782009-05-11 21:14:34 +0000435 preserve_symlinks: try to preserve symlinks instead of
436 transforming them into files/dirs on copy
Luigi Semenzato9b083072016-12-19 16:50:40 -0800437 safe_symlinks: same as preserve_symlinks, but discard links
438 that may point outside the copied tree
Dana Goyette4d864e12019-09-19 11:05:44 -0700439 try_rsync: set to False to skip directly to using scp
jadmanskica7da372008-10-21 16:26:52 +0000440 Raises:
441 AutoservRunError: the scp command failed
442 """
Simran Basi882f15b2013-10-29 14:59:34 -0700443 logging.debug('get_file. source: %s, dest: %s, delete_dest: %s,'
444 'preserve_perm: %s, preserve_symlinks:%s', source, dest,
445 delete_dest, preserve_perm, preserve_symlinks)
Dan Shi4f8c0242017-07-07 15:34:49 -0700446
Derek Beckett4d102242020-12-01 14:24:37 -0800447 # Start a main SSH connection if necessary.
448 self.start_main_ssh()
mblighefccc1b2010-01-11 19:08:42 +0000449
Derek Beckettf73baca2020-08-19 15:08:47 -0700450 if isinstance(source, six.string_types):
jadmanskica7da372008-10-21 16:26:52 +0000451 source = [source]
jadmanskid7b79ed2009-01-07 17:19:48 +0000452 dest = os.path.abspath(dest)
jadmanskica7da372008-10-21 16:26:52 +0000453
mblighc9892c02010-01-06 19:02:16 +0000454 # If rsync is disabled or fails, try scp.
showard6eafb492010-01-15 20:29:06 +0000455 try_scp = True
Dana Goyette4d864e12019-09-19 11:05:44 -0700456 if try_rsync and self.use_rsync():
Simran Basi882f15b2013-10-29 14:59:34 -0700457 logging.debug('Using Rsync.')
mblighc9892c02010-01-06 19:02:16 +0000458 try:
459 remote_source = self._encode_remote_paths(source)
460 local_dest = utils.sh_escape(dest)
Gwendal Grignou36b61702016-02-10 11:57:53 -0800461 rsync = self._make_rsync_cmd(remote_source, local_dest,
Luigi Semenzato9b083072016-12-19 16:50:40 -0800462 delete_dest, preserve_symlinks,
463 safe_symlinks)
mblighc9892c02010-01-06 19:02:16 +0000464 utils.run(rsync)
showard6eafb492010-01-15 20:29:06 +0000465 try_scp = False
Derek Beckettf73baca2020-08-19 15:08:47 -0700466 except error.CmdError as e:
Luigi Semenzato7f9dff12016-11-21 14:01:20 -0800467 # retry on rsync exit values which may be caused by transient
468 # network problems:
469 #
470 # rc 10: Error in socket I/O
471 # rc 12: Error in rsync protocol data stream
472 # rc 23: Partial transfer due to error
473 # rc 255: Ssh error
474 #
475 # Note that rc 23 includes dangling symlinks. In this case
476 # retrying is useless, but not very damaging since rsync checks
477 # for those before starting the transfer (scp does not).
478 status = e.result_obj.exit_status
479 if status in [10, 12, 23, 255] and retry:
480 logging.warning('rsync status %d, retrying', status)
481 self.get_file(source, dest, delete_dest, preserve_perm,
482 preserve_symlinks, retry=False)
483 # The nested get_file() does all that's needed.
484 return
485 else:
486 logging.warning("trying scp, rsync failed: %s (%d)",
487 e, status)
mblighc9892c02010-01-06 19:02:16 +0000488
489 if try_scp:
Simran Basi882f15b2013-10-29 14:59:34 -0700490 logging.debug('Trying scp.')
jadmanskid7b79ed2009-01-07 17:19:48 +0000491 # scp has no equivalent to --delete, just drop the entire dest dir
492 if delete_dest and os.path.isdir(dest):
493 shutil.rmtree(dest)
494 os.mkdir(dest)
jadmanskica7da372008-10-21 16:26:52 +0000495
jadmanskid7b79ed2009-01-07 17:19:48 +0000496 remote_source = self._make_rsync_compatible_source(source, False)
497 if remote_source:
showard56176ec2009-10-28 19:52:30 +0000498 # _make_rsync_compatible_source() already did the escaping
Gwendal Grignou36b61702016-02-10 11:57:53 -0800499 remote_source = self._encode_remote_paths(
500 remote_source, escape=False, use_scp=True)
jadmanskid7b79ed2009-01-07 17:19:48 +0000501 local_dest = utils.sh_escape(dest)
Gwendal Grignou36b61702016-02-10 11:57:53 -0800502 scp = self._make_scp_cmd(remote_source, local_dest)
jadmanskid7b79ed2009-01-07 17:19:48 +0000503 try:
504 utils.run(scp)
Derek Beckettf73baca2020-08-19 15:08:47 -0700505 except error.CmdError as e:
Simran Basi882f15b2013-10-29 14:59:34 -0700506 logging.debug('scp failed: %s', e)
jadmanskid7b79ed2009-01-07 17:19:48 +0000507 raise error.AutoservRunError(e.args[0], e.args[1])
jadmanskica7da372008-10-21 16:26:52 +0000508
mblighfeac0102009-04-28 18:31:12 +0000509 if not preserve_perm:
510 # we have no way to tell scp to not try to preserve the
511 # permissions so set them after copy instead.
512 # for rsync we could use "--no-p --chmod=ugo=rwX" but those
513 # options are only in very recent rsync versions
514 self._set_umask_perms(dest)
515
jadmanskica7da372008-10-21 16:26:52 +0000516
mbligh45561782009-05-11 21:14:34 +0000517 def send_file(self, source, dest, delete_dest=False,
Dan Shi92c34c92017-07-14 15:28:56 -0700518 preserve_symlinks=False, excludes=None):
jadmanskica7da372008-10-21 16:26:52 +0000519 """
520 Copy files from a local path to the remote host.
521
522 Directories will be copied recursively.
523 If a source component is a directory with a trailing slash,
524 the content of the directory will be copied, otherwise, the
525 directory itself and its content will be copied. This
526 behavior is similar to that of the program 'rsync'.
527
528 Args:
529 source: either
530 1) a single file or directory, as a string
531 2) a list of one or more (possibly mixed)
532 files or directories
533 dest: a file or a directory (if source contains a
534 directory or more than one element, you must
535 supply a directory dest)
mbligh89e258d2008-10-24 13:58:08 +0000536 delete_dest: if this is true, the command will also clear
537 out any old files at dest that are not in the
538 source
mbligh45561782009-05-11 21:14:34 +0000539 preserve_symlinks: controls if symlinks on the source will be
540 copied as such on the destination or transformed into the
541 referenced file/directory
Dan Shi92c34c92017-07-14 15:28:56 -0700542 excludes: A list of file pattern that matches files not to be
543 sent. `send_file` will fail if exclude is set, since
544 local copy does not support --exclude, e.g., when
545 using scp to copy file.
jadmanskica7da372008-10-21 16:26:52 +0000546
547 Raises:
548 AutoservRunError: the scp command failed
549 """
Simran Basi882f15b2013-10-29 14:59:34 -0700550 logging.debug('send_file. source: %s, dest: %s, delete_dest: %s,'
551 'preserve_symlinks:%s', source, dest,
552 delete_dest, preserve_symlinks)
Derek Beckett4d102242020-12-01 14:24:37 -0800553 # Start a main SSH connection if necessary.
554 self.start_main_ssh()
mblighefccc1b2010-01-11 19:08:42 +0000555
Derek Beckettf73baca2020-08-19 15:08:47 -0700556 if isinstance(source, six.string_types):
jadmanskica7da372008-10-21 16:26:52 +0000557 source = [source]
558
Gwendal Grignou36b61702016-02-10 11:57:53 -0800559 local_sources = self._encode_local_paths(source)
mukesh agrawal0d3616c2015-07-17 15:47:36 -0700560 if not local_sources:
Gwendal Grignou36b61702016-02-10 11:57:53 -0800561 raise error.TestError('source |%s| yielded an empty string' % (
mukesh agrawal0d3616c2015-07-17 15:47:36 -0700562 source))
Gwendal Grignou36b61702016-02-10 11:57:53 -0800563 if local_sources.find('\x00') != -1:
mukesh agrawal0d3616c2015-07-17 15:47:36 -0700564 raise error.TestError('one or more sources include NUL char')
565
Derek Beckettfc04fbc2020-12-14 16:37:31 -0800566 send_client_symlink = False
567
568 # We want to specifically ensure that the symlink client/autotest_lib
569 # is preserved, but not force the preservation on other files/dirs.
570 client_symlink = _client_symlink(source)
571 if client_symlink and not preserve_symlinks:
572 source, local_sources = self._remove_dir_from_sources(
573 source, local_sources, client_symlink)
574 send_client_symlink = True
575
576 self._send_file(
577 dest=dest,
578 source=source,
579 local_sources=local_sources,
580 delete_dest=delete_dest,
581 excludes=excludes,
582 preserve_symlinks=preserve_symlinks)
583 if send_client_symlink:
584 self._send_file(dest=dest,
585 source=[client_symlink],
586 local_sources=client_symlink,
587 delete_dest=delete_dest,
588 excludes=excludes,
589 preserve_symlinks=True)
590
591 def _remove_dir_from_sources(self, source, local_sources, rm_dir):
592 """Remove the specified dir from the source/local_sources.
593
594 If source is a list, remove it from the list. If a string, and the
595 string is the rm_dir, remove it. Else leave as is.
596 Args:
597 source: either
598 1) a single file or directory, as a string
599 2) a list of one or more (possibly mixed)
600 files or directories
601 local_sources: str created from _encode_local_paths()
602 rm_dir: str of the dir to remove
603 Returns:
604 source, local_sources (with the dir removed)
605
606 """
607 if isinstance(source, list):
608 source.remove(rm_dir)
609 elif rm_dir in source:
610 source = ''
611
612 return source, local_sources.replace('"{}" '.format(rm_dir), '')
613
614 def _send_file(self, dest, source, local_sources, delete_dest, excludes,
615 preserve_symlinks):
616 """Send file(s), trying rsync first, then scp."""
showard6eafb492010-01-15 20:29:06 +0000617 if self.use_rsync():
Derek Beckettfc04fbc2020-12-14 16:37:31 -0800618 rsync_success = self._send_using_rsync(
619 dest=dest,
620 local_sources=local_sources,
621 delete_dest=delete_dest,
622 preserve_symlinks=preserve_symlinks,
623 excludes=excludes)
Derek Beckettb0821d82021-01-08 11:33:39 -0800624 if rsync_success:
625 return
Derek Beckettfc04fbc2020-12-14 16:37:31 -0800626
Derek Beckettb0821d82021-01-08 11:33:39 -0800627 # Send using scp if you cannot via rsync, or rsync fails.
628 self._send_using_scp(dest=dest,
629 source=source,
630 delete_dest=delete_dest,
631 excludes=excludes)
Derek Beckettfc04fbc2020-12-14 16:37:31 -0800632
633 def _send_using_rsync(self, dest, local_sources, delete_dest,
634 preserve_symlinks, excludes):
635 """Send using rsync.
636
637 Args:
638 dest: a file or a directory (if source contains a
639 directory or more than one element, you must
640 supply a directory dest)
641 local_sources: a string of files/dirs to send separated with spaces
642 delete_dest: if this is true, the command will also clear
643 out any old files at dest that are not in the
644 source
645 preserve_symlinks: controls if symlinks on the source will be
646 copied as such on the destination or transformed into the
647 referenced file/directory
648 excludes: A list of file pattern that matches files not to be
649 sent. `send_file` will fail if exclude is set, since
650 local copy does not support --exclude, e.g., when
651 using scp to copy file.
652 Returns:
653 bool: True if the cmd succeeded, else False
654
655 """
656 logging.debug('Using Rsync.')
657 remote_dest = self._encode_remote_paths([dest])
658 try:
659 rsync = self._make_rsync_cmd(local_sources,
660 remote_dest,
661 delete_dest,
662 preserve_symlinks,
663 False,
664 excludes=excludes)
665 utils.run(rsync)
666 return True
667 except error.CmdError as e:
668 logging.warning("trying scp, rsync failed: %s", e)
669 return False
670
671 def _send_using_scp(self, dest, source, delete_dest, excludes):
672 """Send using scp.
673
674 Args:
675 source: either
676 1) a single file or directory, as a string
677 2) a list of one or more (possibly mixed)
678 files or directories
679 dest: a file or a directory (if source contains a
680 directory or more than one element, you must
681 supply a directory dest)
682 delete_dest: if this is true, the command will also clear
683 out any old files at dest that are not in the
684 source
685 excludes: A list of file pattern that matches files not to be
686 sent. `send_file` will fail if exclude is set, since
687 local copy does not support --exclude, e.g., when
688 using scp to copy file.
689
690 Raises:
691 AutoservRunError: the scp command failed
692 """
693 logging.debug('Trying scp.')
694 if excludes:
695 raise error.AutotestHostRunError(
696 '--exclude is not supported in scp, try to use rsync. '
697 'excludes: %s' % ','.join(excludes), None)
698
699 # scp has no equivalent to --delete, just drop the entire dest dir
700 if delete_dest:
701 is_dir = self.run("ls -d %s/" % dest,
702 ignore_status=True).exit_status == 0
703 if is_dir:
704 cmd = "rm -rf %s && mkdir %s"
705 cmd %= (dest, dest)
706 self.run(cmd)
707
708 remote_dest = self._encode_remote_paths([dest], use_scp=True)
709 local_sources = self._make_rsync_compatible_source(source, True)
710 if local_sources:
711 sources = self._encode_local_paths(local_sources, escape=False)
712 scp = self._make_scp_cmd(sources, remote_dest)
mblighc9892c02010-01-06 19:02:16 +0000713 try:
Derek Beckettfc04fbc2020-12-14 16:37:31 -0800714 utils.run(scp)
Derek Beckettf73baca2020-08-19 15:08:47 -0700715 except error.CmdError as e:
Derek Beckettfc04fbc2020-12-14 16:37:31 -0800716 logging.debug('scp failed: %s', e)
717 raise error.AutoservRunError(e.args[0], e.args[1])
718 else:
719 logging.debug('skipping scp for empty source list')
jadmanskid7b79ed2009-01-07 17:19:48 +0000720
Simran Basi1621c632015-10-14 12:22:23 -0700721 def verify_ssh_user_access(self):
722 """Verify ssh access to this host.
723
724 @returns False if ssh_ping fails due to Permissions error, True
725 otherwise.
726 """
727 try:
728 self.ssh_ping()
729 except (error.AutoservSshPermissionDeniedError,
730 error.AutoservSshPingHostError):
731 return False
732 return True
733
734
Luigi Semenzato135574c2016-08-31 17:25:08 -0700735 def ssh_ping(self, timeout=60, connect_timeout=None, base_cmd='true'):
beepsadd66d32013-03-04 17:21:51 -0800736 """
737 Pings remote host via ssh.
738
Philip Chen7ce1e392018-12-09 23:53:32 -0800739 @param timeout: Command execution timeout in seconds.
beepsadd66d32013-03-04 17:21:51 -0800740 Defaults to 60 seconds.
Philip Chen7ce1e392018-12-09 23:53:32 -0800741 @param connect_timeout: ssh connection timeout in seconds.
beeps46dadc92013-11-07 14:07:10 -0800742 @param base_cmd: The base command to run with the ssh ping.
743 Defaults to true.
beepsadd66d32013-03-04 17:21:51 -0800744 @raise AutoservSSHTimeout: If the ssh ping times out.
745 @raise AutoservSshPermissionDeniedError: If ssh ping fails due to
746 permissions.
747 @raise AutoservSshPingHostError: For other AutoservRunErrors.
748 """
Luigi Semenzato135574c2016-08-31 17:25:08 -0700749 ctimeout = min(timeout, connect_timeout or timeout)
jadmanskica7da372008-10-21 16:26:52 +0000750 try:
Allen Liad719c12017-06-27 23:48:04 +0000751 self.run(base_cmd, timeout=timeout, connect_timeout=ctimeout,
752 ssh_failure_retry_ok=True)
jadmanskica7da372008-10-21 16:26:52 +0000753 except error.AutoservSSHTimeout:
mblighd0e94982009-07-11 00:15:18 +0000754 msg = "Host (ssh) verify timed out (timeout = %d)" % timeout
jadmanskica7da372008-10-21 16:26:52 +0000755 raise error.AutoservSSHTimeout(msg)
mbligh9d738d62009-03-09 21:17:10 +0000756 except error.AutoservSshPermissionDeniedError:
Allen Liad719c12017-06-27 23:48:04 +0000757 #let AutoservSshPermissionDeniedError be visible to the callers
mbligh9d738d62009-03-09 21:17:10 +0000758 raise
Derek Beckettf73baca2020-08-19 15:08:47 -0700759 except error.AutoservRunError as e:
mblighc971c5f2009-06-08 16:48:54 +0000760 # convert the generic AutoservRunError into something more
761 # specific for this context
762 raise error.AutoservSshPingHostError(e.description + '\n' +
763 repr(e.result_obj))
jadmanskica7da372008-10-21 16:26:52 +0000764
765
Luigi Semenzato135574c2016-08-31 17:25:08 -0700766 def is_up(self, timeout=60, connect_timeout=None, base_cmd='true'):
jadmanskica7da372008-10-21 16:26:52 +0000767 """
beeps46dadc92013-11-07 14:07:10 -0800768 Check if the remote host is up by ssh-ing and running a base command.
jadmanskica7da372008-10-21 16:26:52 +0000769
Philip Chen7ce1e392018-12-09 23:53:32 -0800770 @param timeout: command execution timeout in seconds.
771 @param connect_timeout: ssh connection timeout in seconds.
beeps46dadc92013-11-07 14:07:10 -0800772 @param base_cmd: a base command to run with ssh. The default is 'true'.
beepsadd66d32013-03-04 17:21:51 -0800773 @returns True if the remote host is up before the timeout expires,
774 False otherwise.
jadmanskica7da372008-10-21 16:26:52 +0000775 """
776 try:
Luigi Semenzato135574c2016-08-31 17:25:08 -0700777 self.ssh_ping(timeout=timeout,
778 connect_timeout=connect_timeout,
779 base_cmd=base_cmd)
jadmanskica7da372008-10-21 16:26:52 +0000780 except error.AutoservError:
781 return False
782 else:
783 return True
784
785
Otabek Kasimovd48389b2020-12-07 02:38:34 -0800786 def is_up_fast(self, count=1):
787 """Return True if the host can be pinged.
788
789 @param count How many time try to ping before decide that host is not
790 reachable by ping.
791 """
792 ping_config = ping_runner.PingConfig(self.hostname,
793 count=count,
794 ignore_result=True,
795 ignore_status=True)
Dan Shi9f92aa62017-07-27 17:07:05 -0700796 return ping_runner.PingRunner().ping(ping_config).received > 0
797
798
Philip Chen7ce1e392018-12-09 23:53:32 -0800799 def wait_up(self, timeout=_DEFAULT_WAIT_UP_TIME_SECONDS):
jadmanskica7da372008-10-21 16:26:52 +0000800 """
801 Wait until the remote host is up or the timeout expires.
802
803 In fact, it will wait until an ssh connection to the remote
804 host can be established, and getty is running.
805
jadmanskic0354912010-01-12 15:57:29 +0000806 @param timeout time limit in seconds before returning even
807 if the host is not up.
jadmanskica7da372008-10-21 16:26:52 +0000808
beepsadd66d32013-03-04 17:21:51 -0800809 @returns True if the host was found to be up before the timeout expires,
810 False otherwise
jadmanskica7da372008-10-21 16:26:52 +0000811 """
Philip Chen7ce1e392018-12-09 23:53:32 -0800812 current_time = int(time.time())
813 end_time = current_time + timeout
jadmanskica7da372008-10-21 16:26:52 +0000814
Luigi Semenzato135574c2016-08-31 17:25:08 -0700815 autoserv_error_logged = False
Philip Chen7ce1e392018-12-09 23:53:32 -0800816 while current_time < end_time:
817 ping_timeout = min(_DEFAULT_MAX_PING_TIMEOUT,
818 end_time - current_time)
819 if self.is_up(timeout=ping_timeout, connect_timeout=ping_timeout):
jadmanskica7da372008-10-21 16:26:52 +0000820 try:
821 if self.are_wait_up_processes_up():
Dean Liaoe3e75f62017-11-14 10:36:43 +0800822 logging.debug('Host %s is now up', self.host_port)
jadmanskica7da372008-10-21 16:26:52 +0000823 return True
Luigi Semenzato135574c2016-08-31 17:25:08 -0700824 except error.AutoservError as e:
825 if not autoserv_error_logged:
826 logging.debug('Ignoring failure to reach %s: %s %s',
Dean Liaoe3e75f62017-11-14 10:36:43 +0800827 self.host_port, e,
Luigi Semenzato135574c2016-08-31 17:25:08 -0700828 '(and further similar failures)')
829 autoserv_error_logged = True
jadmanskica7da372008-10-21 16:26:52 +0000830 time.sleep(1)
beeps46dadc92013-11-07 14:07:10 -0800831 current_time = int(time.time())
jadmanskica7da372008-10-21 16:26:52 +0000832
jadmanski7ebac3d2010-06-17 16:06:31 +0000833 logging.debug('Host %s is still down after waiting %d seconds',
Dean Liaoe3e75f62017-11-14 10:36:43 +0800834 self.host_port, int(timeout + time.time() - end_time))
jadmanskica7da372008-10-21 16:26:52 +0000835 return False
836
837
Lutz Justen043e9c12017-10-27 12:40:47 +0200838 def wait_down(self, timeout=_DEFAULT_WAIT_DOWN_TIME_SECONDS,
839 warning_timer=None, old_boot_id=None,
840 max_ping_timeout=_DEFAULT_MAX_PING_TIMEOUT):
jadmanskica7da372008-10-21 16:26:52 +0000841 """
842 Wait until the remote host is down or the timeout expires.
843
Lutz Justen043e9c12017-10-27 12:40:47 +0200844 If old_boot_id is provided, waits until either the machine is
845 unpingable or self.get_boot_id() returns a value different from
jadmanskic0354912010-01-12 15:57:29 +0000846 old_boot_id. If the boot_id value has changed then the function
Lutz Justen043e9c12017-10-27 12:40:47 +0200847 returns True under the assumption that the machine has shut down
jadmanskic0354912010-01-12 15:57:29 +0000848 and has now already come back up.
jadmanskica7da372008-10-21 16:26:52 +0000849
jadmanskic0354912010-01-12 15:57:29 +0000850 If old_boot_id is None then until the machine becomes unreachable the
851 method assumes the machine has not yet shut down.
jadmanskica7da372008-10-21 16:26:52 +0000852
Lutz Justen043e9c12017-10-27 12:40:47 +0200853 @param timeout Time limit in seconds before returning even if the host
854 is still up.
855 @param warning_timer Time limit in seconds that will generate a warning
856 if the host is not down yet. Can be None for no warning.
jadmanskic0354912010-01-12 15:57:29 +0000857 @param old_boot_id A string containing the result of self.get_boot_id()
858 prior to the host being told to shut down. Can be None if this is
859 not available.
Lutz Justen043e9c12017-10-27 12:40:47 +0200860 @param max_ping_timeout Maximum timeout in seconds for each
861 self.get_boot_id() call. If this timeout is hit, it is assumed that
862 the host went down and became unreachable.
jadmanskic0354912010-01-12 15:57:29 +0000863
Lutz Justen043e9c12017-10-27 12:40:47 +0200864 @returns True if the host was found to be down (max_ping_timeout timeout
865 expired or boot_id changed if provided) and False if timeout
866 expired.
jadmanskica7da372008-10-21 16:26:52 +0000867 """
mblighe5e3cf22010-05-27 23:33:14 +0000868 #TODO: there is currently no way to distinguish between knowing
869 #TODO: boot_id was unsupported and not knowing the boot_id.
beeps46dadc92013-11-07 14:07:10 -0800870 current_time = int(time.time())
Lutz Justen043e9c12017-10-27 12:40:47 +0200871 end_time = current_time + timeout
jadmanskica7da372008-10-21 16:26:52 +0000872
mbligh2ed998f2009-04-08 21:03:47 +0000873 if warning_timer:
874 warn_time = current_time + warning_timer
875
jadmanskic0354912010-01-12 15:57:29 +0000876 if old_boot_id is not None:
877 logging.debug('Host %s pre-shutdown boot_id is %s',
Dean Liaoe3e75f62017-11-14 10:36:43 +0800878 self.host_port, old_boot_id)
jadmanskic0354912010-01-12 15:57:29 +0000879
beepsadd66d32013-03-04 17:21:51 -0800880 # Impose semi real-time deadline constraints, since some clients
881 # (eg: watchdog timer tests) expect strict checking of time elapsed.
882 # Each iteration of this loop is treated as though it atomically
883 # completes within current_time, this is needed because if we used
884 # inline time.time() calls instead then the following could happen:
885 #
Lutz Justen043e9c12017-10-27 12:40:47 +0200886 # while time.time() < end_time: [23 < 30]
beepsadd66d32013-03-04 17:21:51 -0800887 # some code. [takes 10 secs]
888 # try:
889 # new_boot_id = self.get_boot_id(timeout=end_time - time.time())
890 # [30 - 33]
891 # The last step will lead to a return True, when in fact the machine
892 # went down at 32 seconds (>30). Hence we need to pass get_boot_id
893 # the same time that allowed us into that iteration of the loop.
Lutz Justen043e9c12017-10-27 12:40:47 +0200894 while current_time < end_time:
895 ping_timeout = min(end_time - current_time, max_ping_timeout)
jadmanskic0354912010-01-12 15:57:29 +0000896 try:
Lutz Justen043e9c12017-10-27 12:40:47 +0200897 new_boot_id = self.get_boot_id(timeout=ping_timeout)
mblighdbc7e4a2010-01-15 20:34:20 +0000898 except error.AutoservError:
jadmanskic0354912010-01-12 15:57:29 +0000899 logging.debug('Host %s is now unreachable over ssh, is down',
Dean Liaoe3e75f62017-11-14 10:36:43 +0800900 self.host_port)
jadmanskica7da372008-10-21 16:26:52 +0000901 return True
jadmanskic0354912010-01-12 15:57:29 +0000902 else:
903 # if the machine is up but the boot_id value has changed from
904 # old boot id, then we can assume the machine has gone down
905 # and then already come back up
906 if old_boot_id is not None and old_boot_id != new_boot_id:
907 logging.debug('Host %s now has boot_id %s and so must '
Dean Liaoe3e75f62017-11-14 10:36:43 +0800908 'have rebooted', self.host_port, new_boot_id)
jadmanskic0354912010-01-12 15:57:29 +0000909 return True
mbligh2ed998f2009-04-08 21:03:47 +0000910
911 if warning_timer and current_time > warn_time:
Scott Zawalskic86fdeb2013-10-23 10:24:04 -0400912 self.record("INFO", None, "shutdown",
mbligh2ed998f2009-04-08 21:03:47 +0000913 "Shutdown took longer than %ds" % warning_timer)
914 # Print the warning only once.
915 warning_timer = None
mbligha4464402009-04-17 20:13:41 +0000916 # If a machine is stuck switching runlevels
917 # This may cause the machine to reboot.
918 self.run('kill -HUP 1', ignore_status=True)
mbligh2ed998f2009-04-08 21:03:47 +0000919
jadmanskica7da372008-10-21 16:26:52 +0000920 time.sleep(1)
beeps46dadc92013-11-07 14:07:10 -0800921 current_time = int(time.time())
jadmanskica7da372008-10-21 16:26:52 +0000922
923 return False
jadmanskif6562912008-10-21 17:59:01 +0000924
mbligha0a27592009-01-24 01:41:36 +0000925
jadmanskif6562912008-10-21 17:59:01 +0000926 # tunable constants for the verify & repair code
mblighb86bfa12010-02-12 20:22:21 +0000927 AUTOTEST_GB_DISKSPACE_REQUIRED = get_value("SERVER",
928 "gb_diskspace_required",
Fang Deng6b05f5b2013-03-20 13:42:11 -0700929 type=float,
930 default=20.0)
mbligha0a27592009-01-24 01:41:36 +0000931
jadmanskif6562912008-10-21 17:59:01 +0000932
showardca572982009-09-18 21:20:01 +0000933 def verify_connectivity(self):
934 super(AbstractSSHHost, self).verify_connectivity()
jadmanskif6562912008-10-21 17:59:01 +0000935
Dean Liaoe3e75f62017-11-14 10:36:43 +0800936 logging.info('Pinging host ' + self.host_port)
jadmanskif6562912008-10-21 17:59:01 +0000937 self.ssh_ping()
Dean Liaoe3e75f62017-11-14 10:36:43 +0800938 logging.info("Host (ssh) %s is alive", self.host_port)
jadmanskif6562912008-10-21 17:59:01 +0000939
jadmanski80deb752009-01-21 17:14:16 +0000940 if self.is_shutting_down():
mblighc971c5f2009-06-08 16:48:54 +0000941 raise error.AutoservHostIsShuttingDownError("Host is shutting down")
jadmanski80deb752009-01-21 17:14:16 +0000942
mblighb49b5232009-02-12 21:54:49 +0000943
showardca572982009-09-18 21:20:01 +0000944 def verify_software(self):
945 super(AbstractSSHHost, self).verify_software()
jadmanskif6562912008-10-21 17:59:01 +0000946 try:
showardad812bf2009-10-20 23:49:56 +0000947 self.check_diskspace(autotest.Autotest.get_install_dir(self),
948 self.AUTOTEST_GB_DISKSPACE_REQUIRED)
Keith Haddow07f1d3e2017-08-03 17:40:41 -0700949 except error.AutoservDiskFullHostError:
950 # only want to raise if it's a space issue
951 raise
952 except (error.AutoservHostError, autotest.AutodirNotFoundError):
Lutz Justen043e9c12017-10-27 12:40:47 +0200953 logging.exception('autodir space check exception, this is probably '
Keith Haddow07f1d3e2017-08-03 17:40:41 -0700954 'safe to ignore\n')
mblighefccc1b2010-01-11 19:08:42 +0000955
956
957 def close(self):
958 super(AbstractSSHHost, self).close()
Godofredo Contreras773179e2016-05-24 10:17:48 -0700959 self.rpc_server_tracker.disconnect_all()
Hidehiko Abe06893302017-06-24 07:32:38 +0900960 if not self._connection_pool:
Derek Beckett4d102242020-12-01 14:24:37 -0800961 self._main_ssh.close()
xixuand6011f12016-12-08 15:01:58 -0800962 if os.path.exists(self.known_hosts_file):
963 os.remove(self.known_hosts_file)
mblighefccc1b2010-01-11 19:08:42 +0000964
965
Derek Beckett4d102242020-12-01 14:24:37 -0800966 def restart_main_ssh(self):
Luigi Semenzato3b95ede2016-12-09 11:51:01 -0800967 """
Derek Beckett4d102242020-12-01 14:24:37 -0800968 Stop and restart the ssh main connection. This is meant as a last
Luigi Semenzato3b95ede2016-12-09 11:51:01 -0800969 resort when ssh commands fail and we don't understand why.
970 """
Derek Beckett4d102242020-12-01 14:24:37 -0800971 logging.debug('Restarting main ssh connection')
972 self._main_ssh.close()
973 self._main_ssh.maybe_start(timeout=30)
Luigi Semenzato3b95ede2016-12-09 11:51:01 -0800974
975
mblighefccc1b2010-01-11 19:08:42 +0000976
Derek Beckett4d102242020-12-01 14:24:37 -0800977 def start_main_ssh(self, timeout=DEFAULT_START_MAIN_SSH_TIMEOUT_S):
mblighefccc1b2010-01-11 19:08:42 +0000978 """
Derek Beckett4d102242020-12-01 14:24:37 -0800979 Called whenever a non-main SSH connection needs to be initiated (e.g.,
980 by run, rsync, scp). If main SSH support is enabled and a main SSH
mblighefccc1b2010-01-11 19:08:42 +0000981 connection is not active already, start a new one in the background.
Derek Beckett4d102242020-12-01 14:24:37 -0800982 Also, cleanup any zombie main SSH connections (e.g., dead due to
mblighefccc1b2010-01-11 19:08:42 +0000983 reboot).
Aviv Keshet0749a822013-10-17 09:53:26 -0700984
Derek Beckett4d102242020-12-01 14:24:37 -0800985 timeout: timeout in seconds (default 5) to wait for main ssh
Aviv Keshet0749a822013-10-17 09:53:26 -0700986 connection to be established. If timeout is reached, a
987 warning message is logged, but no other action is taken.
mblighefccc1b2010-01-11 19:08:42 +0000988 """
Derek Beckett4d102242020-12-01 14:24:37 -0800989 if not enable_main_ssh:
mblighefccc1b2010-01-11 19:08:42 +0000990 return
Derek Beckett4d102242020-12-01 14:24:37 -0800991 self._main_ssh.maybe_start(timeout=timeout)
mbligh0a883702010-04-21 01:58:34 +0000992
993
994 def clear_known_hosts(self):
995 """Clears out the temporary ssh known_hosts file.
996
997 This is useful if the test SSHes to the machine, then reinstalls it,
998 then SSHes to it again. It can be called after the reinstall to
999 reduce the spam in the logs.
1000 """
1001 logging.info("Clearing known hosts for host '%s', file '%s'.",
Dean Liaoe3e75f62017-11-14 10:36:43 +08001002 self.host_port, self.known_hosts_file)
mbligh0a883702010-04-21 01:58:34 +00001003 # Clear out the file by opening it for writing and then closing.
Fang Deng3af66202013-08-16 15:19:25 -07001004 fh = open(self.known_hosts_file, "w")
mbligh0a883702010-04-21 01:58:34 +00001005 fh.close()
Prashanth B98509c72014-04-04 16:01:34 -07001006
1007
1008 def collect_logs(self, remote_src_dir, local_dest_dir, ignore_errors=True):
1009 """Copy log directories from a host to a local directory.
1010
1011 @param remote_src_dir: A destination directory on the host.
1012 @param local_dest_dir: A path to a local destination directory.
1013 If it doesn't exist it will be created.
1014 @param ignore_errors: If True, ignore exceptions.
1015
1016 @raises OSError: If there were problems creating the local_dest_dir and
1017 ignore_errors is False.
1018 @raises AutoservRunError, AutotestRunError: If something goes wrong
1019 while copying the directories and ignore_errors is False.
1020 """
Dan Shi9f92aa62017-07-27 17:07:05 -07001021 if not self.check_cached_up_status():
1022 logging.warning('Host %s did not answer to ping, skip collecting '
Dean Liaoe3e75f62017-11-14 10:36:43 +08001023 'logs.', self.host_port)
Dan Shi9f92aa62017-07-27 17:07:05 -07001024 return
1025
Prashanth B98509c72014-04-04 16:01:34 -07001026 locally_created_dest = False
1027 if (not os.path.exists(local_dest_dir)
1028 or not os.path.isdir(local_dest_dir)):
1029 try:
1030 os.makedirs(local_dest_dir)
1031 locally_created_dest = True
1032 except OSError as e:
1033 logging.warning('Unable to collect logs from host '
Dean Liaoe3e75f62017-11-14 10:36:43 +08001034 '%s: %s', self.host_port, e)
Prashanth B98509c72014-04-04 16:01:34 -07001035 if not ignore_errors:
1036 raise
1037 return
Dan Shi4f8c0242017-07-07 15:34:49 -07001038
1039 # Build test result directory summary
1040 try:
1041 result_tools_runner.run_on_client(self, remote_src_dir)
1042 except (error.AutotestRunError, error.AutoservRunError,
1043 error.AutoservSSHTimeout) as e:
1044 logging.exception(
1045 'Non-critical failure: Failed to collect and throttle '
Dean Liaoe3e75f62017-11-14 10:36:43 +08001046 'results at %s from host %s', remote_src_dir,
1047 self.host_port)
Dan Shi4f8c0242017-07-07 15:34:49 -07001048
Prashanth B98509c72014-04-04 16:01:34 -07001049 try:
Luigi Semenzato9b083072016-12-19 16:50:40 -08001050 self.get_file(remote_src_dir, local_dest_dir, safe_symlinks=True)
Prashanth B98509c72014-04-04 16:01:34 -07001051 except (error.AutotestRunError, error.AutoservRunError,
1052 error.AutoservSSHTimeout) as e:
1053 logging.warning('Collection of %s to local dir %s from host %s '
1054 'failed: %s', remote_src_dir, local_dest_dir,
Dean Liaoe3e75f62017-11-14 10:36:43 +08001055 self.host_port, e)
Prashanth B98509c72014-04-04 16:01:34 -07001056 if locally_created_dest:
1057 shutil.rmtree(local_dest_dir, ignore_errors=ignore_errors)
1058 if not ignore_errors:
1059 raise
Cheng-Yi Chianga155e7e2015-08-20 20:42:04 +08001060
Dan Shi4f8c0242017-07-07 15:34:49 -07001061 # Clean up directory summary file on the client side.
1062 try:
1063 result_tools_runner.run_on_client(self, remote_src_dir,
1064 cleanup_only=True)
1065 except (error.AutotestRunError, error.AutoservRunError,
1066 error.AutoservSSHTimeout) as e:
1067 logging.exception(
1068 'Non-critical failure: Failed to cleanup result summary '
Lutz Justen043e9c12017-10-27 12:40:47 +02001069 'files at %s in host %s', remote_src_dir, self.hostname)
Dan Shi4f8c0242017-07-07 15:34:49 -07001070
Cheng-Yi Chianga155e7e2015-08-20 20:42:04 +08001071
xixuan6cf6d2f2016-01-29 15:29:00 -08001072 def create_ssh_tunnel(self, port, local_port):
Cheng-Yi Chianga155e7e2015-08-20 20:42:04 +08001073 """Create an ssh tunnel from local_port to port.
1074
xixuan6cf6d2f2016-01-29 15:29:00 -08001075 This is used to forward a port securely through a tunnel process from
1076 the server to the DUT for RPC server connection.
1077
Cheng-Yi Chianga155e7e2015-08-20 20:42:04 +08001078 @param port: remote port on the host.
1079 @param local_port: local forwarding port.
1080
1081 @return: the tunnel process.
1082 """
1083 tunnel_options = '-n -N -q -L %d:localhost:%d' % (local_port, port)
Prathmesh Prabhu817b3f12017-07-31 17:08:41 -07001084 ssh_cmd = self.make_ssh_command(opts=tunnel_options, port=self.port)
Cheng-Yi Chianga155e7e2015-08-20 20:42:04 +08001085 tunnel_cmd = '%s %s' % (ssh_cmd, self.hostname)
1086 logging.debug('Full tunnel command: %s', tunnel_cmd)
xixuan6cf6d2f2016-01-29 15:29:00 -08001087 # Exec the ssh process directly here rather than using a shell.
1088 # Using a shell leaves a dangling ssh process, because we deliver
1089 # signals to the shell wrapping ssh, not the ssh process itself.
1090 args = shlex.split(tunnel_cmd)
Kuang-che Wu0ea03232019-08-31 10:52:31 +08001091 with open('/dev/null', 'w') as devnull:
1092 tunnel_proc = subprocess.Popen(args, stdout=devnull, stderr=devnull,
1093 close_fds=True)
Cheng-Yi Chianga155e7e2015-08-20 20:42:04 +08001094 logging.debug('Started ssh tunnel, local = %d'
1095 ' remote = %d, pid = %d',
1096 local_port, port, tunnel_proc.pid)
1097 return tunnel_proc
Gilad Arnolda76bef02015-09-29 13:55:15 -07001098
1099
Oleg Loskutoff1199bbb2019-10-21 12:27:13 -07001100 def disconnect_ssh_tunnel(self, tunnel_proc):
Roshan Pius58e5dd32015-10-16 15:16:42 -07001101 """
1102 Disconnects a previously forwarded port from the server to the DUT for
1103 RPC server connection.
1104
xixuan6cf6d2f2016-01-29 15:29:00 -08001105 @param tunnel_proc: a tunnel process returned from |create_ssh_tunnel|.
Roshan Pius58e5dd32015-10-16 15:16:42 -07001106 """
1107 if tunnel_proc.poll() is None:
1108 tunnel_proc.terminate()
1109 logging.debug('Terminated tunnel, pid %d', tunnel_proc.pid)
1110 else:
1111 logging.debug('Tunnel pid %d terminated early, status %d',
1112 tunnel_proc.pid, tunnel_proc.returncode)
1113
1114
Gilad Arnolda76bef02015-09-29 13:55:15 -07001115 def get_os_type(self):
1116 """Returns the host OS descriptor (to be implemented in subclasses).
1117
1118 @return A string describing the OS type.
1119 """
Gwendal Grignou36b61702016-02-10 11:57:53 -08001120 raise NotImplementedError
Dan Shi9f92aa62017-07-27 17:07:05 -07001121
1122
1123 def check_cached_up_status(
1124 self, expiration_seconds=_DEFAULT_UP_STATUS_EXPIRATION_SECONDS):
1125 """Check if the DUT responded to ping in the past `expiration_seconds`.
1126
1127 @param expiration_seconds: The number of seconds to keep the cached
1128 status of whether the DUT responded to ping.
1129 @return: True if the DUT has responded to ping during the past
1130 `expiration_seconds`.
1131 """
1132 # Refresh the up status if any of following conditions is true:
1133 # * cached status is never set
1134 # * cached status is False, so the method can check if the host is up
1135 # again.
1136 # * If the cached status is older than `expiration_seconds`
1137 expire_time = time.time() - expiration_seconds
1138 if (self._cached_up_status_updated is None or
1139 not self._cached_up_status or
1140 self._cached_up_status_updated < expire_time):
1141 self._cached_up_status = self.is_up_fast()
1142 self._cached_up_status_updated = time.time()
1143 return self._cached_up_status
Otabek Kasimovf5e0f102020-06-30 19:41:02 -07001144
1145
1146 def _track_class_usage(self):
1147 """Tracking which class was used.
1148
1149 The idea to identify unused classes to be able clean them up.
1150 We skip names with dynamic created classes where the name is
1151 hostname of the device.
1152 """
1153 class_name = None
1154 if 'chrome' not in self.__class__.__name__:
1155 class_name = self.__class__.__name__
1156 else:
1157 for base in self.__class__.__bases__:
1158 if 'chrome' not in base.__name__:
1159 class_name = base.__name__
1160 break
1161 if class_name:
1162 data = {'host_class': class_name}
1163 metrics.Counter(
1164 'chromeos/autotest/used_hosts').increment(fields=data)
Garry Wang81bea592020-08-27 17:25:25 -07001165
1166 def is_file_exists(self, file_path):
1167 """Check whether a given file is exist on the host.
1168 """
1169 result = self.run('test -f ' + file_path,
1170 timeout=30,
1171 ignore_status=True)
1172 return result.exit_status == 0
Derek Beckettfc04fbc2020-12-14 16:37:31 -08001173
1174
1175def _client_symlink(sources):
1176 """Return the client symlink if in sources."""
1177 for source in sources:
1178 if source.endswith(AUTOTEST_CLIENT_SYMLINK_END):
1179 return source
1180 return None