blob: 3d8d9e9b5c3fe6928e15aa126bac887a598cfbc3 [file] [log] [blame]
showardca572982009-09-18 21:20:01 +00001import os, time, types, socket, shutil, glob, logging, traceback
mblighefccc1b2010-01-11 19:08:42 +00002from autotest_lib.client.common_lib import autotemp, error, logging_manager
jadmanski31c49b72008-10-27 20:44:48 +00003from autotest_lib.server import utils, autotest
mblighe8b93af2009-01-30 00:45:53 +00004from autotest_lib.server.hosts import remote
mblighefccc1b2010-01-11 19:08:42 +00005from autotest_lib.client.common_lib.global_config import global_config
jadmanskica7da372008-10-21 16:26:52 +00006
7
mblighb86bfa12010-02-12 20:22:21 +00008get_value = global_config.get_config_value
9enable_master_ssh = get_value('AUTOSERV', 'enable_master_ssh', type=bool,
10 default=False)
mblighefccc1b2010-01-11 19:08:42 +000011
12
lmraf676f32010-02-04 03:36:26 +000013def make_ssh_command(user="root", port=22, opts='', hosts_file='/dev/null',
14 connect_timeout=30, alive_interval=300):
15 base_command = ("/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no "
16 "-o UserKnownHostsFile=%s -o BatchMode=yes "
mblighefccc1b2010-01-11 19:08:42 +000017 "-o ConnectTimeout=%d -o ServerAliveInterval=%d "
jadmanskica7da372008-10-21 16:26:52 +000018 "-l %s -p %d")
19 assert isinstance(connect_timeout, (int, long))
20 assert connect_timeout > 0 # can't disable the timeout
lmraf676f32010-02-04 03:36:26 +000021 return base_command % (opts, hosts_file, connect_timeout,
22 alive_interval, user, port)
jadmanskica7da372008-10-21 16:26:52 +000023
24
mblighe8b93af2009-01-30 00:45:53 +000025# import site specific Host class
26SiteHost = utils.import_site_class(
27 __file__, "autotest_lib.server.hosts.site_host", "SiteHost",
28 remote.RemoteHost)
29
30
31class AbstractSSHHost(SiteHost):
mblighbc9402b2009-12-29 01:15:34 +000032 """
33 This class represents a generic implementation of most of the
jadmanskica7da372008-10-21 16:26:52 +000034 framework necessary for controlling a host via ssh. It implements
35 almost all of the abstract Host methods, except for the core
mblighbc9402b2009-12-29 01:15:34 +000036 Host.run method.
37 """
jadmanskica7da372008-10-21 16:26:52 +000038
jadmanskif6562912008-10-21 17:59:01 +000039 def _initialize(self, hostname, user="root", port=22, password="",
40 *args, **dargs):
41 super(AbstractSSHHost, self)._initialize(hostname=hostname,
42 *args, **dargs)
mbligh6369cf22008-10-24 17:21:57 +000043 self.ip = socket.getaddrinfo(self.hostname, None)[0][4][0]
jadmanskica7da372008-10-21 16:26:52 +000044 self.user = user
45 self.port = port
46 self.password = password
showard6eafb492010-01-15 20:29:06 +000047 self._use_rsync = None
lmraf676f32010-02-04 03:36:26 +000048 self.known_hosts_file = os.tmpfile()
49 known_hosts_fd = self.known_hosts_file.fileno()
50 self.known_hosts_fd = '/dev/fd/%s' % known_hosts_fd
jadmanskica7da372008-10-21 16:26:52 +000051
mblighefccc1b2010-01-11 19:08:42 +000052 """
53 Master SSH connection background job, socket temp directory and socket
54 control path option. If master-SSH is enabled, these fields will be
55 initialized by start_master_ssh when a new SSH connection is initiated.
56 """
57 self.master_ssh_job = None
58 self.master_ssh_tempdir = None
59 self.master_ssh_option = ''
60
showard6eafb492010-01-15 20:29:06 +000061
62 def use_rsync(self):
63 if self._use_rsync is not None:
64 return self._use_rsync
65
mblighc9892c02010-01-06 19:02:16 +000066 # Check if rsync is available on the remote host. If it's not,
67 # don't try to use it for any future file transfers.
showard6eafb492010-01-15 20:29:06 +000068 self._use_rsync = self._check_rsync()
69 if not self._use_rsync:
mblighc9892c02010-01-06 19:02:16 +000070 logging.warn("rsync not available on remote host %s -- disabled",
71 self.hostname)
Eric Lie0493a42010-11-15 13:05:43 -080072 return self._use_rsync
mblighc9892c02010-01-06 19:02:16 +000073
74
75 def _check_rsync(self):
76 """
77 Check if rsync is available on the remote host.
78 """
79 try:
80 self.run("rsync --version", stdout_tee=None, stderr_tee=None)
81 except error.AutoservRunError:
82 return False
83 return True
84
jadmanskica7da372008-10-21 16:26:52 +000085
showard56176ec2009-10-28 19:52:30 +000086 def _encode_remote_paths(self, paths, escape=True):
mblighbc9402b2009-12-29 01:15:34 +000087 """
88 Given a list of file paths, encodes it as a single remote path, in
89 the style used by rsync and scp.
90 """
showard56176ec2009-10-28 19:52:30 +000091 if escape:
92 paths = [utils.scp_remote_escape(path) for path in paths]
93 return '%s@%s:"%s"' % (self.user, self.hostname, " ".join(paths))
jadmanskica7da372008-10-21 16:26:52 +000094
jadmanskica7da372008-10-21 16:26:52 +000095
mbligh45561782009-05-11 21:14:34 +000096 def _make_rsync_cmd(self, sources, dest, delete_dest, preserve_symlinks):
mblighbc9402b2009-12-29 01:15:34 +000097 """
98 Given a list of source paths and a destination path, produces the
jadmanskid7b79ed2009-01-07 17:19:48 +000099 appropriate rsync command for copying them. Remote paths must be
mblighbc9402b2009-12-29 01:15:34 +0000100 pre-encoded.
101 """
lmraf676f32010-02-04 03:36:26 +0000102 ssh_cmd = make_ssh_command(user=self.user, port=self.port,
103 opts=self.master_ssh_option,
104 hosts_file=self.known_hosts_fd)
jadmanskid7b79ed2009-01-07 17:19:48 +0000105 if delete_dest:
106 delete_flag = "--delete"
107 else:
108 delete_flag = ""
mbligh45561782009-05-11 21:14:34 +0000109 if preserve_symlinks:
110 symlink_flag = ""
111 else:
112 symlink_flag = "-L"
113 command = "rsync %s %s --timeout=1800 --rsh='%s' -az %s %s"
114 return command % (symlink_flag, delete_flag, ssh_cmd,
115 " ".join(sources), dest)
jadmanskid7b79ed2009-01-07 17:19:48 +0000116
117
118 def _make_scp_cmd(self, sources, dest):
mblighbc9402b2009-12-29 01:15:34 +0000119 """
120 Given a list of source paths and a destination path, produces the
jadmanskid7b79ed2009-01-07 17:19:48 +0000121 appropriate scp command for encoding it. Remote paths must be
mblighbc9402b2009-12-29 01:15:34 +0000122 pre-encoded.
123 """
mblighc0649d62010-01-15 18:15:58 +0000124 command = ("scp -rq %s -o StrictHostKeyChecking=no "
lmraf676f32010-02-04 03:36:26 +0000125 "-o UserKnownHostsFile=%s -P %d %s '%s'")
126 return command % (self.master_ssh_option, self.known_hosts_fd,
mblighefccc1b2010-01-11 19:08:42 +0000127 self.port, " ".join(sources), dest)
jadmanskid7b79ed2009-01-07 17:19:48 +0000128
129
130 def _make_rsync_compatible_globs(self, path, is_local):
mblighbc9402b2009-12-29 01:15:34 +0000131 """
132 Given an rsync-style path, returns a list of globbed paths
jadmanskid7b79ed2009-01-07 17:19:48 +0000133 that will hopefully provide equivalent behaviour for scp. Does not
134 support the full range of rsync pattern matching behaviour, only that
135 exposed in the get/send_file interface (trailing slashes).
136
137 The is_local param is flag indicating if the paths should be
mblighbc9402b2009-12-29 01:15:34 +0000138 interpreted as local or remote paths.
139 """
jadmanskid7b79ed2009-01-07 17:19:48 +0000140
141 # non-trailing slash paths should just work
142 if len(path) == 0 or path[-1] != "/":
143 return [path]
144
145 # make a function to test if a pattern matches any files
146 if is_local:
showard56176ec2009-10-28 19:52:30 +0000147 def glob_matches_files(path, pattern):
148 return len(glob.glob(path + pattern)) > 0
jadmanskid7b79ed2009-01-07 17:19:48 +0000149 else:
showard56176ec2009-10-28 19:52:30 +0000150 def glob_matches_files(path, pattern):
151 result = self.run("ls \"%s\"%s" % (utils.sh_escape(path),
152 pattern),
153 stdout_tee=None, ignore_status=True)
jadmanskid7b79ed2009-01-07 17:19:48 +0000154 return result.exit_status == 0
155
156 # take a set of globs that cover all files, and see which are needed
157 patterns = ["*", ".[!.]*"]
showard56176ec2009-10-28 19:52:30 +0000158 patterns = [p for p in patterns if glob_matches_files(path, p)]
jadmanskid7b79ed2009-01-07 17:19:48 +0000159
160 # convert them into a set of paths suitable for the commandline
jadmanskid7b79ed2009-01-07 17:19:48 +0000161 if is_local:
showard56176ec2009-10-28 19:52:30 +0000162 return ["\"%s\"%s" % (utils.sh_escape(path), pattern)
163 for pattern in patterns]
jadmanskid7b79ed2009-01-07 17:19:48 +0000164 else:
showard56176ec2009-10-28 19:52:30 +0000165 return [utils.scp_remote_escape(path) + pattern
166 for pattern in patterns]
jadmanskid7b79ed2009-01-07 17:19:48 +0000167
168
169 def _make_rsync_compatible_source(self, source, is_local):
mblighbc9402b2009-12-29 01:15:34 +0000170 """
171 Applies the same logic as _make_rsync_compatible_globs, but
jadmanskid7b79ed2009-01-07 17:19:48 +0000172 applies it to an entire list of sources, producing a new list of
mblighbc9402b2009-12-29 01:15:34 +0000173 sources, properly quoted.
174 """
jadmanskid7b79ed2009-01-07 17:19:48 +0000175 return sum((self._make_rsync_compatible_globs(path, is_local)
176 for path in source), [])
jadmanskica7da372008-10-21 16:26:52 +0000177
178
mblighfeac0102009-04-28 18:31:12 +0000179 def _set_umask_perms(self, dest):
mblighbc9402b2009-12-29 01:15:34 +0000180 """
181 Given a destination file/dir (recursively) set the permissions on
182 all the files and directories to the max allowed by running umask.
183 """
mblighfeac0102009-04-28 18:31:12 +0000184
185 # now this looks strange but I haven't found a way in Python to _just_
186 # get the umask, apparently the only option is to try to set it
187 umask = os.umask(0)
188 os.umask(umask)
189
190 max_privs = 0777 & ~umask
191
192 def set_file_privs(filename):
193 file_stat = os.stat(filename)
194
195 file_privs = max_privs
196 # if the original file permissions do not have at least one
197 # executable bit then do not set it anywhere
198 if not file_stat.st_mode & 0111:
199 file_privs &= ~0111
200
201 os.chmod(filename, file_privs)
202
203 # try a bottom-up walk so changes on directory permissions won't cut
204 # our access to the files/directories inside it
205 for root, dirs, files in os.walk(dest, topdown=False):
206 # when setting the privileges we emulate the chmod "X" behaviour
207 # that sets to execute only if it is a directory or any of the
208 # owner/group/other already has execute right
209 for dirname in dirs:
210 os.chmod(os.path.join(root, dirname), max_privs)
211
212 for filename in files:
213 set_file_privs(os.path.join(root, filename))
214
215
216 # now set privs for the dest itself
217 if os.path.isdir(dest):
218 os.chmod(dest, max_privs)
219 else:
220 set_file_privs(dest)
221
222
mbligh45561782009-05-11 21:14:34 +0000223 def get_file(self, source, dest, delete_dest=False, preserve_perm=True,
224 preserve_symlinks=False):
jadmanskica7da372008-10-21 16:26:52 +0000225 """
226 Copy files from the remote host to a local path.
227
228 Directories will be copied recursively.
229 If a source component is a directory with a trailing slash,
230 the content of the directory will be copied, otherwise, the
231 directory itself and its content will be copied. This
232 behavior is similar to that of the program 'rsync'.
233
234 Args:
235 source: either
236 1) a single file or directory, as a string
237 2) a list of one or more (possibly mixed)
238 files or directories
239 dest: a file or a directory (if source contains a
240 directory or more than one element, you must
241 supply a directory dest)
mbligh89e258d2008-10-24 13:58:08 +0000242 delete_dest: if this is true, the command will also clear
243 out any old files at dest that are not in the
244 source
mblighfeac0102009-04-28 18:31:12 +0000245 preserve_perm: tells get_file() to try to preserve the sources
246 permissions on files and dirs
mbligh45561782009-05-11 21:14:34 +0000247 preserve_symlinks: try to preserve symlinks instead of
248 transforming them into files/dirs on copy
jadmanskica7da372008-10-21 16:26:52 +0000249
250 Raises:
251 AutoservRunError: the scp command failed
252 """
mblighefccc1b2010-01-11 19:08:42 +0000253
254 # Start a master SSH connection if necessary.
255 self.start_master_ssh()
256
jadmanskica7da372008-10-21 16:26:52 +0000257 if isinstance(source, basestring):
258 source = [source]
jadmanskid7b79ed2009-01-07 17:19:48 +0000259 dest = os.path.abspath(dest)
jadmanskica7da372008-10-21 16:26:52 +0000260
mblighc9892c02010-01-06 19:02:16 +0000261 # If rsync is disabled or fails, try scp.
showard6eafb492010-01-15 20:29:06 +0000262 try_scp = True
263 if self.use_rsync():
mblighc9892c02010-01-06 19:02:16 +0000264 try:
265 remote_source = self._encode_remote_paths(source)
266 local_dest = utils.sh_escape(dest)
267 rsync = self._make_rsync_cmd([remote_source], local_dest,
268 delete_dest, preserve_symlinks)
269 utils.run(rsync)
showard6eafb492010-01-15 20:29:06 +0000270 try_scp = False
mblighc9892c02010-01-06 19:02:16 +0000271 except error.CmdError, e:
272 logging.warn("trying scp, rsync failed: %s" % e)
mblighc9892c02010-01-06 19:02:16 +0000273
274 if try_scp:
jadmanskid7b79ed2009-01-07 17:19:48 +0000275 # scp has no equivalent to --delete, just drop the entire dest dir
276 if delete_dest and os.path.isdir(dest):
277 shutil.rmtree(dest)
278 os.mkdir(dest)
jadmanskica7da372008-10-21 16:26:52 +0000279
jadmanskid7b79ed2009-01-07 17:19:48 +0000280 remote_source = self._make_rsync_compatible_source(source, False)
281 if remote_source:
showard56176ec2009-10-28 19:52:30 +0000282 # _make_rsync_compatible_source() already did the escaping
283 remote_source = self._encode_remote_paths(remote_source,
284 escape=False)
jadmanskid7b79ed2009-01-07 17:19:48 +0000285 local_dest = utils.sh_escape(dest)
jadmanski2583a432009-02-10 23:59:11 +0000286 scp = self._make_scp_cmd([remote_source], local_dest)
jadmanskid7b79ed2009-01-07 17:19:48 +0000287 try:
288 utils.run(scp)
289 except error.CmdError, e:
290 raise error.AutoservRunError(e.args[0], e.args[1])
jadmanskica7da372008-10-21 16:26:52 +0000291
mblighfeac0102009-04-28 18:31:12 +0000292 if not preserve_perm:
293 # we have no way to tell scp to not try to preserve the
294 # permissions so set them after copy instead.
295 # for rsync we could use "--no-p --chmod=ugo=rwX" but those
296 # options are only in very recent rsync versions
297 self._set_umask_perms(dest)
298
jadmanskica7da372008-10-21 16:26:52 +0000299
mbligh45561782009-05-11 21:14:34 +0000300 def send_file(self, source, dest, delete_dest=False,
301 preserve_symlinks=False):
jadmanskica7da372008-10-21 16:26:52 +0000302 """
303 Copy files from a local path to the remote host.
304
305 Directories will be copied recursively.
306 If a source component is a directory with a trailing slash,
307 the content of the directory will be copied, otherwise, the
308 directory itself and its content will be copied. This
309 behavior is similar to that of the program 'rsync'.
310
311 Args:
312 source: either
313 1) a single file or directory, as a string
314 2) a list of one or more (possibly mixed)
315 files or directories
316 dest: a file or a directory (if source contains a
317 directory or more than one element, you must
318 supply a directory dest)
mbligh89e258d2008-10-24 13:58:08 +0000319 delete_dest: if this is true, the command will also clear
320 out any old files at dest that are not in the
321 source
mbligh45561782009-05-11 21:14:34 +0000322 preserve_symlinks: controls if symlinks on the source will be
323 copied as such on the destination or transformed into the
324 referenced file/directory
jadmanskica7da372008-10-21 16:26:52 +0000325
326 Raises:
327 AutoservRunError: the scp command failed
328 """
mblighefccc1b2010-01-11 19:08:42 +0000329
330 # Start a master SSH connection if necessary.
331 self.start_master_ssh()
332
jadmanskica7da372008-10-21 16:26:52 +0000333 if isinstance(source, basestring):
334 source = [source]
jadmanski2583a432009-02-10 23:59:11 +0000335 remote_dest = self._encode_remote_paths([dest])
jadmanskica7da372008-10-21 16:26:52 +0000336
mblighc9892c02010-01-06 19:02:16 +0000337 # If rsync is disabled or fails, try scp.
showard6eafb492010-01-15 20:29:06 +0000338 try_scp = True
339 if self.use_rsync():
mblighc9892c02010-01-06 19:02:16 +0000340 try:
341 local_sources = [utils.sh_escape(path) for path in source]
342 rsync = self._make_rsync_cmd(local_sources, remote_dest,
343 delete_dest, preserve_symlinks)
344 utils.run(rsync)
showard6eafb492010-01-15 20:29:06 +0000345 try_scp = False
mblighc9892c02010-01-06 19:02:16 +0000346 except error.CmdError, e:
347 logging.warn("trying scp, rsync failed: %s" % e)
mblighc9892c02010-01-06 19:02:16 +0000348
349 if try_scp:
jadmanskid7b79ed2009-01-07 17:19:48 +0000350 # scp has no equivalent to --delete, just drop the entire dest dir
351 if delete_dest:
showard27160152009-07-15 14:28:42 +0000352 is_dir = self.run("ls -d %s/" % dest,
jadmanskid7b79ed2009-01-07 17:19:48 +0000353 ignore_status=True).exit_status == 0
354 if is_dir:
355 cmd = "rm -rf %s && mkdir %s"
mbligh5a0ca532009-08-03 16:44:34 +0000356 cmd %= (dest, dest)
jadmanskid7b79ed2009-01-07 17:19:48 +0000357 self.run(cmd)
jadmanskica7da372008-10-21 16:26:52 +0000358
jadmanski2583a432009-02-10 23:59:11 +0000359 local_sources = self._make_rsync_compatible_source(source, True)
360 if local_sources:
361 scp = self._make_scp_cmd(local_sources, remote_dest)
jadmanskid7b79ed2009-01-07 17:19:48 +0000362 try:
363 utils.run(scp)
364 except error.CmdError, e:
365 raise error.AutoservRunError(e.args[0], e.args[1])
366
jadmanskica7da372008-10-21 16:26:52 +0000367
368 def ssh_ping(self, timeout=60):
369 try:
370 self.run("true", timeout=timeout, connect_timeout=timeout)
371 except error.AutoservSSHTimeout:
mblighd0e94982009-07-11 00:15:18 +0000372 msg = "Host (ssh) verify timed out (timeout = %d)" % timeout
jadmanskica7da372008-10-21 16:26:52 +0000373 raise error.AutoservSSHTimeout(msg)
mbligh9d738d62009-03-09 21:17:10 +0000374 except error.AutoservSshPermissionDeniedError:
375 #let AutoservSshPermissionDeniedError be visible to the callers
376 raise
jadmanskica7da372008-10-21 16:26:52 +0000377 except error.AutoservRunError, e:
mblighc971c5f2009-06-08 16:48:54 +0000378 # convert the generic AutoservRunError into something more
379 # specific for this context
380 raise error.AutoservSshPingHostError(e.description + '\n' +
381 repr(e.result_obj))
jadmanskica7da372008-10-21 16:26:52 +0000382
383
384 def is_up(self):
385 """
386 Check if the remote host is up.
387
jadmanskic0354912010-01-12 15:57:29 +0000388 @returns True if the remote host is up, False otherwise
jadmanskica7da372008-10-21 16:26:52 +0000389 """
390 try:
391 self.ssh_ping()
392 except error.AutoservError:
393 return False
394 else:
395 return True
396
397
398 def wait_up(self, timeout=None):
399 """
400 Wait until the remote host is up or the timeout expires.
401
402 In fact, it will wait until an ssh connection to the remote
403 host can be established, and getty is running.
404
jadmanskic0354912010-01-12 15:57:29 +0000405 @param timeout time limit in seconds before returning even
406 if the host is not up.
jadmanskica7da372008-10-21 16:26:52 +0000407
jadmanskic0354912010-01-12 15:57:29 +0000408 @returns True if the host was found to be up, False otherwise
jadmanskica7da372008-10-21 16:26:52 +0000409 """
410 if timeout:
411 end_time = time.time() + timeout
412
413 while not timeout or time.time() < end_time:
414 if self.is_up():
415 try:
416 if self.are_wait_up_processes_up():
jadmanski7ebac3d2010-06-17 16:06:31 +0000417 logging.debug('Host %s is now up', self.hostname)
jadmanskica7da372008-10-21 16:26:52 +0000418 return True
419 except error.AutoservError:
420 pass
421 time.sleep(1)
422
jadmanski7ebac3d2010-06-17 16:06:31 +0000423 logging.debug('Host %s is still down after waiting %d seconds',
424 self.hostname, int(timeout + time.time() - end_time))
jadmanskica7da372008-10-21 16:26:52 +0000425 return False
426
427
jadmanskic0354912010-01-12 15:57:29 +0000428 def wait_down(self, timeout=None, warning_timer=None, old_boot_id=None):
jadmanskica7da372008-10-21 16:26:52 +0000429 """
430 Wait until the remote host is down or the timeout expires.
431
jadmanskic0354912010-01-12 15:57:29 +0000432 If old_boot_id is provided, this will wait until either the machine
433 is unpingable or self.get_boot_id() returns a value different from
434 old_boot_id. If the boot_id value has changed then the function
435 returns true under the assumption that the machine has shut down
436 and has now already come back up.
jadmanskica7da372008-10-21 16:26:52 +0000437
jadmanskic0354912010-01-12 15:57:29 +0000438 If old_boot_id is None then until the machine becomes unreachable the
439 method assumes the machine has not yet shut down.
jadmanskica7da372008-10-21 16:26:52 +0000440
jadmanskic0354912010-01-12 15:57:29 +0000441 @param timeout Time limit in seconds before returning even
442 if the host is still up.
443 @param warning_timer Time limit in seconds that will generate
444 a warning if the host is not down yet.
445 @param old_boot_id A string containing the result of self.get_boot_id()
446 prior to the host being told to shut down. Can be None if this is
447 not available.
448
449 @returns True if the host was found to be down, False otherwise
jadmanskica7da372008-10-21 16:26:52 +0000450 """
mblighe5e3cf22010-05-27 23:33:14 +0000451 #TODO: there is currently no way to distinguish between knowing
452 #TODO: boot_id was unsupported and not knowing the boot_id.
mbligh2ed998f2009-04-08 21:03:47 +0000453 current_time = time.time()
jadmanskica7da372008-10-21 16:26:52 +0000454 if timeout:
mbligh2ed998f2009-04-08 21:03:47 +0000455 end_time = current_time + timeout
jadmanskica7da372008-10-21 16:26:52 +0000456
mbligh2ed998f2009-04-08 21:03:47 +0000457 if warning_timer:
458 warn_time = current_time + warning_timer
459
jadmanskic0354912010-01-12 15:57:29 +0000460 if old_boot_id is not None:
461 logging.debug('Host %s pre-shutdown boot_id is %s',
462 self.hostname, old_boot_id)
463
mbligh2ed998f2009-04-08 21:03:47 +0000464 while not timeout or current_time < end_time:
jadmanskic0354912010-01-12 15:57:29 +0000465 try:
466 new_boot_id = self.get_boot_id()
mblighdbc7e4a2010-01-15 20:34:20 +0000467 except error.AutoservError:
jadmanskic0354912010-01-12 15:57:29 +0000468 logging.debug('Host %s is now unreachable over ssh, is down',
469 self.hostname)
jadmanskica7da372008-10-21 16:26:52 +0000470 return True
jadmanskic0354912010-01-12 15:57:29 +0000471 else:
472 # if the machine is up but the boot_id value has changed from
473 # old boot id, then we can assume the machine has gone down
474 # and then already come back up
475 if old_boot_id is not None and old_boot_id != new_boot_id:
476 logging.debug('Host %s now has boot_id %s and so must '
477 'have rebooted', self.hostname, new_boot_id)
478 return True
mbligh2ed998f2009-04-08 21:03:47 +0000479
480 if warning_timer and current_time > warn_time:
481 self.record("WARN", None, "shutdown",
482 "Shutdown took longer than %ds" % warning_timer)
483 # Print the warning only once.
484 warning_timer = None
mbligha4464402009-04-17 20:13:41 +0000485 # If a machine is stuck switching runlevels
486 # This may cause the machine to reboot.
487 self.run('kill -HUP 1', ignore_status=True)
mbligh2ed998f2009-04-08 21:03:47 +0000488
jadmanskica7da372008-10-21 16:26:52 +0000489 time.sleep(1)
mbligh2ed998f2009-04-08 21:03:47 +0000490 current_time = time.time()
jadmanskica7da372008-10-21 16:26:52 +0000491
492 return False
jadmanskif6562912008-10-21 17:59:01 +0000493
mbligha0a27592009-01-24 01:41:36 +0000494
jadmanskif6562912008-10-21 17:59:01 +0000495 # tunable constants for the verify & repair code
mblighb86bfa12010-02-12 20:22:21 +0000496 AUTOTEST_GB_DISKSPACE_REQUIRED = get_value("SERVER",
497 "gb_diskspace_required",
498 type=int,
499 default=20)
mbligha0a27592009-01-24 01:41:36 +0000500
jadmanskif6562912008-10-21 17:59:01 +0000501
showardca572982009-09-18 21:20:01 +0000502 def verify_connectivity(self):
503 super(AbstractSSHHost, self).verify_connectivity()
jadmanskif6562912008-10-21 17:59:01 +0000504
showardb18134f2009-03-20 20:52:18 +0000505 logging.info('Pinging host ' + self.hostname)
jadmanskif6562912008-10-21 17:59:01 +0000506 self.ssh_ping()
mbligh2ba7ab02009-08-24 22:09:26 +0000507 logging.info("Host (ssh) %s is alive", self.hostname)
jadmanskif6562912008-10-21 17:59:01 +0000508
jadmanski80deb752009-01-21 17:14:16 +0000509 if self.is_shutting_down():
mblighc971c5f2009-06-08 16:48:54 +0000510 raise error.AutoservHostIsShuttingDownError("Host is shutting down")
jadmanski80deb752009-01-21 17:14:16 +0000511
mblighb49b5232009-02-12 21:54:49 +0000512
showardca572982009-09-18 21:20:01 +0000513 def verify_software(self):
514 super(AbstractSSHHost, self).verify_software()
jadmanskif6562912008-10-21 17:59:01 +0000515 try:
showardad812bf2009-10-20 23:49:56 +0000516 self.check_diskspace(autotest.Autotest.get_install_dir(self),
517 self.AUTOTEST_GB_DISKSPACE_REQUIRED)
jadmanskif6562912008-10-21 17:59:01 +0000518 except error.AutoservHostError:
519 raise # only want to raise if it's a space issue
showardad812bf2009-10-20 23:49:56 +0000520 except autotest.AutodirNotFoundError:
showardca572982009-09-18 21:20:01 +0000521 # autotest dir may not exist, etc. ignore
522 logging.debug('autodir space check exception, this is probably '
523 'safe to ignore\n' + traceback.format_exc())
mblighefccc1b2010-01-11 19:08:42 +0000524
525
526 def close(self):
527 super(AbstractSSHHost, self).close()
528 self._cleanup_master_ssh()
lmraf676f32010-02-04 03:36:26 +0000529 self.known_hosts_file.close()
mblighefccc1b2010-01-11 19:08:42 +0000530
531
532 def _cleanup_master_ssh(self):
533 """
534 Release all resources (process, temporary directory) used by an active
535 master SSH connection.
536 """
537 # If a master SSH connection is running, kill it.
538 if self.master_ssh_job is not None:
539 utils.nuke_subprocess(self.master_ssh_job.sp)
540 self.master_ssh_job = None
541
542 # Remove the temporary directory for the master SSH socket.
543 if self.master_ssh_tempdir is not None:
544 self.master_ssh_tempdir.clean()
545 self.master_ssh_tempdir = None
546 self.master_ssh_option = ''
547
548
549 def start_master_ssh(self):
550 """
551 Called whenever a slave SSH connection needs to be initiated (e.g., by
552 run, rsync, scp). If master SSH support is enabled and a master SSH
553 connection is not active already, start a new one in the background.
554 Also, cleanup any zombie master SSH connections (e.g., dead due to
555 reboot).
556 """
557 if not enable_master_ssh:
558 return
559
560 # If a previously started master SSH connection is not running
561 # anymore, it needs to be cleaned up and then restarted.
562 if self.master_ssh_job is not None:
563 if self.master_ssh_job.sp.poll() is not None:
564 logging.info("Master ssh connection to %s is down.",
565 self.hostname)
566 self._cleanup_master_ssh()
567
568 # Start a new master SSH connection.
569 if self.master_ssh_job is None:
570 # Create a shared socket in a temp location.
571 self.master_ssh_tempdir = autotemp.tempdir(unique_id='ssh-master')
572 self.master_ssh_option = ("-o ControlPath=%s/socket" %
573 self.master_ssh_tempdir.name)
574
575 # Start the master SSH connection in the background.
mbligh5644c122010-01-29 17:43:26 +0000576 master_cmd = self.ssh_command(options="-N -o ControlMaster=yes")
mblighefccc1b2010-01-11 19:08:42 +0000577 logging.info("Starting master ssh connection '%s'" % master_cmd)
578 self.master_ssh_job = utils.BgJob(master_cmd)
mbligh0a883702010-04-21 01:58:34 +0000579
580
581 def clear_known_hosts(self):
582 """Clears out the temporary ssh known_hosts file.
583
584 This is useful if the test SSHes to the machine, then reinstalls it,
585 then SSHes to it again. It can be called after the reinstall to
586 reduce the spam in the logs.
587 """
588 logging.info("Clearing known hosts for host '%s', file '%s'.",
589 self.hostname, self.known_hosts_fd)
590 # Clear out the file by opening it for writing and then closing.
591 fh = open(self.known_hosts_fd, "w")
592 fh.close()