showard | ca57298 | 2009-09-18 21:20:01 +0000 | [diff] [blame] | 1 | import os, time, types, socket, shutil, glob, logging, traceback |
mbligh | efccc1b | 2010-01-11 19:08:42 +0000 | [diff] [blame] | 2 | from autotest_lib.client.common_lib import autotemp, error, logging_manager |
jadmanski | 31c49b7 | 2008-10-27 20:44:48 +0000 | [diff] [blame] | 3 | from autotest_lib.server import utils, autotest |
mbligh | e8b93af | 2009-01-30 00:45:53 +0000 | [diff] [blame] | 4 | from autotest_lib.server.hosts import remote |
mbligh | efccc1b | 2010-01-11 19:08:42 +0000 | [diff] [blame] | 5 | from autotest_lib.client.common_lib.global_config import global_config |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 6 | |
| 7 | |
mbligh | b86bfa1 | 2010-02-12 20:22:21 +0000 | [diff] [blame] | 8 | get_value = global_config.get_config_value |
| 9 | enable_master_ssh = get_value('AUTOSERV', 'enable_master_ssh', type=bool, |
| 10 | default=False) |
mbligh | efccc1b | 2010-01-11 19:08:42 +0000 | [diff] [blame] | 11 | |
| 12 | |
lmr | af676f3 | 2010-02-04 03:36:26 +0000 | [diff] [blame] | 13 | def make_ssh_command(user="root", port=22, opts='', hosts_file='/dev/null', |
| 14 | connect_timeout=30, alive_interval=300): |
| 15 | base_command = ("/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no " |
| 16 | "-o UserKnownHostsFile=%s -o BatchMode=yes " |
mbligh | efccc1b | 2010-01-11 19:08:42 +0000 | [diff] [blame] | 17 | "-o ConnectTimeout=%d -o ServerAliveInterval=%d " |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 18 | "-l %s -p %d") |
| 19 | assert isinstance(connect_timeout, (int, long)) |
| 20 | assert connect_timeout > 0 # can't disable the timeout |
lmr | af676f3 | 2010-02-04 03:36:26 +0000 | [diff] [blame] | 21 | return base_command % (opts, hosts_file, connect_timeout, |
| 22 | alive_interval, user, port) |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 23 | |
| 24 | |
mbligh | e8b93af | 2009-01-30 00:45:53 +0000 | [diff] [blame] | 25 | # import site specific Host class |
| 26 | SiteHost = utils.import_site_class( |
| 27 | __file__, "autotest_lib.server.hosts.site_host", "SiteHost", |
| 28 | remote.RemoteHost) |
| 29 | |
| 30 | |
| 31 | class AbstractSSHHost(SiteHost): |
mbligh | bc9402b | 2009-12-29 01:15:34 +0000 | [diff] [blame] | 32 | """ |
| 33 | This class represents a generic implementation of most of the |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 34 | framework necessary for controlling a host via ssh. It implements |
| 35 | almost all of the abstract Host methods, except for the core |
mbligh | bc9402b | 2009-12-29 01:15:34 +0000 | [diff] [blame] | 36 | Host.run method. |
| 37 | """ |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 38 | |
jadmanski | f656291 | 2008-10-21 17:59:01 +0000 | [diff] [blame] | 39 | def _initialize(self, hostname, user="root", port=22, password="", |
| 40 | *args, **dargs): |
| 41 | super(AbstractSSHHost, self)._initialize(hostname=hostname, |
| 42 | *args, **dargs) |
mbligh | 6369cf2 | 2008-10-24 17:21:57 +0000 | [diff] [blame] | 43 | self.ip = socket.getaddrinfo(self.hostname, None)[0][4][0] |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 44 | self.user = user |
| 45 | self.port = port |
| 46 | self.password = password |
showard | 6eafb49 | 2010-01-15 20:29:06 +0000 | [diff] [blame] | 47 | self._use_rsync = None |
lmr | af676f3 | 2010-02-04 03:36:26 +0000 | [diff] [blame] | 48 | self.known_hosts_file = os.tmpfile() |
| 49 | known_hosts_fd = self.known_hosts_file.fileno() |
| 50 | self.known_hosts_fd = '/dev/fd/%s' % known_hosts_fd |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 51 | |
mbligh | efccc1b | 2010-01-11 19:08:42 +0000 | [diff] [blame] | 52 | """ |
| 53 | Master SSH connection background job, socket temp directory and socket |
| 54 | control path option. If master-SSH is enabled, these fields will be |
| 55 | initialized by start_master_ssh when a new SSH connection is initiated. |
| 56 | """ |
| 57 | self.master_ssh_job = None |
| 58 | self.master_ssh_tempdir = None |
| 59 | self.master_ssh_option = '' |
| 60 | |
showard | 6eafb49 | 2010-01-15 20:29:06 +0000 | [diff] [blame] | 61 | |
| 62 | def use_rsync(self): |
| 63 | if self._use_rsync is not None: |
| 64 | return self._use_rsync |
| 65 | |
mbligh | c9892c0 | 2010-01-06 19:02:16 +0000 | [diff] [blame] | 66 | # Check if rsync is available on the remote host. If it's not, |
| 67 | # don't try to use it for any future file transfers. |
showard | 6eafb49 | 2010-01-15 20:29:06 +0000 | [diff] [blame] | 68 | self._use_rsync = self._check_rsync() |
| 69 | if not self._use_rsync: |
mbligh | c9892c0 | 2010-01-06 19:02:16 +0000 | [diff] [blame] | 70 | logging.warn("rsync not available on remote host %s -- disabled", |
| 71 | self.hostname) |
Eric Li | e0493a4 | 2010-11-15 13:05:43 -0800 | [diff] [blame^] | 72 | return self._use_rsync |
mbligh | c9892c0 | 2010-01-06 19:02:16 +0000 | [diff] [blame] | 73 | |
| 74 | |
| 75 | def _check_rsync(self): |
| 76 | """ |
| 77 | Check if rsync is available on the remote host. |
| 78 | """ |
| 79 | try: |
| 80 | self.run("rsync --version", stdout_tee=None, stderr_tee=None) |
| 81 | except error.AutoservRunError: |
| 82 | return False |
| 83 | return True |
| 84 | |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 85 | |
showard | 56176ec | 2009-10-28 19:52:30 +0000 | [diff] [blame] | 86 | def _encode_remote_paths(self, paths, escape=True): |
mbligh | bc9402b | 2009-12-29 01:15:34 +0000 | [diff] [blame] | 87 | """ |
| 88 | Given a list of file paths, encodes it as a single remote path, in |
| 89 | the style used by rsync and scp. |
| 90 | """ |
showard | 56176ec | 2009-10-28 19:52:30 +0000 | [diff] [blame] | 91 | if escape: |
| 92 | paths = [utils.scp_remote_escape(path) for path in paths] |
| 93 | return '%s@%s:"%s"' % (self.user, self.hostname, " ".join(paths)) |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 94 | |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 95 | |
mbligh | 4556178 | 2009-05-11 21:14:34 +0000 | [diff] [blame] | 96 | def _make_rsync_cmd(self, sources, dest, delete_dest, preserve_symlinks): |
mbligh | bc9402b | 2009-12-29 01:15:34 +0000 | [diff] [blame] | 97 | """ |
| 98 | Given a list of source paths and a destination path, produces the |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 99 | appropriate rsync command for copying them. Remote paths must be |
mbligh | bc9402b | 2009-12-29 01:15:34 +0000 | [diff] [blame] | 100 | pre-encoded. |
| 101 | """ |
lmr | af676f3 | 2010-02-04 03:36:26 +0000 | [diff] [blame] | 102 | ssh_cmd = make_ssh_command(user=self.user, port=self.port, |
| 103 | opts=self.master_ssh_option, |
| 104 | hosts_file=self.known_hosts_fd) |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 105 | if delete_dest: |
| 106 | delete_flag = "--delete" |
| 107 | else: |
| 108 | delete_flag = "" |
mbligh | 4556178 | 2009-05-11 21:14:34 +0000 | [diff] [blame] | 109 | if preserve_symlinks: |
| 110 | symlink_flag = "" |
| 111 | else: |
| 112 | symlink_flag = "-L" |
| 113 | command = "rsync %s %s --timeout=1800 --rsh='%s' -az %s %s" |
| 114 | return command % (symlink_flag, delete_flag, ssh_cmd, |
| 115 | " ".join(sources), dest) |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 116 | |
| 117 | |
| 118 | def _make_scp_cmd(self, sources, dest): |
mbligh | bc9402b | 2009-12-29 01:15:34 +0000 | [diff] [blame] | 119 | """ |
| 120 | Given a list of source paths and a destination path, produces the |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 121 | appropriate scp command for encoding it. Remote paths must be |
mbligh | bc9402b | 2009-12-29 01:15:34 +0000 | [diff] [blame] | 122 | pre-encoded. |
| 123 | """ |
mbligh | c0649d6 | 2010-01-15 18:15:58 +0000 | [diff] [blame] | 124 | command = ("scp -rq %s -o StrictHostKeyChecking=no " |
lmr | af676f3 | 2010-02-04 03:36:26 +0000 | [diff] [blame] | 125 | "-o UserKnownHostsFile=%s -P %d %s '%s'") |
| 126 | return command % (self.master_ssh_option, self.known_hosts_fd, |
mbligh | efccc1b | 2010-01-11 19:08:42 +0000 | [diff] [blame] | 127 | self.port, " ".join(sources), dest) |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 128 | |
| 129 | |
| 130 | def _make_rsync_compatible_globs(self, path, is_local): |
mbligh | bc9402b | 2009-12-29 01:15:34 +0000 | [diff] [blame] | 131 | """ |
| 132 | Given an rsync-style path, returns a list of globbed paths |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 133 | that will hopefully provide equivalent behaviour for scp. Does not |
| 134 | support the full range of rsync pattern matching behaviour, only that |
| 135 | exposed in the get/send_file interface (trailing slashes). |
| 136 | |
| 137 | The is_local param is flag indicating if the paths should be |
mbligh | bc9402b | 2009-12-29 01:15:34 +0000 | [diff] [blame] | 138 | interpreted as local or remote paths. |
| 139 | """ |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 140 | |
| 141 | # non-trailing slash paths should just work |
| 142 | if len(path) == 0 or path[-1] != "/": |
| 143 | return [path] |
| 144 | |
| 145 | # make a function to test if a pattern matches any files |
| 146 | if is_local: |
showard | 56176ec | 2009-10-28 19:52:30 +0000 | [diff] [blame] | 147 | def glob_matches_files(path, pattern): |
| 148 | return len(glob.glob(path + pattern)) > 0 |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 149 | else: |
showard | 56176ec | 2009-10-28 19:52:30 +0000 | [diff] [blame] | 150 | def glob_matches_files(path, pattern): |
| 151 | result = self.run("ls \"%s\"%s" % (utils.sh_escape(path), |
| 152 | pattern), |
| 153 | stdout_tee=None, ignore_status=True) |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 154 | return result.exit_status == 0 |
| 155 | |
| 156 | # take a set of globs that cover all files, and see which are needed |
| 157 | patterns = ["*", ".[!.]*"] |
showard | 56176ec | 2009-10-28 19:52:30 +0000 | [diff] [blame] | 158 | patterns = [p for p in patterns if glob_matches_files(path, p)] |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 159 | |
| 160 | # convert them into a set of paths suitable for the commandline |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 161 | if is_local: |
showard | 56176ec | 2009-10-28 19:52:30 +0000 | [diff] [blame] | 162 | return ["\"%s\"%s" % (utils.sh_escape(path), pattern) |
| 163 | for pattern in patterns] |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 164 | else: |
showard | 56176ec | 2009-10-28 19:52:30 +0000 | [diff] [blame] | 165 | return [utils.scp_remote_escape(path) + pattern |
| 166 | for pattern in patterns] |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 167 | |
| 168 | |
| 169 | def _make_rsync_compatible_source(self, source, is_local): |
mbligh | bc9402b | 2009-12-29 01:15:34 +0000 | [diff] [blame] | 170 | """ |
| 171 | Applies the same logic as _make_rsync_compatible_globs, but |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 172 | applies it to an entire list of sources, producing a new list of |
mbligh | bc9402b | 2009-12-29 01:15:34 +0000 | [diff] [blame] | 173 | sources, properly quoted. |
| 174 | """ |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 175 | return sum((self._make_rsync_compatible_globs(path, is_local) |
| 176 | for path in source), []) |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 177 | |
| 178 | |
mbligh | feac010 | 2009-04-28 18:31:12 +0000 | [diff] [blame] | 179 | def _set_umask_perms(self, dest): |
mbligh | bc9402b | 2009-12-29 01:15:34 +0000 | [diff] [blame] | 180 | """ |
| 181 | Given a destination file/dir (recursively) set the permissions on |
| 182 | all the files and directories to the max allowed by running umask. |
| 183 | """ |
mbligh | feac010 | 2009-04-28 18:31:12 +0000 | [diff] [blame] | 184 | |
| 185 | # now this looks strange but I haven't found a way in Python to _just_ |
| 186 | # get the umask, apparently the only option is to try to set it |
| 187 | umask = os.umask(0) |
| 188 | os.umask(umask) |
| 189 | |
| 190 | max_privs = 0777 & ~umask |
| 191 | |
| 192 | def set_file_privs(filename): |
| 193 | file_stat = os.stat(filename) |
| 194 | |
| 195 | file_privs = max_privs |
| 196 | # if the original file permissions do not have at least one |
| 197 | # executable bit then do not set it anywhere |
| 198 | if not file_stat.st_mode & 0111: |
| 199 | file_privs &= ~0111 |
| 200 | |
| 201 | os.chmod(filename, file_privs) |
| 202 | |
| 203 | # try a bottom-up walk so changes on directory permissions won't cut |
| 204 | # our access to the files/directories inside it |
| 205 | for root, dirs, files in os.walk(dest, topdown=False): |
| 206 | # when setting the privileges we emulate the chmod "X" behaviour |
| 207 | # that sets to execute only if it is a directory or any of the |
| 208 | # owner/group/other already has execute right |
| 209 | for dirname in dirs: |
| 210 | os.chmod(os.path.join(root, dirname), max_privs) |
| 211 | |
| 212 | for filename in files: |
| 213 | set_file_privs(os.path.join(root, filename)) |
| 214 | |
| 215 | |
| 216 | # now set privs for the dest itself |
| 217 | if os.path.isdir(dest): |
| 218 | os.chmod(dest, max_privs) |
| 219 | else: |
| 220 | set_file_privs(dest) |
| 221 | |
| 222 | |
mbligh | 4556178 | 2009-05-11 21:14:34 +0000 | [diff] [blame] | 223 | def get_file(self, source, dest, delete_dest=False, preserve_perm=True, |
| 224 | preserve_symlinks=False): |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 225 | """ |
| 226 | Copy files from the remote host to a local path. |
| 227 | |
| 228 | Directories will be copied recursively. |
| 229 | If a source component is a directory with a trailing slash, |
| 230 | the content of the directory will be copied, otherwise, the |
| 231 | directory itself and its content will be copied. This |
| 232 | behavior is similar to that of the program 'rsync'. |
| 233 | |
| 234 | Args: |
| 235 | source: either |
| 236 | 1) a single file or directory, as a string |
| 237 | 2) a list of one or more (possibly mixed) |
| 238 | files or directories |
| 239 | dest: a file or a directory (if source contains a |
| 240 | directory or more than one element, you must |
| 241 | supply a directory dest) |
mbligh | 89e258d | 2008-10-24 13:58:08 +0000 | [diff] [blame] | 242 | delete_dest: if this is true, the command will also clear |
| 243 | out any old files at dest that are not in the |
| 244 | source |
mbligh | feac010 | 2009-04-28 18:31:12 +0000 | [diff] [blame] | 245 | preserve_perm: tells get_file() to try to preserve the sources |
| 246 | permissions on files and dirs |
mbligh | 4556178 | 2009-05-11 21:14:34 +0000 | [diff] [blame] | 247 | preserve_symlinks: try to preserve symlinks instead of |
| 248 | transforming them into files/dirs on copy |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 249 | |
| 250 | Raises: |
| 251 | AutoservRunError: the scp command failed |
| 252 | """ |
mbligh | efccc1b | 2010-01-11 19:08:42 +0000 | [diff] [blame] | 253 | |
| 254 | # Start a master SSH connection if necessary. |
| 255 | self.start_master_ssh() |
| 256 | |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 257 | if isinstance(source, basestring): |
| 258 | source = [source] |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 259 | dest = os.path.abspath(dest) |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 260 | |
mbligh | c9892c0 | 2010-01-06 19:02:16 +0000 | [diff] [blame] | 261 | # If rsync is disabled or fails, try scp. |
showard | 6eafb49 | 2010-01-15 20:29:06 +0000 | [diff] [blame] | 262 | try_scp = True |
| 263 | if self.use_rsync(): |
mbligh | c9892c0 | 2010-01-06 19:02:16 +0000 | [diff] [blame] | 264 | try: |
| 265 | remote_source = self._encode_remote_paths(source) |
| 266 | local_dest = utils.sh_escape(dest) |
| 267 | rsync = self._make_rsync_cmd([remote_source], local_dest, |
| 268 | delete_dest, preserve_symlinks) |
| 269 | utils.run(rsync) |
showard | 6eafb49 | 2010-01-15 20:29:06 +0000 | [diff] [blame] | 270 | try_scp = False |
mbligh | c9892c0 | 2010-01-06 19:02:16 +0000 | [diff] [blame] | 271 | except error.CmdError, e: |
| 272 | logging.warn("trying scp, rsync failed: %s" % e) |
mbligh | c9892c0 | 2010-01-06 19:02:16 +0000 | [diff] [blame] | 273 | |
| 274 | if try_scp: |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 275 | # scp has no equivalent to --delete, just drop the entire dest dir |
| 276 | if delete_dest and os.path.isdir(dest): |
| 277 | shutil.rmtree(dest) |
| 278 | os.mkdir(dest) |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 279 | |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 280 | remote_source = self._make_rsync_compatible_source(source, False) |
| 281 | if remote_source: |
showard | 56176ec | 2009-10-28 19:52:30 +0000 | [diff] [blame] | 282 | # _make_rsync_compatible_source() already did the escaping |
| 283 | remote_source = self._encode_remote_paths(remote_source, |
| 284 | escape=False) |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 285 | local_dest = utils.sh_escape(dest) |
jadmanski | 2583a43 | 2009-02-10 23:59:11 +0000 | [diff] [blame] | 286 | scp = self._make_scp_cmd([remote_source], local_dest) |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 287 | try: |
| 288 | utils.run(scp) |
| 289 | except error.CmdError, e: |
| 290 | raise error.AutoservRunError(e.args[0], e.args[1]) |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 291 | |
mbligh | feac010 | 2009-04-28 18:31:12 +0000 | [diff] [blame] | 292 | if not preserve_perm: |
| 293 | # we have no way to tell scp to not try to preserve the |
| 294 | # permissions so set them after copy instead. |
| 295 | # for rsync we could use "--no-p --chmod=ugo=rwX" but those |
| 296 | # options are only in very recent rsync versions |
| 297 | self._set_umask_perms(dest) |
| 298 | |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 299 | |
mbligh | 4556178 | 2009-05-11 21:14:34 +0000 | [diff] [blame] | 300 | def send_file(self, source, dest, delete_dest=False, |
| 301 | preserve_symlinks=False): |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 302 | """ |
| 303 | Copy files from a local path to the remote host. |
| 304 | |
| 305 | Directories will be copied recursively. |
| 306 | If a source component is a directory with a trailing slash, |
| 307 | the content of the directory will be copied, otherwise, the |
| 308 | directory itself and its content will be copied. This |
| 309 | behavior is similar to that of the program 'rsync'. |
| 310 | |
| 311 | Args: |
| 312 | source: either |
| 313 | 1) a single file or directory, as a string |
| 314 | 2) a list of one or more (possibly mixed) |
| 315 | files or directories |
| 316 | dest: a file or a directory (if source contains a |
| 317 | directory or more than one element, you must |
| 318 | supply a directory dest) |
mbligh | 89e258d | 2008-10-24 13:58:08 +0000 | [diff] [blame] | 319 | delete_dest: if this is true, the command will also clear |
| 320 | out any old files at dest that are not in the |
| 321 | source |
mbligh | 4556178 | 2009-05-11 21:14:34 +0000 | [diff] [blame] | 322 | preserve_symlinks: controls if symlinks on the source will be |
| 323 | copied as such on the destination or transformed into the |
| 324 | referenced file/directory |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 325 | |
| 326 | Raises: |
| 327 | AutoservRunError: the scp command failed |
| 328 | """ |
mbligh | efccc1b | 2010-01-11 19:08:42 +0000 | [diff] [blame] | 329 | |
| 330 | # Start a master SSH connection if necessary. |
| 331 | self.start_master_ssh() |
| 332 | |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 333 | if isinstance(source, basestring): |
| 334 | source = [source] |
jadmanski | 2583a43 | 2009-02-10 23:59:11 +0000 | [diff] [blame] | 335 | remote_dest = self._encode_remote_paths([dest]) |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 336 | |
mbligh | c9892c0 | 2010-01-06 19:02:16 +0000 | [diff] [blame] | 337 | # If rsync is disabled or fails, try scp. |
showard | 6eafb49 | 2010-01-15 20:29:06 +0000 | [diff] [blame] | 338 | try_scp = True |
| 339 | if self.use_rsync(): |
mbligh | c9892c0 | 2010-01-06 19:02:16 +0000 | [diff] [blame] | 340 | try: |
| 341 | local_sources = [utils.sh_escape(path) for path in source] |
| 342 | rsync = self._make_rsync_cmd(local_sources, remote_dest, |
| 343 | delete_dest, preserve_symlinks) |
| 344 | utils.run(rsync) |
showard | 6eafb49 | 2010-01-15 20:29:06 +0000 | [diff] [blame] | 345 | try_scp = False |
mbligh | c9892c0 | 2010-01-06 19:02:16 +0000 | [diff] [blame] | 346 | except error.CmdError, e: |
| 347 | logging.warn("trying scp, rsync failed: %s" % e) |
mbligh | c9892c0 | 2010-01-06 19:02:16 +0000 | [diff] [blame] | 348 | |
| 349 | if try_scp: |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 350 | # scp has no equivalent to --delete, just drop the entire dest dir |
| 351 | if delete_dest: |
showard | 2716015 | 2009-07-15 14:28:42 +0000 | [diff] [blame] | 352 | is_dir = self.run("ls -d %s/" % dest, |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 353 | ignore_status=True).exit_status == 0 |
| 354 | if is_dir: |
| 355 | cmd = "rm -rf %s && mkdir %s" |
mbligh | 5a0ca53 | 2009-08-03 16:44:34 +0000 | [diff] [blame] | 356 | cmd %= (dest, dest) |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 357 | self.run(cmd) |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 358 | |
jadmanski | 2583a43 | 2009-02-10 23:59:11 +0000 | [diff] [blame] | 359 | local_sources = self._make_rsync_compatible_source(source, True) |
| 360 | if local_sources: |
| 361 | scp = self._make_scp_cmd(local_sources, remote_dest) |
jadmanski | d7b79ed | 2009-01-07 17:19:48 +0000 | [diff] [blame] | 362 | try: |
| 363 | utils.run(scp) |
| 364 | except error.CmdError, e: |
| 365 | raise error.AutoservRunError(e.args[0], e.args[1]) |
| 366 | |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 367 | |
| 368 | def ssh_ping(self, timeout=60): |
| 369 | try: |
| 370 | self.run("true", timeout=timeout, connect_timeout=timeout) |
| 371 | except error.AutoservSSHTimeout: |
mbligh | d0e9498 | 2009-07-11 00:15:18 +0000 | [diff] [blame] | 372 | msg = "Host (ssh) verify timed out (timeout = %d)" % timeout |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 373 | raise error.AutoservSSHTimeout(msg) |
mbligh | 9d738d6 | 2009-03-09 21:17:10 +0000 | [diff] [blame] | 374 | except error.AutoservSshPermissionDeniedError: |
| 375 | #let AutoservSshPermissionDeniedError be visible to the callers |
| 376 | raise |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 377 | except error.AutoservRunError, e: |
mbligh | c971c5f | 2009-06-08 16:48:54 +0000 | [diff] [blame] | 378 | # convert the generic AutoservRunError into something more |
| 379 | # specific for this context |
| 380 | raise error.AutoservSshPingHostError(e.description + '\n' + |
| 381 | repr(e.result_obj)) |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 382 | |
| 383 | |
| 384 | def is_up(self): |
| 385 | """ |
| 386 | Check if the remote host is up. |
| 387 | |
jadmanski | c035491 | 2010-01-12 15:57:29 +0000 | [diff] [blame] | 388 | @returns True if the remote host is up, False otherwise |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 389 | """ |
| 390 | try: |
| 391 | self.ssh_ping() |
| 392 | except error.AutoservError: |
| 393 | return False |
| 394 | else: |
| 395 | return True |
| 396 | |
| 397 | |
| 398 | def wait_up(self, timeout=None): |
| 399 | """ |
| 400 | Wait until the remote host is up or the timeout expires. |
| 401 | |
| 402 | In fact, it will wait until an ssh connection to the remote |
| 403 | host can be established, and getty is running. |
| 404 | |
jadmanski | c035491 | 2010-01-12 15:57:29 +0000 | [diff] [blame] | 405 | @param timeout time limit in seconds before returning even |
| 406 | if the host is not up. |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 407 | |
jadmanski | c035491 | 2010-01-12 15:57:29 +0000 | [diff] [blame] | 408 | @returns True if the host was found to be up, False otherwise |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 409 | """ |
| 410 | if timeout: |
| 411 | end_time = time.time() + timeout |
| 412 | |
| 413 | while not timeout or time.time() < end_time: |
| 414 | if self.is_up(): |
| 415 | try: |
| 416 | if self.are_wait_up_processes_up(): |
jadmanski | 7ebac3d | 2010-06-17 16:06:31 +0000 | [diff] [blame] | 417 | logging.debug('Host %s is now up', self.hostname) |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 418 | return True |
| 419 | except error.AutoservError: |
| 420 | pass |
| 421 | time.sleep(1) |
| 422 | |
jadmanski | 7ebac3d | 2010-06-17 16:06:31 +0000 | [diff] [blame] | 423 | logging.debug('Host %s is still down after waiting %d seconds', |
| 424 | self.hostname, int(timeout + time.time() - end_time)) |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 425 | return False |
| 426 | |
| 427 | |
jadmanski | c035491 | 2010-01-12 15:57:29 +0000 | [diff] [blame] | 428 | def wait_down(self, timeout=None, warning_timer=None, old_boot_id=None): |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 429 | """ |
| 430 | Wait until the remote host is down or the timeout expires. |
| 431 | |
jadmanski | c035491 | 2010-01-12 15:57:29 +0000 | [diff] [blame] | 432 | If old_boot_id is provided, this will wait until either the machine |
| 433 | is unpingable or self.get_boot_id() returns a value different from |
| 434 | old_boot_id. If the boot_id value has changed then the function |
| 435 | returns true under the assumption that the machine has shut down |
| 436 | and has now already come back up. |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 437 | |
jadmanski | c035491 | 2010-01-12 15:57:29 +0000 | [diff] [blame] | 438 | If old_boot_id is None then until the machine becomes unreachable the |
| 439 | method assumes the machine has not yet shut down. |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 440 | |
jadmanski | c035491 | 2010-01-12 15:57:29 +0000 | [diff] [blame] | 441 | @param timeout Time limit in seconds before returning even |
| 442 | if the host is still up. |
| 443 | @param warning_timer Time limit in seconds that will generate |
| 444 | a warning if the host is not down yet. |
| 445 | @param old_boot_id A string containing the result of self.get_boot_id() |
| 446 | prior to the host being told to shut down. Can be None if this is |
| 447 | not available. |
| 448 | |
| 449 | @returns True if the host was found to be down, False otherwise |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 450 | """ |
mbligh | e5e3cf2 | 2010-05-27 23:33:14 +0000 | [diff] [blame] | 451 | #TODO: there is currently no way to distinguish between knowing |
| 452 | #TODO: boot_id was unsupported and not knowing the boot_id. |
mbligh | 2ed998f | 2009-04-08 21:03:47 +0000 | [diff] [blame] | 453 | current_time = time.time() |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 454 | if timeout: |
mbligh | 2ed998f | 2009-04-08 21:03:47 +0000 | [diff] [blame] | 455 | end_time = current_time + timeout |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 456 | |
mbligh | 2ed998f | 2009-04-08 21:03:47 +0000 | [diff] [blame] | 457 | if warning_timer: |
| 458 | warn_time = current_time + warning_timer |
| 459 | |
jadmanski | c035491 | 2010-01-12 15:57:29 +0000 | [diff] [blame] | 460 | if old_boot_id is not None: |
| 461 | logging.debug('Host %s pre-shutdown boot_id is %s', |
| 462 | self.hostname, old_boot_id) |
| 463 | |
mbligh | 2ed998f | 2009-04-08 21:03:47 +0000 | [diff] [blame] | 464 | while not timeout or current_time < end_time: |
jadmanski | c035491 | 2010-01-12 15:57:29 +0000 | [diff] [blame] | 465 | try: |
| 466 | new_boot_id = self.get_boot_id() |
mbligh | dbc7e4a | 2010-01-15 20:34:20 +0000 | [diff] [blame] | 467 | except error.AutoservError: |
jadmanski | c035491 | 2010-01-12 15:57:29 +0000 | [diff] [blame] | 468 | logging.debug('Host %s is now unreachable over ssh, is down', |
| 469 | self.hostname) |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 470 | return True |
jadmanski | c035491 | 2010-01-12 15:57:29 +0000 | [diff] [blame] | 471 | else: |
| 472 | # if the machine is up but the boot_id value has changed from |
| 473 | # old boot id, then we can assume the machine has gone down |
| 474 | # and then already come back up |
| 475 | if old_boot_id is not None and old_boot_id != new_boot_id: |
| 476 | logging.debug('Host %s now has boot_id %s and so must ' |
| 477 | 'have rebooted', self.hostname, new_boot_id) |
| 478 | return True |
mbligh | 2ed998f | 2009-04-08 21:03:47 +0000 | [diff] [blame] | 479 | |
| 480 | if warning_timer and current_time > warn_time: |
| 481 | self.record("WARN", None, "shutdown", |
| 482 | "Shutdown took longer than %ds" % warning_timer) |
| 483 | # Print the warning only once. |
| 484 | warning_timer = None |
mbligh | a446440 | 2009-04-17 20:13:41 +0000 | [diff] [blame] | 485 | # If a machine is stuck switching runlevels |
| 486 | # This may cause the machine to reboot. |
| 487 | self.run('kill -HUP 1', ignore_status=True) |
mbligh | 2ed998f | 2009-04-08 21:03:47 +0000 | [diff] [blame] | 488 | |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 489 | time.sleep(1) |
mbligh | 2ed998f | 2009-04-08 21:03:47 +0000 | [diff] [blame] | 490 | current_time = time.time() |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 491 | |
| 492 | return False |
jadmanski | f656291 | 2008-10-21 17:59:01 +0000 | [diff] [blame] | 493 | |
mbligh | a0a2759 | 2009-01-24 01:41:36 +0000 | [diff] [blame] | 494 | |
jadmanski | f656291 | 2008-10-21 17:59:01 +0000 | [diff] [blame] | 495 | # tunable constants for the verify & repair code |
mbligh | b86bfa1 | 2010-02-12 20:22:21 +0000 | [diff] [blame] | 496 | AUTOTEST_GB_DISKSPACE_REQUIRED = get_value("SERVER", |
| 497 | "gb_diskspace_required", |
| 498 | type=int, |
| 499 | default=20) |
mbligh | a0a2759 | 2009-01-24 01:41:36 +0000 | [diff] [blame] | 500 | |
jadmanski | f656291 | 2008-10-21 17:59:01 +0000 | [diff] [blame] | 501 | |
showard | ca57298 | 2009-09-18 21:20:01 +0000 | [diff] [blame] | 502 | def verify_connectivity(self): |
| 503 | super(AbstractSSHHost, self).verify_connectivity() |
jadmanski | f656291 | 2008-10-21 17:59:01 +0000 | [diff] [blame] | 504 | |
showard | b18134f | 2009-03-20 20:52:18 +0000 | [diff] [blame] | 505 | logging.info('Pinging host ' + self.hostname) |
jadmanski | f656291 | 2008-10-21 17:59:01 +0000 | [diff] [blame] | 506 | self.ssh_ping() |
mbligh | 2ba7ab0 | 2009-08-24 22:09:26 +0000 | [diff] [blame] | 507 | logging.info("Host (ssh) %s is alive", self.hostname) |
jadmanski | f656291 | 2008-10-21 17:59:01 +0000 | [diff] [blame] | 508 | |
jadmanski | 80deb75 | 2009-01-21 17:14:16 +0000 | [diff] [blame] | 509 | if self.is_shutting_down(): |
mbligh | c971c5f | 2009-06-08 16:48:54 +0000 | [diff] [blame] | 510 | raise error.AutoservHostIsShuttingDownError("Host is shutting down") |
jadmanski | 80deb75 | 2009-01-21 17:14:16 +0000 | [diff] [blame] | 511 | |
mbligh | b49b523 | 2009-02-12 21:54:49 +0000 | [diff] [blame] | 512 | |
showard | ca57298 | 2009-09-18 21:20:01 +0000 | [diff] [blame] | 513 | def verify_software(self): |
| 514 | super(AbstractSSHHost, self).verify_software() |
jadmanski | f656291 | 2008-10-21 17:59:01 +0000 | [diff] [blame] | 515 | try: |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 516 | self.check_diskspace(autotest.Autotest.get_install_dir(self), |
| 517 | self.AUTOTEST_GB_DISKSPACE_REQUIRED) |
jadmanski | f656291 | 2008-10-21 17:59:01 +0000 | [diff] [blame] | 518 | except error.AutoservHostError: |
| 519 | raise # only want to raise if it's a space issue |
showard | ad812bf | 2009-10-20 23:49:56 +0000 | [diff] [blame] | 520 | except autotest.AutodirNotFoundError: |
showard | ca57298 | 2009-09-18 21:20:01 +0000 | [diff] [blame] | 521 | # autotest dir may not exist, etc. ignore |
| 522 | logging.debug('autodir space check exception, this is probably ' |
| 523 | 'safe to ignore\n' + traceback.format_exc()) |
mbligh | efccc1b | 2010-01-11 19:08:42 +0000 | [diff] [blame] | 524 | |
| 525 | |
| 526 | def close(self): |
| 527 | super(AbstractSSHHost, self).close() |
| 528 | self._cleanup_master_ssh() |
lmr | af676f3 | 2010-02-04 03:36:26 +0000 | [diff] [blame] | 529 | self.known_hosts_file.close() |
mbligh | efccc1b | 2010-01-11 19:08:42 +0000 | [diff] [blame] | 530 | |
| 531 | |
| 532 | def _cleanup_master_ssh(self): |
| 533 | """ |
| 534 | Release all resources (process, temporary directory) used by an active |
| 535 | master SSH connection. |
| 536 | """ |
| 537 | # If a master SSH connection is running, kill it. |
| 538 | if self.master_ssh_job is not None: |
| 539 | utils.nuke_subprocess(self.master_ssh_job.sp) |
| 540 | self.master_ssh_job = None |
| 541 | |
| 542 | # Remove the temporary directory for the master SSH socket. |
| 543 | if self.master_ssh_tempdir is not None: |
| 544 | self.master_ssh_tempdir.clean() |
| 545 | self.master_ssh_tempdir = None |
| 546 | self.master_ssh_option = '' |
| 547 | |
| 548 | |
| 549 | def start_master_ssh(self): |
| 550 | """ |
| 551 | Called whenever a slave SSH connection needs to be initiated (e.g., by |
| 552 | run, rsync, scp). If master SSH support is enabled and a master SSH |
| 553 | connection is not active already, start a new one in the background. |
| 554 | Also, cleanup any zombie master SSH connections (e.g., dead due to |
| 555 | reboot). |
| 556 | """ |
| 557 | if not enable_master_ssh: |
| 558 | return |
| 559 | |
| 560 | # If a previously started master SSH connection is not running |
| 561 | # anymore, it needs to be cleaned up and then restarted. |
| 562 | if self.master_ssh_job is not None: |
| 563 | if self.master_ssh_job.sp.poll() is not None: |
| 564 | logging.info("Master ssh connection to %s is down.", |
| 565 | self.hostname) |
| 566 | self._cleanup_master_ssh() |
| 567 | |
| 568 | # Start a new master SSH connection. |
| 569 | if self.master_ssh_job is None: |
| 570 | # Create a shared socket in a temp location. |
| 571 | self.master_ssh_tempdir = autotemp.tempdir(unique_id='ssh-master') |
| 572 | self.master_ssh_option = ("-o ControlPath=%s/socket" % |
| 573 | self.master_ssh_tempdir.name) |
| 574 | |
| 575 | # Start the master SSH connection in the background. |
mbligh | 5644c12 | 2010-01-29 17:43:26 +0000 | [diff] [blame] | 576 | master_cmd = self.ssh_command(options="-N -o ControlMaster=yes") |
mbligh | efccc1b | 2010-01-11 19:08:42 +0000 | [diff] [blame] | 577 | logging.info("Starting master ssh connection '%s'" % master_cmd) |
| 578 | self.master_ssh_job = utils.BgJob(master_cmd) |
mbligh | 0a88370 | 2010-04-21 01:58:34 +0000 | [diff] [blame] | 579 | |
| 580 | |
| 581 | def clear_known_hosts(self): |
| 582 | """Clears out the temporary ssh known_hosts file. |
| 583 | |
| 584 | This is useful if the test SSHes to the machine, then reinstalls it, |
| 585 | then SSHes to it again. It can be called after the reinstall to |
| 586 | reduce the spam in the logs. |
| 587 | """ |
| 588 | logging.info("Clearing known hosts for host '%s', file '%s'.", |
| 589 | self.hostname, self.known_hosts_fd) |
| 590 | # Clear out the file by opening it for writing and then closing. |
| 591 | fh = open(self.known_hosts_fd, "w") |
| 592 | fh.close() |