mbligh | 321b1f5 | 2008-04-09 16:23:43 +0000 | [diff] [blame] | 1 | """This class defines the Remote host class, mixing in the SiteHost class |
| 2 | if it is available.""" |
| 3 | |
jadmanski | 96b7807 | 2009-05-21 22:21:04 +0000 | [diff] [blame] | 4 | import os, logging |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 5 | from autotest_lib.client.common_lib import error |
jadmanski | 96b7807 | 2009-05-21 22:21:04 +0000 | [diff] [blame] | 6 | from autotest_lib.server import utils |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 7 | from autotest_lib.server.hosts import base_classes, bootloader |
mbligh | 321b1f5 | 2008-04-09 16:23:43 +0000 | [diff] [blame] | 8 | |
| 9 | |
jadmanski | 1c5e3a1 | 2008-08-15 23:08:20 +0000 | [diff] [blame] | 10 | class RemoteHost(base_classes.Host): |
jadmanski | d60321a | 2008-10-28 20:32:05 +0000 | [diff] [blame] | 11 | """ |
| 12 | This class represents a remote machine on which you can run |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 13 | programs. |
mbligh | 321b1f5 | 2008-04-09 16:23:43 +0000 | [diff] [blame] | 14 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 15 | It may be accessed through a network, a serial line, ... |
| 16 | It is not the machine autoserv is running on. |
mbligh | 321b1f5 | 2008-04-09 16:23:43 +0000 | [diff] [blame] | 17 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 18 | Implementation details: |
| 19 | This is an abstract class, leaf subclasses must implement the methods |
| 20 | listed here and in parent classes which have no implementation. They |
| 21 | may reimplement methods which already have an implementation. You |
| 22 | must not instantiate this class but should instantiate one of those |
jadmanski | d60321a | 2008-10-28 20:32:05 +0000 | [diff] [blame] | 23 | leaf subclasses. |
| 24 | """ |
mbligh | 321b1f5 | 2008-04-09 16:23:43 +0000 | [diff] [blame] | 25 | |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 26 | DEFAULT_REBOOT_TIMEOUT = base_classes.Host.DEFAULT_REBOOT_TIMEOUT |
| 27 | LAST_BOOT_TAG = object() |
| 28 | |
jadmanski | 4900b3b | 2009-07-02 22:12:08 +0000 | [diff] [blame] | 29 | VAR_LOG_MESSAGES_COPY_PATH = "/var/log/messages.autotest_start" |
| 30 | |
jadmanski | f656291 | 2008-10-21 17:59:01 +0000 | [diff] [blame] | 31 | def _initialize(self, hostname, autodir=None, *args, **dargs): |
| 32 | super(RemoteHost, self)._initialize(*args, **dargs) |
mbligh | 321b1f5 | 2008-04-09 16:23:43 +0000 | [diff] [blame] | 33 | |
jadmanski | 1c5e3a1 | 2008-08-15 23:08:20 +0000 | [diff] [blame] | 34 | self.hostname = hostname |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 35 | self.autodir = autodir |
| 36 | self.tmp_dirs = [] |
jadmanski | a2db941 | 2008-08-22 21:47:24 +0000 | [diff] [blame] | 37 | |
| 38 | |
jadmanski | edf33e0 | 2009-05-22 16:47:27 +0000 | [diff] [blame] | 39 | def __repr__(self): |
| 40 | return "<remote host: %s>" % self.hostname |
| 41 | |
| 42 | |
jadmanski | 53aaf38 | 2008-11-17 16:22:31 +0000 | [diff] [blame] | 43 | def close(self): |
| 44 | super(RemoteHost, self).close() |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 45 | self.stop_loggers() |
| 46 | |
| 47 | if hasattr(self, 'tmp_dirs'): |
| 48 | for dir in self.tmp_dirs: |
| 49 | try: |
| 50 | self.run('rm -rf "%s"' % (utils.sh_escape(dir))) |
| 51 | except error.AutoservRunError: |
| 52 | pass |
| 53 | |
| 54 | |
jadmanski | d60321a | 2008-10-28 20:32:05 +0000 | [diff] [blame] | 55 | def job_start(self): |
| 56 | """ |
| 57 | Abstract method, called the first time a remote host object |
| 58 | is created for a specific host after a job starts. |
| 59 | |
| 60 | This method depends on the create_host factory being used to |
| 61 | construct your host object. If you directly construct host objects |
| 62 | you will need to call this method yourself (and enforce the |
| 63 | single-call rule). |
| 64 | """ |
jadmanski | 4900b3b | 2009-07-02 22:12:08 +0000 | [diff] [blame] | 65 | try: |
| 66 | self.run('rm -f %s' % self.VAR_LOG_MESSAGES_COPY_PATH) |
| 67 | self.run('cp /var/log/messages %s' % |
| 68 | self.VAR_LOG_MESSAGES_COPY_PATH) |
| 69 | except Exception, e: |
| 70 | # Non-fatal error |
| 71 | logging.info('Failed to copy /var/log/messages at startup: %s', e) |
jadmanski | d60321a | 2008-10-28 20:32:05 +0000 | [diff] [blame] | 72 | |
| 73 | |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 74 | def get_autodir(self): |
| 75 | return self.autodir |
| 76 | |
| 77 | |
| 78 | def set_autodir(self, autodir): |
jadmanski | d60321a | 2008-10-28 20:32:05 +0000 | [diff] [blame] | 79 | """ |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 80 | This method is called to make the host object aware of the |
| 81 | where autotest is installed. Called in server/autotest.py |
| 82 | after a successful install |
jadmanski | d60321a | 2008-10-28 20:32:05 +0000 | [diff] [blame] | 83 | """ |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 84 | self.autodir = autodir |
| 85 | |
| 86 | |
| 87 | def sysrq_reboot(self): |
| 88 | self.run('echo b > /proc/sysrq-trigger &') |
| 89 | |
| 90 | |
| 91 | def reboot(self, timeout=DEFAULT_REBOOT_TIMEOUT, label=LAST_BOOT_TAG, |
mbligh | 959ed87 | 2009-04-17 22:18:25 +0000 | [diff] [blame] | 92 | kernel_args=None, wait=True, fastsync=False, |
| 93 | reboot_cmd=None, **dargs): |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 94 | """ |
| 95 | Reboot the remote host. |
| 96 | |
| 97 | Args: |
| 98 | timeout - How long to wait for the reboot. |
| 99 | label - The label we should boot into. If None, we will |
| 100 | boot into the default kernel. If it's LAST_BOOT_TAG, |
| 101 | we'll boot into whichever kernel was .boot'ed last |
| 102 | (or the default kernel if we haven't .boot'ed in this |
| 103 | job). If it's None, we'll boot into the default kernel. |
| 104 | If it's something else, we'll boot into that. |
| 105 | wait - Should we wait to see if the machine comes back up. |
mbligh | 2b94977 | 2009-02-26 00:59:36 +0000 | [diff] [blame] | 106 | fastsync - Don't wait for the sync to complete, just start one |
| 107 | and move on. This is for cases where rebooting prompty |
| 108 | is more important than data integrity and/or the |
| 109 | machine may have disks that cause sync to never return. |
mbligh | 959ed87 | 2009-04-17 22:18:25 +0000 | [diff] [blame] | 110 | reboot_cmd - Reboot command to execute. |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 111 | """ |
| 112 | if self.job: |
| 113 | if label == self.LAST_BOOT_TAG: |
| 114 | label = self.job.last_boot_tag |
| 115 | else: |
| 116 | self.job.last_boot_tag = label |
| 117 | |
| 118 | self.reboot_setup(label=label, kernel_args=kernel_args, **dargs) |
| 119 | |
| 120 | if label or kernel_args: |
| 121 | self.bootloader.install_boottool() |
| 122 | if not label: |
| 123 | default = int(self.bootloader.get_default()) |
| 124 | label = self.bootloader.get_titles()[default] |
| 125 | self.bootloader.boot_once(label) |
| 126 | if kernel_args: |
| 127 | self.bootloader.add_args(label, kernel_args) |
| 128 | |
| 129 | # define a function for the reboot and run it in a group |
| 130 | print "Reboot: initiating reboot" |
| 131 | def reboot(): |
| 132 | self.record("GOOD", None, "reboot.start") |
| 133 | try: |
jadmanski | d544a35 | 2009-01-14 23:36:28 +0000 | [diff] [blame] | 134 | # sync before starting the reboot, so that a long sync during |
| 135 | # shutdown isn't timed out by wait_down's short timeout |
mbligh | 2b94977 | 2009-02-26 00:59:36 +0000 | [diff] [blame] | 136 | if not fastsync: |
mbligh | 959ed87 | 2009-04-17 22:18:25 +0000 | [diff] [blame] | 137 | self.run('sync; sync', timeout=timeout, ignore_status=True) |
jadmanski | d544a35 | 2009-01-14 23:36:28 +0000 | [diff] [blame] | 138 | |
mbligh | 959ed87 | 2009-04-17 22:18:25 +0000 | [diff] [blame] | 139 | if reboot_cmd: |
| 140 | self.run(reboot_cmd) |
| 141 | else: |
| 142 | # Try several methods of rebooting in increasing harshness. |
| 143 | self.run('((' |
| 144 | ' sync &' |
| 145 | ' sleep 5; reboot &' |
| 146 | ' sleep 60; reboot -f &' |
| 147 | ' sleep 10; reboot -nf &' |
| 148 | ' sleep 10; telinit 6 &' |
| 149 | ') </dev/null >/dev/null 2>&1 &)') |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 150 | except error.AutoservRunError: |
| 151 | self.record("ABORT", None, "reboot.start", |
| 152 | "reboot command failed") |
| 153 | raise |
| 154 | if wait: |
jadmanski | d778ae4 | 2009-01-07 15:07:36 +0000 | [diff] [blame] | 155 | self.wait_for_restart(timeout, **dargs) |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 156 | |
| 157 | # if this is a full reboot-and-wait, run the reboot inside a group |
| 158 | if wait: |
| 159 | self.log_reboot(reboot) |
| 160 | else: |
| 161 | reboot() |
| 162 | |
| 163 | |
jadmanski | 4f90925 | 2008-12-01 20:47:10 +0000 | [diff] [blame] | 164 | def reboot_followup(self, *args, **dargs): |
| 165 | super(RemoteHost, self).reboot_followup(*args, **dargs) |
| 166 | if self.job: |
| 167 | self.job.profilers.handle_reboot(self) |
| 168 | |
| 169 | |
jadmanski | d778ae4 | 2009-01-07 15:07:36 +0000 | [diff] [blame] | 170 | def wait_for_restart(self, timeout=DEFAULT_REBOOT_TIMEOUT, **dargs): |
jadmanski | d60321a | 2008-10-28 20:32:05 +0000 | [diff] [blame] | 171 | """ |
| 172 | Wait for the host to come back from a reboot. This wraps the |
| 173 | generic wait_for_restart implementation in a reboot group. |
| 174 | """ |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 175 | def reboot_func(): |
jadmanski | d778ae4 | 2009-01-07 15:07:36 +0000 | [diff] [blame] | 176 | super(RemoteHost, self).wait_for_restart(timeout=timeout, **dargs) |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 177 | self.log_reboot(reboot_func) |
| 178 | |
| 179 | |
mbligh | 1264b51 | 2008-11-05 22:21:49 +0000 | [diff] [blame] | 180 | def cleanup(self): |
| 181 | super(RemoteHost, self).cleanup() |
| 182 | self.reboot() |
| 183 | |
| 184 | |
mbligh | e48bcfb | 2008-11-11 17:09:44 +0000 | [diff] [blame] | 185 | def get_tmp_dir(self, parent='/tmp'): |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 186 | """ |
| 187 | Return the pathname of a directory on the host suitable |
| 188 | for temporary file storage. |
| 189 | |
| 190 | The directory and its content will be deleted automatically |
| 191 | on the destruction of the Host object that was used to obtain |
| 192 | it. |
| 193 | """ |
jadmanski | 9f7dd11 | 2008-11-17 16:40:05 +0000 | [diff] [blame] | 194 | self.run("mkdir -p %s" % parent) |
mbligh | e48bcfb | 2008-11-11 17:09:44 +0000 | [diff] [blame] | 195 | template = os.path.join(parent, 'autoserv-XXXXXX') |
jadmanski | 9f7dd11 | 2008-11-17 16:40:05 +0000 | [diff] [blame] | 196 | dir_name = self.run("mktemp -d %s" % template).stdout.rstrip() |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 197 | self.tmp_dirs.append(dir_name) |
| 198 | return dir_name |
| 199 | |
| 200 | |
jadmanski | ea45566 | 2009-03-25 22:25:39 +0000 | [diff] [blame] | 201 | def delete_tmp_dir(self, tmpdir): |
| 202 | """ |
| 203 | Delete the given temporary directory on the remote machine. |
| 204 | """ |
| 205 | self.run('rm -rf "%s"' % utils.sh_escape(tmpdir), ignore_status=True) |
| 206 | self.tmp_dirs.remove(tmpdir) |
| 207 | |
| 208 | |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 209 | def check_uptime(self): |
| 210 | """ |
| 211 | Check that uptime is available and monotonically increasing. |
| 212 | """ |
mbligh | a43f6d2 | 2009-08-24 22:09:44 +0000 | [diff] [blame] | 213 | if not self.is_up(): |
| 214 | raise error.AutoservHostError('Client does not appear to be up') |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 215 | result = self.run("/bin/cat /proc/uptime", 30) |
| 216 | return result.stdout.strip().split()[0] |
| 217 | |
| 218 | |
jadmanski | ca7da37 | 2008-10-21 16:26:52 +0000 | [diff] [blame] | 219 | def are_wait_up_processes_up(self): |
mbligh | f2c3376 | 2008-10-18 14:42:34 +0000 | [diff] [blame] | 220 | """ |
| 221 | Checks if any HOSTS waitup processes are running yet on the |
| 222 | remote host. |
| 223 | |
| 224 | Returns True if any the waitup processes are running, False |
| 225 | otherwise. |
| 226 | """ |
| 227 | processes = self.get_wait_up_processes() |
| 228 | if len(processes) == 0: |
| 229 | return True # wait up processes aren't being used |
| 230 | for procname in processes: |
| 231 | exit_status = self.run("{ ps -e || ps; } | grep '%s'" % procname, |
| 232 | ignore_status=True).exit_status |
| 233 | if exit_status == 0: |
| 234 | return True |
| 235 | return False |