Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # -*- coding: utf-8 -*-" |
| 3 | # |
| 4 | # Copyright 2020 The Chromium OS Authors. All rights reserved. |
| 5 | # Use of this source code is governed by a BSD-style license that can be |
| 6 | # found in the LICENSE file. |
| 7 | |
| 8 | """Module containing shared helper methods.""" |
| 9 | |
| 10 | from __future__ import print_function |
Guenter Roeck | c1cf669 | 2020-05-08 17:45:47 -0700 | [diff] [blame] | 11 | import logging |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 12 | import os |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 13 | import re |
Hirthanan Subenderan | a43fd4d | 2020-03-30 13:01:45 -0700 | [diff] [blame] | 14 | import time |
Hirthanan Subenderan | cfada21 | 2020-02-28 15:12:39 -0800 | [diff] [blame] | 15 | from enum import Enum |
Hirthanan Subenderan | cd9cbcb | 2020-03-25 19:43:12 -0700 | [diff] [blame] | 16 | import subprocess |
Hirthanan Subenderan | cfada21 | 2020-02-28 15:12:39 -0800 | [diff] [blame] | 17 | import MySQLdb |
| 18 | |
| 19 | import initdb_upstream |
| 20 | import initdb_stable |
| 21 | import initdb_chromeos |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 22 | |
Hirthanan Subenderan | 00f1804 | 2020-02-11 17:24:38 -0800 | [diff] [blame] | 23 | |
| 24 | KERNEL_SITE = 'https://git.kernel.org/' |
| 25 | UPSTREAM_REPO = KERNEL_SITE + 'pub/scm/linux/kernel/git/torvalds/linux' |
| 26 | STABLE_REPO = KERNEL_SITE + 'pub/scm/linux/kernel/git/stable/linux-stable' |
Guenter Roeck | 760f581 | 2020-04-18 08:49:40 -0700 | [diff] [blame] | 27 | STABLE_RC_REPO = KERNEL_SITE + 'pub/scm/linux/kernel/git/stable/linux-stable-rc' |
Hirthanan Subenderan | 00f1804 | 2020-02-11 17:24:38 -0800 | [diff] [blame] | 28 | |
| 29 | CHROMIUM_SITE = 'https://chromium.googlesource.com/' |
Hirthanan Subenderan | c44a0b3 | 2020-03-11 22:34:39 -0700 | [diff] [blame] | 30 | CHROMEOS_KERNEL_DIR = 'chromiumos/third_party/kernel' |
| 31 | CHROMEOS_REPO = os.path.join(CHROMIUM_SITE, CHROMEOS_KERNEL_DIR) |
Hirthanan Subenderan | 4036800 | 2020-03-10 15:36:48 -0700 | [diff] [blame] | 32 | CHROMIUM_REVIEW_BASEURL = 'https://chromium-review.googlesource.com/a' |
Hirthanan Subenderan | 00f1804 | 2020-02-11 17:24:38 -0800 | [diff] [blame] | 33 | |
Hirthanan Subenderan | 1a97fab | 2020-04-16 08:55:03 -0700 | [diff] [blame] | 34 | # Order BRANCHES from oldest to newest |
| 35 | CHROMEOS_BRANCHES = ['4.4', '4.14', '4.19', '5.4'] |
Guenter Roeck | 77d0ec7 | 2020-04-22 06:33:52 -0700 | [diff] [blame] | 36 | STABLE_BRANCHES = ['4.4', '4.9', '4.14', '4.19', '5.4', '5.6'] |
Hirthanan Subenderan | 1a97fab | 2020-04-16 08:55:03 -0700 | [diff] [blame] | 37 | |
| 38 | UPSTREAM_START_BRANCH = 'v%s' % CHROMEOS_BRANCHES[0] |
Hirthanan Subenderan | 00f1804 | 2020-02-11 17:24:38 -0800 | [diff] [blame] | 39 | |
Hirthanan Subenderan | 7d15b28 | 2020-03-02 16:29:08 -0800 | [diff] [blame] | 40 | CHROMEOS_PATH = 'linux_chrome' |
| 41 | STABLE_PATH = 'linux_stable' |
Guenter Roeck | 760f581 | 2020-04-18 08:49:40 -0700 | [diff] [blame] | 42 | STABLE_RC_PATH = 'linux_stable_rc' |
Hirthanan Subenderan | 7d15b28 | 2020-03-02 16:29:08 -0800 | [diff] [blame] | 43 | UPSTREAM_PATH = 'linux_upstream' |
Hirthanan Subenderan | 00f1804 | 2020-02-11 17:24:38 -0800 | [diff] [blame] | 44 | |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 45 | WORKDIR = os.getcwd() |
Hirthanan Subenderan | 1a99a52 | 2020-04-11 00:51:50 -0700 | [diff] [blame] | 46 | HOMEDIR = os.path.expanduser('~') |
| 47 | GCE_GIT_COOKIE_PATH = os.path.join(HOMEDIR, '.git-credential-cache/cookie') |
| 48 | LOCAL_GIT_COOKIE_PATH = os.path.join(HOMEDIR, '.gitcookies') |
Hirthanan Subenderan | 4036800 | 2020-03-10 15:36:48 -0700 | [diff] [blame] | 49 | |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 50 | |
Hirthanan Subenderan | b81acc2 | 2020-03-03 16:34:20 -0800 | [diff] [blame] | 51 | class Status(Enum): |
| 52 | """Text representation of database enum to track status of gerrit CL.""" |
| 53 | OPEN = 1 # Gerrit ticket was created for clean fix patch |
| 54 | MERGED = 2 # Gerrit ticket was merged and closed |
| 55 | ABANDONED = 3 # Gerrit ticket was abandoned |
| 56 | CONFLICT = 4 # Gerrit ticket NOT created since patch doesn't apply properly |
| 57 | |
| 58 | |
Hirthanan Subenderan | cfada21 | 2020-02-28 15:12:39 -0800 | [diff] [blame] | 59 | class Kernel(Enum): |
| 60 | """Enum representing which Kernel we are representing.""" |
| 61 | linux_stable = 1 |
Guenter Roeck | 760f581 | 2020-04-18 08:49:40 -0700 | [diff] [blame] | 62 | linux_stable_rc = 2 |
| 63 | linux_chrome = 3 |
| 64 | linux_upstream = 4 |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 65 | |
Hirthanan Subenderan | 00f1804 | 2020-02-11 17:24:38 -0800 | [diff] [blame] | 66 | |
Hirthanan Subenderan | 53dac7d | 2020-02-28 15:34:06 -0800 | [diff] [blame] | 67 | class KernelMetadata(object): |
| 68 | """Object to group kernel Metadata.""" |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 69 | path = None |
| 70 | repo = None |
Hirthanan Subenderan | 53dac7d | 2020-02-28 15:34:06 -0800 | [diff] [blame] | 71 | kernel_fixes_table = None |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 72 | branches = None |
| 73 | tag_template = None |
Hirthanan Subenderan | 53dac7d | 2020-02-28 15:34:06 -0800 | [diff] [blame] | 74 | get_kernel_branch = None |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 75 | update_table = None |
Hirthanan Subenderan | 53dac7d | 2020-02-28 15:34:06 -0800 | [diff] [blame] | 76 | |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 77 | def __init__(self, _path, _repo, _kernel_fixes_table, _branches, _tag_template, |
| 78 | _get_kernel_branch, _update_table): |
| 79 | self.path = _path |
| 80 | self.repo = _repo |
Hirthanan Subenderan | 53dac7d | 2020-02-28 15:34:06 -0800 | [diff] [blame] | 81 | self.kernel_fixes_table = _kernel_fixes_table |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 82 | self.branches = _branches |
| 83 | self.tag_template = _tag_template |
Hirthanan Subenderan | 53dac7d | 2020-02-28 15:34:06 -0800 | [diff] [blame] | 84 | self.get_kernel_branch = _get_kernel_branch |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 85 | self.update_table = _update_table |
Hirthanan Subenderan | 53dac7d | 2020-02-28 15:34:06 -0800 | [diff] [blame] | 86 | |
| 87 | |
Hirthanan Subenderan | a43fd4d | 2020-03-30 13:01:45 -0700 | [diff] [blame] | 88 | def get_current_time(): |
| 89 | """Returns DATETIME in specific time format required by SQL.""" |
| 90 | return time.strftime('%Y-%m-%d %H:%M%:%S') |
| 91 | |
| 92 | |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 93 | def stable_branch(version): |
| 94 | """Stable branch name""" |
| 95 | return 'linux-%s.y' % version |
| 96 | |
| 97 | |
| 98 | def chromeos_branch(version): |
| 99 | """Chromeos branch name""" |
| 100 | return 'chromeos-%s' % version |
| 101 | |
Hirthanan Subenderan | a43fd4d | 2020-03-30 13:01:45 -0700 | [diff] [blame] | 102 | |
Hirthanan Subenderan | cd9cbcb | 2020-03-25 19:43:12 -0700 | [diff] [blame] | 103 | def search_upstream_sha(kernel_sha): |
| 104 | """Search for upstream sha that kernel_sha is cherry-picked from. |
| 105 | |
| 106 | If found, return upstream_sha, otherwise return None. |
| 107 | """ |
| 108 | usha = None |
| 109 | desc = subprocess.check_output(['git', 'show', '-s', kernel_sha], |
| 110 | encoding='utf-8', errors='ignore') |
| 111 | |
| 112 | # "commit" is sometimes seen multiple times, such as with commit 6093aabdd0ee |
Hirthanan Subenderan | d26485a | 2020-04-06 14:37:55 -0700 | [diff] [blame] | 113 | m = re.findall(r'cherry picked from (?:commit )+([0-9a-f]+)', desc, re.M) |
Hirthanan Subenderan | cd9cbcb | 2020-03-25 19:43:12 -0700 | [diff] [blame] | 114 | if not m: |
Hirthanan Subenderan | d26485a | 2020-04-06 14:37:55 -0700 | [diff] [blame] | 115 | m = re.findall(r'^\s*(?:commit )+([a-f0-9]+) upstream', desc, re.M) |
Hirthanan Subenderan | cd9cbcb | 2020-03-25 19:43:12 -0700 | [diff] [blame] | 116 | if not m: |
Hirthanan Subenderan | d26485a | 2020-04-06 14:37:55 -0700 | [diff] [blame] | 117 | m = re.findall(r'^\s*\[\s*Upstream (?:commit )+([0-9a-f]+)\s*\]', desc, re.M) |
Hirthanan Subenderan | cd9cbcb | 2020-03-25 19:43:12 -0700 | [diff] [blame] | 118 | if m: |
Hirthanan Subenderan | d26485a | 2020-04-06 14:37:55 -0700 | [diff] [blame] | 119 | # The patch may have been picked multiple times; only record the last entry. |
| 120 | usha = m[-1] |
| 121 | return usha[:12] |
Hirthanan Subenderan | cd9cbcb | 2020-03-25 19:43:12 -0700 | [diff] [blame] | 122 | return usha |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 123 | |
Hirthanan Subenderan | a43fd4d | 2020-03-30 13:01:45 -0700 | [diff] [blame] | 124 | |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 125 | def patch_link(changeID): |
| 126 | """Link to patch on gerrit""" |
| 127 | return 'https://chromium-review.googlesource.com/q/%s' % changeID |
| 128 | |
Hirthanan Subenderan | a43fd4d | 2020-03-30 13:01:45 -0700 | [diff] [blame] | 129 | |
Hirthanan Subenderan | cfada21 | 2020-02-28 15:12:39 -0800 | [diff] [blame] | 130 | def update_previous_fetch(db, kernel, branch, last_sha): |
| 131 | """Updates the previous_fetch table for a kernel branch.""" |
| 132 | c = db.cursor() |
| 133 | q = """UPDATE previous_fetch |
| 134 | SET sha_tip = %s |
| 135 | WHERE linux = %s AND branch = %s""" |
| 136 | c.execute(q, [last_sha, kernel.name, branch]) |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 137 | |
Hirthanan Subenderan | cfada21 | 2020-02-28 15:12:39 -0800 | [diff] [blame] | 138 | db.commit() |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 139 | |
Hirthanan Subenderan | a43fd4d | 2020-03-30 13:01:45 -0700 | [diff] [blame] | 140 | |
Hirthanan Subenderan | dc72160 | 2020-03-13 15:48:10 -0700 | [diff] [blame] | 141 | def get_kernel_absolute_path(repo_name): |
| 142 | """Returns absolute path to kernel repositories""" |
Hirthanan Subenderan | 1a99a52 | 2020-04-11 00:51:50 -0700 | [diff] [blame] | 143 | return os.path.join(HOMEDIR, 'kernel_repositories', repo_name) |
Hirthanan Subenderan | dc72160 | 2020-03-13 15:48:10 -0700 | [diff] [blame] | 144 | |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 145 | |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 146 | def update_kernel_db(db, kernel_metadata): |
Hirthanan Subenderan | cfada21 | 2020-02-28 15:12:39 -0800 | [diff] [blame] | 147 | """Update (upstream/stable/chrome) previous_fetch, fixes and commits SQL tables.""" |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 148 | path = kernel_metadata.path |
Hirthanan Subenderan | dc72160 | 2020-03-13 15:48:10 -0700 | [diff] [blame] | 149 | os.chdir(get_kernel_absolute_path(path)) |
Hirthanan Subenderan | cfada21 | 2020-02-28 15:12:39 -0800 | [diff] [blame] | 150 | |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 151 | for branch in kernel_metadata.branches: |
| 152 | start = kernel_metadata.tag_template % branch |
Hirthanan Subenderan | cfada21 | 2020-02-28 15:12:39 -0800 | [diff] [blame] | 153 | |
Guenter Roeck | c1cf669 | 2020-05-08 17:45:47 -0700 | [diff] [blame] | 154 | logging.info('Handling %s', kernel_metadata.get_kernel_branch(branch)) |
Hirthanan Subenderan | cfada21 | 2020-02-28 15:12:39 -0800 | [diff] [blame] | 155 | |
| 156 | try: |
| 157 | c = db.cursor() |
| 158 | q = """SELECT sha_tip |
| 159 | FROM previous_fetch |
| 160 | WHERE linux = %s AND branch = %s""" |
| 161 | c.execute(q, [path, branch]) |
| 162 | sha = c.fetchone() |
| 163 | if sha and sha[0]: |
| 164 | start = sha[0] |
| 165 | else: |
| 166 | q = """INSERT INTO previous_fetch (linux, branch, sha_tip) |
| 167 | VALUES (%s, %s, %s)""" |
| 168 | c.execute(q, [path, branch, start]) |
| 169 | except MySQLdb.Error as e: # pylint: disable=no-member |
Guenter Roeck | c1cf669 | 2020-05-08 17:45:47 -0700 | [diff] [blame] | 170 | logging.error('Make sure the tables have been initialized in \ |
| 171 | ./scripts/sql/initialize_sql_tables.sql') |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 172 | raise e |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 173 | |
| 174 | |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 175 | kernel_metadata.update_table(branch, start, db) |
| 176 | db.commit() |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 177 | |
Hirthanan Subenderan | cfada21 | 2020-02-28 15:12:39 -0800 | [diff] [blame] | 178 | os.chdir(WORKDIR) |
Hirthanan Subenderan | 53dac7d | 2020-02-28 15:34:06 -0800 | [diff] [blame] | 179 | |
Hirthanan Subenderan | a43fd4d | 2020-03-30 13:01:45 -0700 | [diff] [blame] | 180 | |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 181 | def get_kernel_metadata(kernel): |
| 182 | """Returns KernelMetadata for each Kernel Enum""" |
| 183 | stable_kernel_metadata = KernelMetadata(STABLE_PATH, STABLE_REPO, 'stable_fixes', |
Hirthanan Subenderan | 1a97fab | 2020-04-16 08:55:03 -0700 | [diff] [blame] | 184 | STABLE_BRANCHES, 'v%s', stable_branch, initdb_stable.update_stable_table) |
Guenter Roeck | 760f581 | 2020-04-18 08:49:40 -0700 | [diff] [blame] | 185 | stable_rc_kernel_metadata = KernelMetadata(STABLE_RC_PATH, STABLE_RC_REPO, 'stable_fixes', |
| 186 | STABLE_BRANCHES, 'v%s', stable_branch, initdb_stable.update_stable_table) |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 187 | chrome_kernel_metadata = KernelMetadata(CHROMEOS_PATH, CHROMEOS_REPO, 'chrome_fixes', |
Hirthanan Subenderan | 1a97fab | 2020-04-16 08:55:03 -0700 | [diff] [blame] | 188 | CHROMEOS_BRANCHES, 'v%s', chromeos_branch, initdb_chromeos.update_chrome_table) |
Hirthanan Subenderan | dc72160 | 2020-03-13 15:48:10 -0700 | [diff] [blame] | 189 | upstream_kernel_metadata = KernelMetadata(UPSTREAM_PATH, UPSTREAM_REPO, 'upstream_fixes', |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 190 | [UPSTREAM_START_BRANCH], '%s', lambda *args: 'master', |
| 191 | initdb_upstream.update_upstream_table) |
| 192 | |
| 193 | kernel_metadata_lookup = { |
| 194 | Kernel.linux_stable: stable_kernel_metadata, |
Guenter Roeck | 760f581 | 2020-04-18 08:49:40 -0700 | [diff] [blame] | 195 | Kernel.linux_stable_rc: stable_rc_kernel_metadata, |
Hirthanan Subenderan | c77dee3 | 2020-03-04 12:07:56 -0800 | [diff] [blame] | 196 | Kernel.linux_chrome: chrome_kernel_metadata, |
| 197 | Kernel.linux_upstream: upstream_kernel_metadata |
| 198 | } |
| 199 | |
| 200 | try: |
| 201 | return kernel_metadata_lookup[kernel] |
| 202 | except KeyError as e: |
| 203 | raise KeyError('Conditionals should match Kernel Enum types.', e) |