Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # -*- coding: utf-8 -*-" |
| 3 | # |
| 4 | # Copyright 2020 The Chromium OS Authors. All rights reserved. |
| 5 | # Use of this source code is governed by a BSD-style license that can be |
| 6 | # found in the LICENSE file. |
| 7 | |
| 8 | """Module rebuilding database with metadata about chromeos patches.""" |
| 9 | |
| 10 | from __future__ import print_function |
| 11 | |
| 12 | import sqlite3 |
| 13 | import os |
| 14 | import re |
| 15 | import subprocess |
| 16 | from config import CHROMEOS_PATH, CHROMEOS_BRANCHES |
| 17 | from common import WORKDIR, CHERRYPICK, STABLE, STABLE2, make_downstream_table |
| 18 | from common import stabledb, chromeosdb, chromeos_branch, createdb |
| 19 | |
| 20 | UPSTREAM = re.compile(r'(ANDROID: *|UPSTREAM: *|FROMGIT: *|BACKPORT: *)+(.*)') |
| 21 | CHROMIUM = re.compile(r'(CHROMIUM: *|FROMLIST: *)+(.*)') |
Hirthanan Subenderan | b8402a1 | 2020-02-05 14:11:00 -0800 | [diff] [blame^] | 22 | CHANGEID = re.compile(r'^( )*Change-Id: [a-zA-Z0-9]*$') |
| 23 | |
| 24 | |
| 25 | def parse_changeID(chromeos_sha): |
| 26 | """String searches for Change-Id in a chromeos git commit. |
| 27 | |
| 28 | Returns Change-Id or None if commit doesn't have associated Change-Id |
| 29 | """ |
| 30 | commit = subprocess.check_output(['git', 'show', \ |
| 31 | chromeos_sha]).decode('utf-8', errors='ignore') |
| 32 | |
| 33 | for line in commit.splitlines(): |
| 34 | if CHANGEID.match(line): |
| 35 | # removes whitespace prefixing Change-Id |
| 36 | line = line.lstrip() |
| 37 | commit_changeID = line[(line.index(' ') + 1):] |
| 38 | return commit_changeID |
| 39 | |
| 40 | return None |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 41 | |
| 42 | |
| 43 | def search_usha(sha, description): |
| 44 | """Search for upstream SHA. |
| 45 | |
| 46 | If found, return upstream sha associated with this commit sha. |
| 47 | """ |
| 48 | |
| 49 | usha = '' |
| 50 | if not CHROMIUM.match(description): |
| 51 | desc = subprocess.check_output(['git', 'show', |
| 52 | '-s', sha]).decode('utf-8', errors='ignore') |
| 53 | for d in desc.splitlines(): |
| 54 | m = CHERRYPICK.search(d) |
| 55 | if not m: |
| 56 | m = STABLE.search(d) |
| 57 | if not m: |
| 58 | m = STABLE2.search(d) |
| 59 | if m: |
| 60 | # The patch may have been picked multiple times; only record |
| 61 | # the first entry. |
| 62 | usha = m.group(2)[:12] |
| 63 | return usha |
| 64 | return usha |
| 65 | |
| 66 | |
| 67 | |
| 68 | def update_commits(start, cdb, sdb): |
| 69 | """Get list of commits from selected branch, starting with commit 'start'. |
| 70 | |
| 71 | Branch must be checked out in current directory. |
| 72 | Skip commit if it is contained in sdb, otherwise add to cdb if it isn't |
| 73 | already there. |
| 74 | """ |
| 75 | |
| 76 | try: |
| 77 | conn = sqlite3.connect(cdb) |
| 78 | conn.text_factory = str |
| 79 | c = conn.cursor() |
| 80 | |
| 81 | sconn = sqlite3.connect(sdb) |
| 82 | sconn.text_factory = str |
| 83 | sc = sconn.cursor() |
| 84 | except sqlite3.Error as e: |
| 85 | print('Could not update chromeos commits, raising error: ', e) |
| 86 | raise |
| 87 | |
| 88 | subprocess_cmd = ['git', 'log', '--no-merges', '--abbrev=12', |
| 89 | '--oneline', '--reverse', '%s..' % start] |
| 90 | commits = subprocess.check_output(subprocess_cmd).decode('utf-8') |
| 91 | |
| 92 | last = None |
| 93 | for commit in commits.splitlines(): |
| 94 | if commit: |
| 95 | elem = commit.split(' ', 1) |
| 96 | sha = elem[0] |
| 97 | description = elem[1].rstrip('\n') |
| 98 | ps = subprocess.Popen(['git', 'show', sha], |
| 99 | stdout=subprocess.PIPE, encoding='utf-8') |
| 100 | spid = subprocess.check_output(['git', 'patch-id', '--stable'], |
| 101 | stdin=ps.stdout).decode('utf-8', errors='ignore') |
| 102 | patchid = spid.split(' ', 1)[0] |
| 103 | |
| 104 | # Do nothing if sha is in stable database |
| 105 | sc.execute("select sha from commits where sha='%s'" % sha) |
| 106 | found = sc.fetchone() |
| 107 | if found: |
| 108 | continue |
| 109 | |
| 110 | # Do nothing if sha is already in database |
| 111 | c.execute("select sha from commits where sha='%s'" % sha) |
| 112 | found = c.fetchone() |
| 113 | if found: |
| 114 | continue |
| 115 | |
| 116 | last = sha |
| 117 | |
| 118 | usha = search_usha(sha, description) |
Hirthanan Subenderan | b8402a1 | 2020-02-05 14:11:00 -0800 | [diff] [blame^] | 119 | changeid = parse_changeID(sha) |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 120 | |
| 121 | c.execute('INSERT INTO commits(sha, usha, patchid,' \ |
Hirthanan Subenderan | b8402a1 | 2020-02-05 14:11:00 -0800 | [diff] [blame^] | 122 | 'description, changeid) VALUES (?, ?, ?, ?, ?)', |
| 123 | (sha, usha, patchid, description, changeid)) |
Hirthanan Subenderan | 3e884d6 | 2020-01-23 13:12:45 -0800 | [diff] [blame] | 124 | if last: |
| 125 | c.execute("UPDATE tip set sha='%s' where ref=1" % last) |
| 126 | |
| 127 | conn.commit() |
| 128 | conn.close() |
| 129 | sconn.close() |
| 130 | |
| 131 | |
| 132 | def update_chromeosdb(): |
| 133 | """Updates the chromeosdb for all chromeos branches.""" |
| 134 | os.chdir(CHROMEOS_PATH) |
| 135 | |
| 136 | for branch in CHROMEOS_BRANCHES: |
| 137 | start = 'v%s' % branch |
| 138 | cdb = chromeosdb(branch) |
| 139 | sdb = stabledb(branch) |
| 140 | bname = chromeos_branch(branch) |
| 141 | |
| 142 | print('Handling %s' % bname) |
| 143 | |
| 144 | try: |
| 145 | conn = sqlite3.connect(cdb) |
| 146 | conn.text_factory = str |
| 147 | |
| 148 | c = conn.cursor() |
| 149 | c.execute('select sha from tip') |
| 150 | sha = c.fetchone() |
| 151 | conn.close() |
| 152 | if sha and sha[0]: |
| 153 | start = sha[0] |
| 154 | except sqlite3.Error: |
| 155 | createdb(cdb, make_downstream_table) |
| 156 | |
| 157 | subprocess.run(['git', 'checkout', bname]) |
| 158 | subprocess.run(['git', 'pull']) |
| 159 | |
| 160 | update_commits(start, cdb, sdb) |
| 161 | |
| 162 | os.chdir(WORKDIR) |
| 163 | |
| 164 | |
| 165 | if __name__ == '__main__': |
| 166 | update_chromeosdb() |