automation: python scripts sync'ing linux repos/db
1. Synchronize linux-upstream, linux-stable, linux-chromeos repositories
2. Updates SQL tables with most recently pulled repo data
BUG=None
TEST=None
Change-Id: I8909dfd9c38124abb66daf0498f4c24b6ae70e6f
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/2050096
Reviewed-by: Guenter Roeck <groeck@chromium.org>
Tested-by: Hirthanan Subenderan <hirthanan@google.com>
Commit-Queue: Hirthanan Subenderan <hirthanan@google.com>
diff --git a/contrib/findmissing/common.py b/contrib/findmissing/common.py
index dfa93b1..dc17dff 100755
--- a/contrib/findmissing/common.py
+++ b/contrib/findmissing/common.py
@@ -8,11 +8,25 @@
"""Module containing shared helper methods."""
from __future__ import print_function
-
import os
import sqlite3
import re
+
+KERNEL_SITE = 'https://git.kernel.org/'
+UPSTREAM_REPO = KERNEL_SITE + 'pub/scm/linux/kernel/git/torvalds/linux'
+STABLE_REPO = KERNEL_SITE + 'pub/scm/linux/kernel/git/stable/linux-stable'
+
+CHROMIUM_SITE = 'https://chromium.googlesource.com/'
+CHROMEOS_REPO = CHROMIUM_SITE + 'chromiumos/third_party/kernel'
+CHROMIUM_REVIEW_BASEURL = 'https://chromium-review.googlesource.com/'
+
+SUPPORTED_KERNELS = ('4.4', '4.14', '4.19', '5.4')
+
+CHROMEOS_PATH = 'linux-chrome'
+STABLE_PATH = 'linux-stable'
+UPSTREAM_PATH = 'linux-upstream'
+
WORKDIR = os.getcwd()
DBDIR = os.path.join(WORKDIR, 'database')
UPSTREAMDB = os.path.join(DBDIR, 'upstream.db')
@@ -37,10 +51,12 @@
"""Path of patchdb for stable versions."""
return os.path.join(DBDIR, 'patch-stable-%s.db' % version)
+
def patchdb_chromeos(version):
"""Path of patchdb for chromeos versions."""
return os.path.join(DBDIR, 'patch-chromeos-%s.db' % version)
+
def stable_branch(version):
"""Stable branch name"""
return 'linux-%s.y' % version
diff --git a/contrib/findmissing/config.py b/contrib/findmissing/config.py
deleted file mode 100644
index 00b1dfb..0000000
--- a/contrib/findmissing/config.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# -*- coding: utf-8 -*-"
-#
-# Copyright 2020 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Module containing kernels configuration information."""
-
-KERNEL_SITE = 'https://git.kernel.org/'
-UPSTREAM_REPO = KERNEL_SITE + 'pub/scm/linux/kernel/git/torvalds/linux'
-STABLE_REPO = KERNEL_SITE + 'pub/scm/linux/kernel/git/stable/linux-stable'
-
-CHROMIUM_SITE = 'https://chromium.googlesource.com/'
-CHROMEOS_REPO = CHROMIUM_SITE + 'chromiumos/third_party/kernel'
-CHROMIUM_REVIEW_BASEURL = 'https://chromium-review.googlesource.com/'
-
-STABLE_BRANCHES = ('4.4', '4.14', '4.19', '5.4')
-STABLE_PATTERN = 'linux-%s.y'
-
-CHROMEOS_BRANCHES = ('4.4', '4.14', '4.19', '5.4')
-CHROMEOS_PATTERN = 'chromeos-%s'
-
-CHROMEOS_PATH = 'linux-chrome'
-STABLE_PATH = 'linux-stable'
-UPSTREAM_PATH = 'linux-upstream'
diff --git a/contrib/findmissing/gerrit_interface.py b/contrib/findmissing/gerrit_interface.py
index 150e237..646d31c 100755
--- a/contrib/findmissing/gerrit_interface.py
+++ b/contrib/findmissing/gerrit_interface.py
@@ -11,11 +11,10 @@
"""
from __future__ import print_function
-
import json
import requests
-from config import CHROMIUM_REVIEW_BASEURL
+from common import CHROMIUM_REVIEW_BASEURL
def get_commit(changeid):
diff --git a/contrib/findmissing/initdb_chromeos.py b/contrib/findmissing/initdb_chromeos.py
index ff368e5..0d3e908 100755
--- a/contrib/findmissing/initdb_chromeos.py
+++ b/contrib/findmissing/initdb_chromeos.py
@@ -8,14 +8,13 @@
"""Module rebuilding database with metadata about chromeos patches."""
from __future__ import print_function
-
import sqlite3
import os
import re
import subprocess
-from config import CHROMEOS_PATH, CHROMEOS_BRANCHES
-from common import WORKDIR, CHERRYPICK, STABLE, STABLE2, make_downstream_table
-from common import stabledb, chromeosdb, chromeos_branch, createdb
+from common import CHROMEOS_PATH, SUPPORTED_KERNELS, \
+ WORKDIR, CHERRYPICK, STABLE, STABLE2, make_downstream_table, \
+ stabledb, chromeosdb, chromeos_branch, createdb
UPSTREAM = re.compile(r'(ANDROID: *|UPSTREAM: *|FROMGIT: *|BACKPORT: *)+(.*)')
CHROMIUM = re.compile(r'(CHROMIUM: *|FROMLIST: *)+(.*)')
@@ -133,7 +132,7 @@
"""Updates the chromeosdb for all chromeos branches."""
os.chdir(CHROMEOS_PATH)
- for branch in CHROMEOS_BRANCHES:
+ for branch in SUPPORTED_KERNELS:
start = 'v%s' % branch
cdb = chromeosdb(branch)
sdb = stabledb(branch)
diff --git a/contrib/findmissing/initdb_stable.py b/contrib/findmissing/initdb_stable.py
index 709c374..c4da40f 100755
--- a/contrib/findmissing/initdb_stable.py
+++ b/contrib/findmissing/initdb_stable.py
@@ -8,13 +8,12 @@
"""Module parses and stores data from stable linux patch."""
from __future__ import print_function
-
import sqlite3
import os
import subprocess
-from config import STABLE_PATH, STABLE_BRANCHES
-from common import WORKDIR, CHERRYPICK, STABLE, STABLE2, make_downstream_table
-from common import stabledb, stable_branch, createdb
+from common import STABLE_PATH, SUPPORTED_KERNELS, \
+ WORKDIR, CHERRYPICK, STABLE, STABLE2, make_downstream_table, \
+ stabledb, stable_branch, createdb
def search_usha(sha):
@@ -89,7 +88,7 @@
"""Updates the stabledb index for all stable branches."""
os.chdir(STABLE_PATH)
- for branch in STABLE_BRANCHES:
+ for branch in SUPPORTED_KERNELS:
start = 'v%s' % branch
db = stabledb(branch)
bname = stable_branch(branch)
diff --git a/contrib/findmissing/initdb_upstream.py b/contrib/findmissing/initdb_upstream.py
index f8ccef2..cd059f4 100755
--- a/contrib/findmissing/initdb_upstream.py
+++ b/contrib/findmissing/initdb_upstream.py
@@ -8,16 +8,15 @@
"""Module parses and stores mainline linux patches to be easily accessible."""
from __future__ import print_function
-
import os
import re
import sqlite3
import subprocess
-from config import UPSTREAM_PATH, STABLE_BRANCHES
-from common import WORKDIR, UPSTREAMDB, createdb
+
+from common import WORKDIR, UPSTREAMDB, createdb, UPSTREAM_PATH, SUPPORTED_KERNELS
-UPSTREAM_BASE = 'v' + STABLE_BRANCHES[0]
+UPSTREAM_BASE = 'v' + SUPPORTED_KERNELS[0]
RF = re.compile(r'^\s*Fixes: (?:commit )*([0-9a-f]+).*')
RDESC = re.compile(r'.* \("([^"]+)"\).*')
diff --git a/contrib/findmissing/missing.py b/contrib/findmissing/missing.py
index 852aff7..fda3e99 100755
--- a/contrib/findmissing/missing.py
+++ b/contrib/findmissing/missing.py
@@ -14,7 +14,7 @@
import sqlite3
from enum import Enum
-import config
+import common
from common import stabledb, UPSTREAMDB, stable_branch, chromeosdb, \
chromeos_branch, patch_link, patchdb_stable, patchdb_chromeos, createdb
from patch import PatchEntry, Status, make_patch_table
@@ -198,11 +198,10 @@
if len(sys.argv) > 1:
branches = sys.argv[1:]
else:
- branches = config.STABLE_BRANCHES if release == Path.stable \
- else config.CHROMEOS_BRANCHES
+ branches = common.SUPPORTED_KERNELS
- path = config.STABLE_PATH if release == Path.stable \
- else config.CHROMEOS_PATH
+ path = common.STABLE_PATH if release == Path.stable \
+ else common.CHROMEOS_PATH
os.chdir(path)
for b in branches:
diff --git a/contrib/findmissing/setup.sh b/contrib/findmissing/setup.sh
deleted file mode 100755
index 8cf64e9..0000000
--- a/contrib/findmissing/setup.sh
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2020 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Note: Collabora repository with pending patches
-# https://git.collabora.com/cgit/linux.git/log/?h=topic/chromeos/waiting-for-upstream
-
-stable_path=$(python -c "from config import STABLE_PATH; print(STABLE_PATH);")
-stable_repo=$(python -c "from config import STABLE_REPO; print(STABLE_REPO);")
-
-chromeos_path=$(python -c "from config import CHROMEOS_PATH; print(CHROMEOS_PATH);")
-chromeos_repo=$(python -c "from config import CHROMEOS_REPO; print(CHROMEOS_REPO);")
-
-upstream_path=$(python -c "from config import UPSTREAM_PATH; print(UPSTREAM_PATH);")
-if [[ "$(dirname "${upstream_path}")" = "." ]]; then
- # Needs to be an absolute path name
- upstream_path="$(pwd)/${upstream_path}"
-fi
-upstream_repo=$(python -c "from config import UPSTREAM_REPO; print(UPSTREAM_REPO);")
-
-sbranches=("$(python -c "from config import STABLE_BRANCHES; print(STABLE_BRANCHES)" | tr -d "(),'")")
-spattern=("$(python -c "from config import STABLE_PATTERN; print(STABLE_PATTERN)" | tr -d "(),'")")
-cbranches=("$(python -c "from config import CHROMEOS_BRANCHES; print(CHROMEOS_BRANCHES)" | tr -d "(),'")")
-cpattern=("$(python -c "from config import CHROMEOS_PATTERN; print(CHROMEOS_PATTERN)" | tr -d "(),'")")
-
-# Simple clone:
-# Clone repository, do not add 'upstream' remote
-clone_simple()
-{
- local destdir=$1
- local repository=$2
- local force=$3
-
- echo "Cloning ${repository} into ${destdir}"
-
- if [[ -d "${destdir}" ]]; then
- pushd "${destdir}" >/dev/null || exit
- git checkout master
- if [[ -n "${force}" ]]; then
- # This is needed if the origin may have been rebased
- git fetch origin
- git reset --hard origin/master
- else
- git pull
- fi
- popd >/dev/null || exit
- else
- git clone "${repository}" "${destdir}"
- fi
-}
-
-clone_simple "${upstream_path}" "${upstream_repo}"
-
-# Complex clone:
-# Clone repository, add 'upstream' remote,
-# check out and update list of branches
-clone_complex()
-{
- local destdir=$1
- local repository=$2
- local pattern=$3
- local branches=("${!4}")
-
- echo "Cloning ${repository} into ${destdir}"
-
- if [[ -d "${destdir}" ]]; then
- pushd "${destdir}" >/dev/null || exit
- git reset --hard HEAD
- git fetch origin
- for branch in ${branches[*]}; do
- branch="$(printf '%s %s' "${pattern}" "${branch}")"
- if git rev-parse --verify "${branch}" >/dev/null 2>&1; then
- git checkout "${branch}"
- if ! git pull; then
- # git pull may fail if the remote repository was rebased.
- # Pull it the hard way.
- git reset --hard "origin/${branch}"
- fi
- else
- git checkout -b "${branch}" "origin/${branch}"
- fi
- done
- git remote -v | grep upstream || {
- git remote add upstream "${upstream_path}"
- }
- git fetch upstream
- popd >/dev/null || exit
- else
- git clone "${repository}" "${destdir}"
- pushd "${destdir}" >/dev/null || exit
- for branch in ${branches[*]}; do
- branch="$(printf '%s %s' "${pattern}" "${branch}")"
- git checkout -b "${branch}" "origin/${branch}"
- done
- git remote add upstream "${upstream_path}"
- git fetch upstream
- popd >/dev/null || exit
- fi
-}
-
-clone_complex "${stable_path}" "${stable_repo}" "${spattern[0]}" "${sbranches[*]}"
-clone_complex "${chromeos_path}" "${chromeos_repo}" "${cpattern[0]}" "${cbranches[*]}"
-
-echo "Initializing databases"
-python initdb_upstream.py
-python initdb_stable.py
-python initdb_chromeos.py
diff --git a/contrib/findmissing/synchronize.py b/contrib/findmissing/synchronize.py
new file mode 100755
index 0000000..d05907e
--- /dev/null
+++ b/contrib/findmissing/synchronize.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-"
+#
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Setup module containing script to Synchronize kernel repositories + database."""
+
+from __future__ import print_function
+import os
+import subprocess
+
+from common import UPSTREAM_PATH, CHROMEOS_PATH, STABLE_PATH, \
+ UPSTREAM_REPO, CHROMEOS_REPO, STABLE_REPO, \
+ SUPPORTED_KERNELS, stable_branch, chromeos_branch
+
+from initdb_upstream import update_upstreamdb
+from initdb_stable import update_stabledb
+from initdb_chromeos import update_chromeosdb
+
+
+def synchronize_upstream():
+ """Synchronizes locally cloned repo with linux upstream remote."""
+ cwd = os.getcwd()
+ destdir = os.path.join(cwd, UPSTREAM_PATH)
+ repo = UPSTREAM_REPO
+
+ print(destdir, repo, cwd)
+
+ if not os.path.exists(destdir):
+ print(f'Cloning {repo} into {destdir}')
+ cmd = f'git clone {repo} {destdir}'.split(' ')
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ p.wait()
+
+ else:
+ os.chdir(destdir)
+
+ print(f'Updating {repo} into {destdir}')
+ cmd = 'git checkout master; git pull'.split(' ')
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
+ p.wait()
+
+ os.chdir(cwd)
+
+
+def synchronize_custom(path, repo):
+ """Synchronizes locally cloned repo with linux stable/chromeos remote."""
+ cwd = os.getcwd()
+ destdir = os.path.join(cwd, path)
+ upstream_destdir = os.path.join(cwd, UPSTREAM_PATH)
+
+ get_branch = stable_branch if path == 'linux-stable' else chromeos_branch
+
+ if not os.path.exists(destdir):
+ print(f'Cloning {repo} into {destdir}')
+ cmd = f'git clone {repo} {destdir}'.split(' ')
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ p.wait()
+
+ os.chdir(destdir)
+ for kernel in SUPPORTED_KERNELS:
+ cmd = f'git checkout -b {get_branch(kernel)} origin/{get_branch(kernel)}'.split(' ')
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ p.wait()
+
+ cmd = f'git remote add upstream {upstream_destdir}; git fetch upstream'
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
+ p.wait()
+
+ else:
+ os.chdir(destdir)
+
+ print(f'Updating {repo} into {destdir}')
+ cmd = 'git reset --hard HEAD; git fetch origin'
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
+ p.wait()
+
+ for kernel in SUPPORTED_KERNELS:
+ branch = get_branch(kernel)
+ cmd = f'git rev-parse --verify {branch}'.split(' ')
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ p.wait()
+
+ output, _ = p.communicate()
+ if output:
+ cmd = f'git checkout {branch}'.split(' ')
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ p.wait()
+
+ cmd = 'git pull'.split(' ')
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ p.wait()
+
+ output, _ = p.communicate()
+ if not output:
+ cmd = f'git reset --hard origin/{branch}'.split(' ')
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ p.wait()
+ else:
+ cmd = f'git checkout -b {branch} origin/{branch}'.split(' ')
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ p.wait()
+
+ os.chdir(cwd)
+
+
+def synchronize_repositories():
+ """Deep clones linux-upstream, linux-stable, and linux-chromeos repositories"""
+ synchronize_upstream()
+ synchronize_custom(STABLE_PATH, STABLE_REPO)
+ synchronize_custom(CHROMEOS_PATH, CHROMEOS_REPO)
+
+def synchronize_database():
+ """Synchronizes the databases for upstream, stable, and chromeos."""
+ update_upstreamdb()
+ update_stabledb()
+ update_chromeosdb()
+
+
+if __name__ == '__main__':
+ synchronize_repositories()
+ synchronize_database()