scripts/common.py

# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import dataclasses
import multiprocessing
import os
import queue
import sys
import threading
import time
import urllib.parse


site = 'https://www.chromium.org'

REPO_DIR = os.path.dirname(os.path.dirname(__file__))
SOURCE_DIR = 'site'
BUILD_DIR = 'build'
DEFAULT_TEMPLATE = '/_includes/page.html'



alternates = [
    site,
    'http://dev.chromium.org',
    'https://dev.chromium.org',
    'https://sites.google.com/a/chromium.org/dev',
    'https://ssl.gstatic.com/sites/p/058338',
    'http://www.gstatic.com/sites/p/058338',
]


def cpu_count():
    return multiprocessing.cpu_count()


def read_text_file(path):
    return read_binary_file(path).decode('utf-8')


def read_binary_file(path):
    with open(path, 'rb') as fp:
        return fp.read()


def write_binary_file(path, content):
    with open(path, 'wb') as fp:
        return fp.write(content)

def read_paths(path):
    paths = set()
    with open(path) as fp:
        for line in fp.readlines():
            idx = line.find('#')
            if idx != -1:
                line = line[:idx]
            line = line.strip()
            if line:
                paths.add(line)
    return paths


def to_path(page, top=SOURCE_DIR, ext='.md'):
    page = page.strip()
    if page == '/':
        page = ''
    if os.path.isdir(top + page):
        return page + '/index' + ext
    if os.path.exists(top + page):
        return page
    if os.path.exists(top + page + ext):
        return page + ext
    return page


def walk(top, skip=None):
    skip = skip or set()
    paths = set()
    for dirpath, dnames, fnames in os.walk(top):
        for dname in dnames:
            rpath = os.path.relpath(os.path.join(dirpath, dname), top)
            if rpath in skip or dname.startswith('.'):
                dnames.remove(dname)
        for fname in fnames:
            rpath = os.path.relpath(os.path.join(dirpath, fname), top)
            if rpath in skip or fname.startswith('.'):
                continue
            paths.add(rpath)
    return sorted(paths)


def write_if_changed(path, content):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    if os.path.exists(path):
        with open(path, 'rb') as fp:
            old_content = fp.read()
            if content == old_content:
                return False
    write_binary_file(path, content)
    return True


def should_update(dest_page, source_pages):
    if not os.path.exists(dest_page):
        return True

    dest_pages = [dest_page]
    max_source_mtime = max(os.stat(p).st_mtime for p in source_pages)
    max_dest_mtime = max(os.stat(p).st_mtime for p in dest_pages)
    return max_source_mtime > max_dest_mtime 


class JobQueue:
    def __init__(self, handler, jobs, multiprocess=None):
        self.handler = handler
        self.jobs = jobs
        self.pending = set()
        self.started = set()
        self.finished = set()
        if multiprocess is None:
            self.multiprocess = (jobs > 1)
        else:
            self.multiprocess = multiprocess
        if self.multiprocess:
            self._request_q = multiprocessing.Queue()
            self._response_q = multiprocessing.Queue()
        else:
            self._request_q = queue.Queue()
            self._response_q = queue.Queue()
        self._start_time = None
        self._threads = []
        self._last_msg = None
        self._isatty = sys.stdout.isatty()

    def all_tasks(self):
        return self.pending | self.started | self.finished

    def request(self, task, obj):
        self.pending.add(task)
        self._request_q.put(('handle', task, obj))

    def results(self):
        self._start_time = time.time()
        self._spawn()

        while self.pending | self.started:
            msg, task, res, obj = self._response_q.get()

            if msg == 'started':
                self._mark_started(task)
            elif msg == 'finished':
                self._mark_finished(task, res)
                yield (task, res, obj)
            else:
                raise AssertionError

        for _ in self._threads:
            self._request_q.put(('exit', None, None))
        for thread in self._threads:
            thread.join()
        if self._isatty:
            print()

    def _spawn(self):
        args = (self._request_q, self._response_q, self.handler)
        for i in range(self.jobs):
            if self.multiprocess:
                thread = multiprocessing.Process(target=_worker,
                                                 name='worker-%d' % i,
                                                 args=args)
            else:
                thread = threading.Thread(target=_worker,
                                          name='worker-%d' % i,
                                          args=args)
            self._threads.append(thread)
            thread.start()

    def _mark_started(self, task):
        self.pending.remove(task)
        self.started.add(task)

    def _mark_finished(self, task, res):
        self.started.remove(task)
        self.finished.add(task)
        if res:
            self._print('%s failed:' % task, truncate=False)
            print()
            print(res)
        else:
            self._print('%s' % task)
        sys.stdout.flush()

    def _print(self, msg, truncate=True):
        if not self._isatty:
            print('[%d/%d] %s' % (len(self.finished), len(self.all_tasks()),
                                  msg))
            return

        if len(msg) > 76 and truncate:
            msg = msg[:76] + '...'
        if self._last_msg is not None:
            print('\r', end='')
        msg = '[%d/%d] %s' % (len(self.finished), len(self.all_tasks()), msg)
        print(msg, end='' if self._isatty else '\n')
        if self._last_msg is not None and len(self._last_msg) > len(msg):
            print(' ' * (len(self._last_msg) - len(msg)), end='')
            print('\r', end='')
        self._last_msg = msg


def _worker(request_q, response_q, handler):
    while True:
        message, task, obj = request_q.get()
        if message == 'exit':
            break
        elif message == 'handle':
            response_q.put(('started', task, '', None))
            res, resp = handler(task, obj)
            response_q.put(('finished', task, res, resp))
        else:
            raise AssertionError