scripts/fetch_node_modules.py

#!/usr/bin/env python3
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Simple script to download the pinned Node modules from GCS.

This script exists because the node_modules archive currently contains
a node_modules/.bin directory with a bunch of symlinked files in it,
and download_from_google_storage.py won't let you have archives with
symlinks.

In theory we should probably rebuild the node_modules distro without
the ./bin directory (using `npm install --no-bin-lnks`) but that would
cause the build scripts to fail and we'd have to replace `npmw` with
something else.
"""

import argparse
import hashlib
import os
import sys
import tarfile
from urllib.request import urlopen

SRC_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))

def main():
  parser = argparse.ArgumentParser(description=__doc__)
  parser.parse_args()

  with open(os.path.join(SRC_ROOT, 'node_modules.tar.gz.sha1')) as fp:
    expected_sha1 = fp.read().strip()

  actual_sha1 = None
  tgz = os.path.join(SRC_ROOT, 'node_modules.tar.gz')
  if os.path.exists(tgz):
    with open(tgz, 'rb') as fp:
      s = hashlib.sha1()
      s.update(fp.read())
      actual_sha1 = s.hexdigest()

  # TODO(dpranke): Consider whether we should validate that node_modules/
  # and all of the expected files exist as well.
  if actual_sha1 == expected_sha1:
    return 0

  url = 'https://storage.googleapis.com/%s/%s' % (
      'chromium-website-lob-storage', expected_sha1)
  with urlopen(url) as url_fp, open(tgz, 'wb') as tgz_fp:
    tgz_fp.write(url_fp.read())

  try:
    # TODO(dpranke): download_from_google_storage puts in a fair amount
    # of effort to not clobber an existing directory until it is sure it
    # can extract the archive completely. Consider whether we should do
    # the same.
    with tarfile.open(tgz, 'r:gz') as tar:
      tar.extractall(path=SRC_ROOT)
    return 0
  except Exception as e:
    print(e)
    return 1

if __name__ == '__main__':
  sys.exit(main())