diff options
Diffstat (limited to 'circuitpython/tools/build_board_info.py')
| -rwxr-xr-x | circuitpython/tools/build_board_info.py | 339 |
1 files changed, 339 insertions, 0 deletions
diff --git a/circuitpython/tools/build_board_info.py b/circuitpython/tools/build_board_info.py new file mode 100755 index 0000000..1534288 --- /dev/null +++ b/circuitpython/tools/build_board_info.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors) +# +# SPDX-License-Identifier: MIT + +import json +import os +import subprocess +import sys +import sh +import base64 +from datetime import date +from sh.contrib import git + +sys.path.append("adabot") +import adabot.github_requests as github + +sys.path.append("../docs") +from shared_bindings_matrix import ( + SUPPORTED_PORTS, + aliases_by_board, + support_matrix_by_board, +) + +BIN = ("bin",) +UF2 = ("uf2",) +BIN_UF2 = ("bin", "uf2") +HEX = ("hex",) +HEX_UF2 = ("hex", "uf2") +SPK = ("spk",) +DFU = ("dfu",) +BIN_DFU = ("bin", "dfu") +COMBINED_HEX = ("combined.hex",) +KERNEL8_IMG = ("disk.img.zip", "kernel8.img") +KERNEL_IMG = ("disk.img.zip", "kernel.img") + +# Default extensions +extension_by_port = { + "atmel-samd": UF2, + "broadcom": KERNEL8_IMG, + "cxd56": SPK, + "espressif": BIN_UF2, + "litex": DFU, + "mimxrt10xx": HEX_UF2, + "nrf": UF2, + "raspberrypi": UF2, + "stm": BIN, +} + +# Per board overrides +extension_by_board = { + # samd + "arduino_mkr1300": BIN_UF2, + "arduino_mkrzero": BIN_UF2, + "arduino_nano_33_iot": BIN_UF2, + "arduino_zero": BIN_UF2, + "feather_m0_adalogger": BIN_UF2, + "feather_m0_basic": BIN_UF2, + "feather_m0_rfm69": BIN_UF2, + "feather_m0_rfm9x": BIN_UF2, + "uchip": BIN_UF2, + # nRF52840 dev kits that may not have UF2 bootloaders, + "makerdiary_nrf52840_mdk": HEX, + "makerdiary_nrf52840_mdk_usb_dongle": HEX_UF2, + "pca10056": BIN_UF2, + "pca10059": BIN_UF2, + "electronut_labs_blip": HEX, + "microbit_v2": COMBINED_HEX, + # stm32 + "meowbit_v121": UF2, + # esp32c3 + "adafruit_qtpy_esp32c3": BIN, + "ai_thinker_esp32-c3s": BIN, + "ai_thinker_esp32-c3s-2m": BIN, + "espressif_esp32c3_devkitm_1_n4": BIN, + "lilygo_ttgo_t-01c3": BIN, + "microdev_micro_c3": BIN, + "lilygo_ttgo_t-oi-plus": BIN, + # broadcom + "raspberrypi_zero": KERNEL_IMG, + "raspberrypi_zero_w": KERNEL_IMG, +} + +language_allow_list = set( + [ + "ID", + "de_DE", + "en_GB", + "en_US", + "en_x_pirate", + "es", + "fil", + "fr", + "it_IT", + "ja", + "nl", + "pl", + "pt_BR", + "ru", + "sv", + "tr", + "zh_Latn_pinyin", + ] +) + + +def get_languages(list_all=False): + languages = set() + for f in os.scandir("../locale"): + if f.name.endswith(".po"): + languages.add(f.name[:-3]) + if not list_all: + languages = languages & language_allow_list + return sorted(list(languages), key=str.casefold) + + +def get_board_mapping(): + boards = {} + for port in SUPPORTED_PORTS: + board_path = os.path.join("../ports", port, "boards") + for board_path in os.scandir(board_path): + if board_path.is_dir(): + board_files = os.listdir(board_path.path) + board_id = board_path.name + extensions = extension_by_port[port] + extensions = extension_by_board.get(board_path.name, extensions) + aliases = aliases_by_board.get(board_path.name, []) + boards[board_id] = { + "port": port, + "extensions": extensions, + "download_count": 0, + "aliases": aliases, + } + for alias in aliases: + boards[alias] = { + "port": port, + "extensions": extensions, + "download_count": 0, + "alias": True, + "aliases": [], + } + return boards + + +def get_version_info(): + version = None + sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8") + try: + version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8").strip() + except sh.ErrorReturnCode_128: + # No exact match + pass + + if "GITHUB_SHA" in os.environ: + sha = os.environ["GITHUB_SHA"] + + if not version: + version = "{}-{}".format(date.today().strftime("%Y%m%d"), sha[:7]) + + return sha, version + + +def get_current_info(): + response = github.get("/repos/adafruit/circuitpython-org/git/refs/heads/main") + if not response.ok: + print(response.text) + raise RuntimeError("cannot get main sha") + commit_sha = response.json()["object"]["sha"] + + response = github.get( + "/repos/adafruit/circuitpython-org/contents/_data/files.json?ref=" + commit_sha + ) + if not response.ok: + print(response.text) + raise RuntimeError("cannot get previous files.json") + + response = response.json() + + git_info = commit_sha, response["sha"] + current_list = json.loads(base64.b64decode(response["content"]).decode("utf-8")) + current_info = {} + for info in current_list: + current_info[info["id"]] = info + return git_info, current_info + + +def create_json(updated): + # Convert the dictionary to a list of boards. Liquid templates only handle arrays. + updated_list = [] + all_ids = sorted(updated.keys()) + for id in all_ids: + info = updated[id] + info["id"] = id + updated_list.append(info) + return json.dumps(updated_list, sort_keys=True, indent=1).encode("utf-8") + b"\n" + + +def create_pr(changes, updated, git_info, user): + commit_sha, original_blob_sha = git_info + branch_name = "new_release_" + changes["new_release"] + updated = create_json(updated) + # print(updated.decode("utf-8")) + + pr_title = "Automated website update for release {}".format(changes["new_release"]) + boards = "" + if changes["new_boards"]: + boards = "New boards:\n* " + "\n* ".join(changes["new_boards"]) + languages = "" + if changes["new_languages"]: + languages = "New languages:\n* " + "\n* ".join(changes["new_languages"]) + message = "Automated website update for release {} by Blinka.\n\n{}\n\n{}\n".format( + changes["new_release"], boards, languages + ) + + create_branch = {"ref": "refs/heads/" + branch_name, "sha": commit_sha} + response = github.post("/repos/{}/circuitpython-org/git/refs".format(user), json=create_branch) + if not response.ok and response.json()["message"] != "Reference already exists": + print("unable to create branch") + print(response.text) + return + + update_file = { + "message": message, + "content": base64.b64encode(updated).decode("utf-8"), + "sha": original_blob_sha, + "branch": branch_name, + } + + response = github.put( + "/repos/{}/circuitpython-org/contents/_data/files.json".format(user), json=update_file + ) + if not response.ok: + print("unable to post new file") + print(response.text) + return + pr_info = { + "title": pr_title, + "head": user + ":" + branch_name, + "base": "main", + "body": message, + "maintainer_can_modify": True, + } + response = github.post("/repos/adafruit/circuitpython-org/pulls", json=pr_info) + if not response.ok: + print("unable to create pr") + print(response.text) + return + print(changes) + print(pr_info) + + +def print_active_user(): + response = github.get("/user") + if response.ok: + user = response.json()["login"] + print("Logged in as {}".format(user)) + return user + else: + print("Not logged in") + return None + + +def generate_download_info(): + boards = {} + errors = [] + + new_tag = os.environ["RELEASE_TAG"] + + changes = {"new_release": new_tag, "new_boards": [], "new_languages": []} + + user = print_active_user() + + sha, this_version = get_version_info() + + git_info, current_info = get_current_info() + + languages = get_languages() + + support_matrix = support_matrix_by_board(use_branded_name=False) + + new_stable = "-" not in new_tag + + previous_releases = set() + previous_languages = set() + + # Delete the release we are replacing + for board in current_info: + info = current_info[board] + for version in list(info["versions"]): + previous_releases.add(version["version"]) + previous_languages.update(version["languages"]) + if version["stable"] == new_stable or ( + new_stable and version["version"].startswith(this_version) + ): + info["versions"].remove(version) + + board_mapping = get_board_mapping() + + for port in SUPPORTED_PORTS: + board_path = os.path.join("../ports", port, "boards") + for board_path in os.scandir(board_path): + if board_path.is_dir(): + board_files = os.listdir(board_path.path) + board_id = board_path.name + board_info = board_mapping[board_id] + + for alias in [board_id] + board_info["aliases"]: + alias_info = board_mapping[alias] + if alias not in current_info: + changes["new_boards"].append(alias) + current_info[alias] = {"downloads": 0, "versions": []} + + new_version = { + "stable": new_stable, + "version": new_tag, + "modules": support_matrix[alias][0], + "languages": languages, + "extensions": board_info["extensions"], + "frozen_libraries": [frozen[0] for frozen in support_matrix[alias][1]], + } + current_info[alias]["downloads"] = alias_info["download_count"] + current_info[alias]["versions"].append(new_version) + + changes["new_languages"] = set(languages) - previous_languages + + if changes["new_release"] and user: + create_pr(changes, current_info, git_info, user) + else: + print("No new release to update") + if "DEBUG" in os.environ: + print(create_json(current_info).decode("utf8")) + + +if __name__ == "__main__": + if "RELEASE_TAG" in os.environ and os.environ["RELEASE_TAG"]: + generate_download_info() + else: + print("skipping website update because this isn't a tag") |