path: root/upstream-layers/yocto-docs/documentation/set_versions.py

#!/usr/bin/env python3
#
# Add version information to poky.yaml based upon current git branch/tags
# Also generate the list of available manuals (releases.rst file)
#
# Copyright Linux Foundation
# Author: Richard Purdie <richard.purdie@linuxfoundation.org>
# Author: Quentin Schulz <foss@0leil.net>
#
# SPDX-License-Identifier: MIT
#


import json
import subprocess
import collections
import sys
import os
import itertools

from urllib.request import urlopen, URLError

# NOTE: these variables contain default values in case we are not able to fetch
# the releases.json file from https://dashboard.yoctoproject.org/releases.json
activereleases = ["whinlatter", "scarthgap", "kirkstone"]
devbranch = "wrynose"
ltsseries = ["wrynose", "scarthgap", "kirkstone"]
release_series = collections.OrderedDict({
    "wrynose": "6.0",
    "whinlatter": "5.3",
    "scarthgap": "5.0",
    "kirkstone": "4.0",
})
bitbake_mapping = collections.OrderedDict({
    "wrynose": "2.18",
    "whinlatter": "2.16",
    "scarthgap": "2.8",
    "kirkstone": "2.0",
})

releases_from_json = {}

# Use the local releases.json file if found, fetch it from the dashboard otherwise
releases_json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "releases.json")
try:
    with open(releases_json_path, "r") as f:
        releases_from_json = json.load(f)
except FileNotFoundError:
    print("Fetching releases.json from https://dashboard.yoctoproject.org/releases.json...",
          file=sys.stderr)
    try:
        with urlopen("https://dashboard.yoctoproject.org/releases.json") as r, \
                open(releases_json_path, "w") as f:
            releases_from_json = json.load(r)
            json.dump(releases_from_json, f)
    except URLError:
        print("WARNING: tried to fetch https://dashboard.yoctoproject.org/releases.json "
              "but failed, using default values for active releases", file=sys.stderr)
        pass

if releases_from_json:
    release_series = collections.OrderedDict()
    activereleases = []
    devbranch = ""
    ltsseries = []
    bitbake_mapping = collections.OrderedDict()

    for release in releases_from_json:
        codename = release["release_codename"].lower()
        release_series[codename] = release["series_version"]
        if release["status"] == "Active Development":
            devbranch = codename
        if "LTS until" in release["status"]:
            ltsseries.append(codename)
        if release["bitbake_version"]:
            bitbake_mapping[codename] = release["bitbake_version"]

    # Find the first non-dev release, which should be displayed as the default
    # page on the docs website.
    current_branch = ""
    for release in releases_from_json:
        if release["status"] != "Active Development":
            current_branch = release["release_codename"].lower()
            break

    if not current_branch:
        sys.exit("Unable to find a current release! Exiting...")

    # make the list of releases unique, there can be duplication when the
    # current releases is also an LTS
    activereleases = list(dict.fromkeys([current_branch] + ltsseries))

# used by run-docs-builds to get the default page
if len(sys.argv) > 1 and sys.argv[1] == "getlatest":
    print(activereleases[0])
    sys.exit(0)

print(f"activereleases calculated to be {activereleases}")
print(f"devbranch calculated to be {devbranch}")
print(f"ltsseries calculated to be {ltsseries}")

ourversion = None
ourseries = None
ourbranch = None
bitbakeversion = None
docconfver = None
head_commit = None
is_branch_tip = False

# Test that we are building from a Git repository
try:
    subprocess.run(["git", "rev-parse", "--is-inside-work-tree"],
                   stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
except subprocess.CalledProcessError:
    sys.exit("Building yocto-docs must be done from its Git repository.\n"
             "Clone the documentation repository with the following command:\n"
             "git clone https://git.yoctoproject.org/yocto-docs ")

# Test tags exist and inform the user to fetch if not
try:
    subprocess.run(["git", "show", "yocto-%s" % release_series[activereleases[0]]],
                   stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
except subprocess.CalledProcessError:
    sys.exit("Please run 'git fetch --tags' before building the documentation")

# Try and figure out what we are
tags = subprocess.run(["git", "tag", "--points-at", "HEAD"],
                      stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).stdout
for t in tags.split():
    if t.startswith("yocto-"):
        ourversion = t[6:]
        # remove "_MX" milestone information to get the series
        seriesversion = ourversion.split("_")[0]
        seriesversion = ".".join(seriesversion.split(".")[0:2])
        for series in release_series:
            if release_series[series] == seriesversion:
                ourseries = series
                if ourseries == devbranch:
                    bitbakeversion = "dev"
                else:
                    bitbakeversion = bitbake_mapping[ourseries]
                break
        break

if ourversion is None:
    # We're floating on a branch
    branch = subprocess.run(["git", "branch", "--show-current"],
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                            universal_newlines=True).stdout.strip()

    if branch == "" or branch not in list(release_series.keys()) + ["master", "master-next"]:
        # We're not on a known release branch so we have to guess. Compare the
        # numbers of commits from each release branch and assume the smallest
        # number of commits is the one we're based off
        possible_branch = None
        branch_count = 0
        for b in itertools.chain(release_series.keys(), ["master"]):
            # The laverne branch was created but never branched off, which
            # breaks this algorithm as this always gets count 0. Skip this
            # branch as it is old and we'll never branch off of it now.
            if b == "laverne":
                continue
            result = subprocess.run(["git", "log", "--format=oneline", "HEAD..origin/" + b],
                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                    universal_newlines=True)
            if result.returncode == 0:
                count = result.stdout.count('\n')
                if not possible_branch or count < branch_count:
                    print("Branch %s has count %s" % (b, count))
                    possible_branch = b
                    branch_count = count
        if possible_branch:
            branch = possible_branch
        else:
            branch = "master"
        print("Nearest release branch estimated to be %s" % branch)

    if branch == "master":
        ourversion = "dev"
        ourseries = devbranch
        bitbakeversion = ourversion
    elif branch == "master-next":
        ourversion = "next"
        ourseries = devbranch
        bitbakeversion = ourversion
    else:
        ourseries = branch
        bitbakeversion = bitbake_mapping[ourseries]
        ourversion = release_series[branch]
        head_commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"],
                                     stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                     universal_newlines=True).stdout.strip()
        branch_commit = subprocess.run(["git", "rev-parse", "--short", branch],
                                       stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                       universal_newlines=True).stdout.strip()
        if head_commit != branch_commit:
            ourversion += f"-{head_commit}"
        else:
            is_branch_tip = True
            ourversion += "-tip"

series = [k for k in release_series]
previousseries = series[series.index(ourseries) + 1:] or [""]
lastlts = [k for k in previousseries if k in ltsseries] or "dunfell"

latestreltag = subprocess.run(["git", "describe", "--abbrev=0", "--tags", "--match", "yocto-*"],
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).stdout
latestreltag = latestreltag.strip()
if latestreltag:
    if latestreltag.startswith("yocto-"):
        latesttag = latestreltag[6:]
else:
    # fallback on the calculated version
    print("Did not find a tag with 'git describe', falling back to %s" % ourversion)
    latestreltag = "yocto-" + ourversion
    latesttag = ourversion

print("Version calculated to be %s" % ourversion)
print("Latest release tag found is %s" % latestreltag)
print("Release series calculated to be %s" % ourseries)
print("Bitbake version calculated to be %s" % bitbakeversion)

# The &DISTRO; replacement will be mostly right when just equaling ourversion.
# When we're on a random commit, in that case the DISTRO value will
# contain the hash. This would for example be "5.3-4d2acc043".
# This will not happen when publishing the documentation from the
# autobuilder as we don't build specific commits - this should only happen
# for development.
distro = ourversion
# Few exceptions though:
if is_branch_tip:
    # we're on a branch tip, in that case the closest match is the latest tag
    # from that branch. Some instructions rely on DISTRO to provide tag names,
    # etc. so it makes sense provide the latest tag from that branch.
    distro = latesttag
elif distro in ["dev", "next"]:
    # When building on master or master-next, make the distro the version of the
    # devbranch.
    distro = release_series[devbranch]

print(f"DISTRO calculated to be {distro}")

replacements = {
    "DISTRO": distro,
    "DISTRO_LATEST_TAG": latesttag,
    "DISTRO_RELEASE_SERIES": release_series[ourseries],
    "DISTRO_NAME_NO_CAP": ourseries,
    "DISTRO_NAME": ourseries.capitalize(),
    "DISTRO_NAME_NO_CAP_MINUS_ONE": previousseries[0],
    "DISTRO_NAME_NO_CAP_LTS": lastlts[0],
    "YOCTO_DOC_VERSION": ourversion,
    "DOCCONF_VERSION": ourversion,
    "BITBAKE_SERIES": bitbakeversion,
}

# 3.4 onwards doesn't have poky version
# Early 3.4 release docs do reference it though
poky_mapping = {
    "3.4": "26.0",
    "3.3": "25.0",
    "3.2": "24.0",
    "3.1": "23.0",
}

if release_series[ourseries] in poky_mapping:
    pokyversion = poky_mapping[release_series[ourseries]]
    if ourversion != release_series[ourseries]:
        pokyversion = pokyversion + "." + ourversion.rsplit(".", 1)[1]
    else:
        pokyversion = pokyversion + ".0"
    replacements["POKYVERSION"] = pokyversion

if os.path.exists("poky.yaml.in"):
    with open("poky.yaml.in", "r") as r, open("poky.yaml", "w") as w:
        lines = r.readlines()
        for line in lines:
            data = line.split(":")
            k = data[0].strip()
            if k in replacements:
                w.write("%s : \"%s\"\n" % (k, replacements[k]))
            else:
                w.write(line)

    print("poky.yaml generated from poky.yaml.in")

def get_latest_tag(branch: str) -> str:
    """
    Get the latest tag of `branch`.
    """
    branch_versions = subprocess.run(["git", "tag", "--list", f'yocto-{release_series[branch]}*'],
                                     stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                     universal_newlines=True).stdout.split()
    # Sort the branches as integers
    # On milestone tags (e.g. 6.0_M2), remove everything after "_" (including
    # "_")
    branch_versions = sorted(
        [v.replace("yocto-" + release_series[branch] + ".", "")
         .replace("yocto-" + release_series[branch], "0")
         .split("_")[0] for v in branch_versions],
        key=int)
    if not branch_versions:
        return ""

    version = release_series[branch]
    if branch_versions[-1] != "0":
        version = version + "." + branch_versions[-1]
    return version

def get_abbrev_hash(ref: str) -> str:
    """
    Get the abbreviated hash of a ref using a call to git rev-parse.
    """
    return subprocess.run(["git", "rev-parse", "--short", ref],
                          stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                          universal_newlines=True).stdout.strip()

versions = []
with open("sphinx-static/switchers.js.in", "r") as r, open("sphinx-static/switchers.js", "w") as w:
    lines = r.readlines()
    for line in lines:
        if "VERSIONS_PLACEHOLDER" in line:
            w.write(f"    'dev': {{'title': 'Unstable (dev)', 'hash': '{get_abbrev_hash('master')}'}},\n")
            for branch in activereleases:
                series = release_series[branch]
                w.write(f"    '{series}-tip': {{'title': '{branch.capitalize()} ({get_latest_tag(branch)})', "
                        f"'hash': '{get_abbrev_hash(branch)}'}},\n")
        elif "ALL_RELEASES_PLACEHOLDER" in line:
            for series in release_series:
                w.write(f"    '{release_series[series]}': "
                        f"{{'codename': '{series.capitalize()}', "
                        f"'latest_tag': '{get_latest_tag(series)}'}},\n")
        elif "LATEST_VERSION_PLACEHOLDER" in line:
            latest_series = release_series[activereleases[0]]
            w.write(f"    '{latest_series}-tip'\n")
        else:
            w.write(line)

print("switchers.js generated from switchers.js.in")

# generate releases.rst

yocto_tags = subprocess.run(["git", "tag", "--list", "--sort=version:refname", "yocto-*"],
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).stdout.split()
tags = [tag[6:] for tag in yocto_tags]

with open('releases.rst', 'w') as f:
    f.write('===========================\n')
    f.write(' Supported Release Manuals\n')
    f.write('===========================\n')
    f.write('\n')

    for activerelease in activereleases:
        title = "Release Series %s (%s)" % (release_series[activerelease], activerelease)
        f.write('*' * len(title) + '\n')
        f.write(title + '\n')
        f.write('*' * len(title) + '\n')
        f.write('\n')

        for tag in tags:
            if tag == release_series[activerelease] or tag.startswith('%s.' % release_series[activerelease]):
                f.write('- :yocto_docs:`%s Documentation </%s>`\n' % (tag, tag))
        f.write('\n')

    f.write('==========================\n')
    f.write(' Outdated Release Manuals\n')
    f.write('==========================\n')
    f.write('\n')

    for series in release_series:
        if series == devbranch or series in activereleases:
            continue

        if series == "jethro-pre":
            continue

        title = "Release Series %s (%s)" % (release_series[series], series)
        f.write('*' * len(title) + '\n')
        f.write(title + '\n')
        f.write('*' * len(title) + '\n')
        f.write('\n')
        if series == "jethro":
            f.write('- :yocto_docs:`1.9 Documentation </1.9>`\n')
        for tag in tags:
            if tag == release_series[series] or tag.startswith('%s.' % release_series[series]):
                f.write('- :yocto_docs:`%s Documentation </%s>`\n' % (tag, tag))
        f.write('\n')