chore(tools): centralize all versions related utils in a single script

- update version to be semver compliant with the new tools
- update make targets and CI workflow to use the new version tool
- update release issue template

.github/ISSUE_TEMPLATE/release.md

@@ -1,7 +1,7 @@
 ---
 name: Release
 about: Issue template to prepare a release step by step.
-title: "Release vX.Y.Z (or vX.Y.Zrc?)"
+title: "Release vX.Y.Z (or vX.Y.Z-rc?)"
 ---
 <!-- Make sure to set the proper version in the issue template -->
 Please check all steps if it was either done/already done, at the end of a release all check boxes must have been checked.
@@ -9,20 +9,20 @@ Please check all steps if it was either done/already done, at the end of a relea
 Release check-list:
 <!-- Note that some of these steps will be automated in the future -->
 If it was not already done:
-- [ ] Choose the version number, e.g. `vX.Y.Z` (can be `vX.Y.Zrc?` for Release Candidates) following semantic versioning: https://semver.org/
-- [ ] Update the project version to `X.Y.Z` (or `X.Y.Zrc?`) by running:
+- [ ] Choose the version number, e.g. `vX.Y.Z` (can be `vX.Y.Z-rc?` for Release Candidates) following semantic versioning: https://semver.org/
+- [ ] Update the project version to `X.Y.Z` (or `X.Y.Z-rc?`) by running:
 
 ```bash
 VERSION=X.Y.Z make set_version
 # or
-VERSION=X.Y.Zrc? make set_version
+VERSION=X.Y.Z-rc? make set_version
 ```
 
 Then:
 - [ ] For non RC releases: check the release milestone issues, cut out what can't be completed in time and change the milestones for these issues
-- [ ] Checkout the commit for release, create a signed tag (requires GPG keys setup, see [here](https://docs.github.com/en/github/authenticating-to-github/managing-commit-signature-verification)) with the version name (careful for RC) `git tag -s -a -m "vX.Y.Z release" vX.Y.Z`, (or `vX.Y.Zrc?`) push it to GitHub with `git push origin refs/tags/vX.Y.Z` (or `vX.Y.Zrc?`)
+- [ ] Checkout the commit for release, create a signed tag (requires GPG keys setup, see [here](https://docs.github.com/en/github/authenticating-to-github/managing-commit-signature-verification)) with the version name (careful for RC) `git tag -s -a -m "vX.Y.Z release" vX.Y.Z`, (or `vX.Y.Z-rc?`) push it to GitHub with `git push origin refs/tags/vX.Y.Z` (or `vX.Y.Z-rc?`)
 - [ ] Wait for the release workflow to finish and get the image url from the notification or the logs
-- [ ] See [here](https://docs.github.com/en/github/administering-a-repository/releasing-projects-on-github/managing-releases-in-a-repository#creating-a-release) to create the release in GitHub using the existing tag, add the pull link copied from the step before \(`ghcr.io/zama-ai/concretefhe:vX.Y.Z`\) (or `vX.Y.Zrc?`) for the uploaded docker image. Mark release as pre-release for an `rc` version. See template below:
+- [ ] See [here](https://docs.github.com/en/github/administering-a-repository/releasing-projects-on-github/managing-releases-in-a-repository#creating-a-release) to create the release in GitHub using the existing tag, add the pull link copied from the step before \(`ghcr.io/zama-ai/concretefhe:vX.Y.Z`\) (or `vX.Y.Z-rc?`) for the uploaded docker image. Mark release as pre-release for an `rc` version. See template below:
 
 This is the release markdown template you should copy and update:
 ```
@@ -31,13 +31,13 @@ This is the release markdown template you should copy and update:
 ```
 
 To continue the release cycle:
-- [ ] Choose the version number for next release, e.g. `vA.B.C` (can be `vA.B.Crc?` for Release Candidates) following semantic versioning: https://semver.org/
-- [ ] Update the project version to `A.B.C` (or `A.B.Crc?`) by running:
+- [ ] Choose the version number for next release, e.g. `vA.B.C` (can be `vA.B.C-rc?` for Release Candidates) following semantic versioning: https://semver.org/
+- [ ] Update the project version to `A.B.C` (or `A.B.C-rc?`) by running:
 
 ```bash
 VERSION=A.B.C make set_version
 # or
-VERSION=A.B.Crc? make set_version
+VERSION=A.B.C-rc? make set_version
 ```
 
 All done!

.github/workflows/continuous-integration.yaml

@@ -460,7 +460,8 @@ jobs:
 
           # We want the space separated list of versions to be expanded
           # shellcheck disable=SC2086
-          REQUIRES_LATEST_TAG=$(python script/actions_utils/version_comparison.py \
+          REQUIRES_LATEST_TAG=$(python script/make_utils/version_utils.py \
+          islatest \
           --new-version "${GIT_TAG}" \
           --existing-versions $EXISTING_TAGS)
 

Makefile

@@ -222,17 +222,9 @@ set_version:
 		echo "VERSION env variable is empty. Please set to desired version.";	\
 		exit 1;																	\
 	fi;
-	./script/make_utils/set_version.sh --version "$${VERSION}" --src-dir "$(SRC_DIR)"
+	poetry run python ./script/make_utils/version_utils.py set-version --version "$${VERSION}"
 .PHONY: set_version
 
 check_version_coherence:
-	@SRC_VER=$$(poetry run python -c "from $(SRC_DIR) import __version__; print(__version__);");\
-	PROJECT_VER=($$(poetry version));															\
-	PROJECT_VER="$${PROJECT_VER[1]}";															\
-	echo "Source version:  $${SRC_VER}";														\
-	echo "Project version: $${PROJECT_VER}";													\
-	if [[ "$${SRC_VER}" != "$${PROJECT_VER}" ]]; then											\
-		echo "Version mismatch between source and pyproject.toml re-run make set_version";		\
-		exit 1;																					\
-	fi
+	poetry run python ./script/make_utils/version_utils.py check-version
 .PHONY: check_version_coherence

concrete/version.py

@@ -1,4 +1,4 @@
 """Package version module."""
 # Auto-generated by "make set_version" do not modify
 
-__version__ = "0.2.0rc1"
+__version__ = "0.2.0-rc1"

docs/conf.py

@@ -22,7 +22,7 @@ copyright = "2021, Zama"
 author = "Zama"
 
 # The full version, including alpha/beta/rc tags
-release = "0.1"
+release = "0.2.0-rc1"
 
 
 # -- General configuration ---------------------------------------------------

poetry.lock

@@ -1901,7 +1901,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes
 [metadata]
 lock-version = "1.1"
 python-versions = ">=3.8,<3.9"
-content-hash = "1c69a2c569fdacbcf43e47d88dc2d93ff098c50395f94c93b05428202f047cb6"
+content-hash = "783e41a9b79babbea7b986de2d4b901e2472e202613952e9881ecf340ff69d18"
 
 [metadata.files]
 alabaster = [

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "concretefhe"
-version = "0.2.0rc1"
+version = "0.2.0-rc1"
 description = "Concrete Framework"
 authors = ["Zama <hello@zama.ai>"]
 packages = [
@@ -40,6 +40,7 @@ sphinx-copybutton = "^0.4.0"
 nbmake = "^0.9"
 python-semantic-release = "^7.19.2"
 semver = "^2.13.0"
+tomlkit = "^0.7.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

script/actions_utils/version_comparison.py

@@ -1,51 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""Helper script for github actions to compare versions"""
-import argparse
-import re
-import sys
-
-
-def main(args):
-    """Entry point"""
-    print(args, file=sys.stderr)
-    semver_matcher = re.compile(r"^(v)?([\d.]+)(rc\d+)?$")
-    # Keep versions that are not release candidate
-    all_versions = [
-        tuple(map(int, match.group(2).split(".")))
-        for version in args.existing_versions
-        if (match := semver_matcher.match(version)) is not None and match.group(3) is None
-    ]
-
-    nv_match = semver_matcher.match(args.new_version)
-    new_version = (
-        tuple(map(int, nv_match.group(2).split(".")))
-        if nv_match is not None and nv_match.group(3) is None
-        else None
-    )
-
-    all_versions.append(new_version)
-
-    nv_is_rc = new_version is None
-    nv_is_latest = not nv_is_rc and max(all_versions) == new_version
-    print(str(nv_is_latest).lower())
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        "Compare new version to previous versions and determine if it's the latest",
-        allow_abbrev=False,
-    )
-
-    parser.add_argument("--new-version", type=str, required=True, help="The new version to compare")
-    parser.add_argument(
-        "--existing-versions",
-        type=str,
-        nargs="+",
-        required=True,
-        help="The list of existing versions",
-    )
-
-    cli_args = parser.parse_args()
-
-    main(cli_args)

script/make_utils/set_version.sh

@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-
-VERSION_TO_SET=
-SRC_DIR=
-
-while [ -n "$1" ]
-do
-   case "$1" in
-        "--version" )
-            shift
-            VERSION_TO_SET="$1"
-            ;;
-
-        "--src-dir" )
-            shift
-            SRC_DIR="$1"
-            ;;
-
-        *)
-            echo "Unknown param : $1"
-            exit 1
-            ;;
-   esac
-   shift
-done
-
-if [[ "${VERSION_TO_SET}" == "" ]]; then
-    echo "--version is required. Aborting"
-    exit 1
-fi
-
-if [[ "${SRC_DIR}" == "" ]]; then
-    echo "--src-dir is required. Aborting"
-    exit 1
-fi
-
-rx='^(v)?([0-9]+\.){2}[0-9]+(rc[0-9]+)?$'
-
-if [[ ! "${VERSION_TO_SET}" =~ $rx ]]; then
-    echo "ERROR: Unable to validate version: '${VERSION_TO_SET}'"
-    exit 1
-fi
-
-echo "INFO: Version ${VERSION_TO_SET}"
-
-VERSION_TO_SET="${VERSION_TO_SET/v/}"
-echo "${VERSION_TO_SET}"
-
-poetry version "${VERSION_TO_SET}"
-
-VERSION_FILE="${SRC_DIR}/version.py"
-
-rm "${VERSION_FILE}"
-
-{
-    echo '"""Package version module."""'
-    echo '# Auto-generated by "make set_version" do not modify'
-    echo ''
-    echo "__version__ = \"${VERSION_TO_SET}\""
-} >> "${VERSION_FILE}"

script/make_utils/version_utils.py

@@ -0,0 +1,266 @@
+"""Tool to manage version in the project"""
+
+import argparse
+import os
+import re
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+import tomlkit
+from semver import VersionInfo
+
+
+def strip_leading_v(version_str: str):
+    """Strip leading v of a version which is not SemVer compatible."""
+    if version_str and version_str[0] == "v":
+        return version_str[1:]
+    return version_str
+
+
+def islatest(args):
+    """islatest command entry point."""
+    print(args, file=sys.stderr)
+
+    new_version_is_latest = False
+
+    new_version_str = strip_leading_v(args.new_version)
+    if VersionInfo.isvalid(new_version_str):
+        new_version_info = VersionInfo.parse(new_version_str)
+        if new_version_info.prerelease is None:
+            # If it's an actual release
+            all_versions_str = (
+                strip_leading_v(version_str) for version_str in args.existing_versions
+            )
+
+            # Keep versions that are not release candidate
+            all_non_prerelease_version_infos = [
+                version_info
+                for version_str in all_versions_str
+                if VersionInfo.isvalid(version_str)
+                and (version_info := VersionInfo.parse(version_str))
+                and version_info.prerelease is None
+            ]
+
+            all_non_prerelease_version_infos.append(new_version_info)
+
+            new_version_is_latest = max(all_non_prerelease_version_infos) == new_version_info
+    print(str(new_version_is_latest).lower())
+
+
+def update_variable_in_py_file(file_path: Path, var_name: str, version_str: str):
+    """Update the version in a .py file."""
+
+    file_content = None
+    with open(file_path, encoding="utf-8") as f:
+        file_content = f.read()
+
+    updated_file_content = re.sub(
+        rf'{var_name} *[:=] *["\'](.+)["\']',
+        rf'{var_name} = "{version_str}"',
+        file_content,
+    )
+
+    with open(file_path, "w", encoding="utf-8", newline="\n") as f:
+        f.write(updated_file_content)
+
+
+def update_variable_in_toml_file(file_path: Path, var_name: str, version_str: str):
+    """Update the version in a .toml file."""
+    toml_content = None
+    with open(file_path, encoding="utf-8") as f:
+        toml_content = tomlkit.loads(f.read())
+
+    toml_keys = var_name.split(".")
+    current_content = toml_content
+    for toml_key in toml_keys[:-1]:
+        current_content = current_content[toml_key]
+    last_toml_key = toml_keys[-1]
+    current_content[last_toml_key] = version_str
+
+    with open(file_path, "w", encoding="utf-8", newline="\n") as f:
+        f.write(tomlkit.dumps(toml_content))
+
+
+def load_file_vars_set(pyproject_path: os.PathLike, cli_file_vars: Optional[List[str]]):
+    """Load files and their version variables set-up in pyproject.toml and passed as arguments."""
+
+    file_vars_set = set()
+    if cli_file_vars is not None:
+        file_vars_set.update(cli_file_vars)
+
+    pyproject_path = Path(pyproject_path).resolve()
+
+    # Check if there is a semantic release configuration
+    if pyproject_path.exists():
+        pyproject_content = None
+        with open(pyproject_path, encoding="utf-8") as f:
+            pyproject_content = tomlkit.loads(f.read())
+
+        try:
+            sr_conf = pyproject_content["tool"]["semantic_release"]
+            sr_version_toml: str = sr_conf.get("version_toml", "")
+            file_vars_set.update(sr_version_toml.split(","))
+            sr_version_variable: str = sr_conf.get("version_variable", "")
+            file_vars_set.update(sr_version_variable.split(","))
+        except KeyError:
+            print("No configuration for semantic release in pyproject.toml")
+
+    return file_vars_set
+
+
+def set_version(args):
+    """set-version command entry point."""
+
+    version_str = strip_leading_v(args.version)
+    if not VersionInfo.isvalid(version_str):
+        raise RuntimeError(f"Unable to validate version: {args.version}")
+
+    file_vars_set = load_file_vars_set(args.pyproject_file, args.file_vars)
+
+    for file_var_str in sorted(file_vars_set):
+        print(f"Processing {file_var_str}")
+        file, var_name = file_var_str.split(":", 1)
+        file_path = Path(file).resolve()
+
+        if file_path.suffix == ".py":
+            update_variable_in_py_file(file_path, var_name, version_str)
+        elif file_path.suffix == ".toml":
+            update_variable_in_toml_file(file_path, var_name, version_str)
+        else:
+            raise RuntimeError(f"Unsupported file extension: {file_path.suffix}")
+
+
+def get_variable_from_py_file(file_path: Path, var_name: str):
+    """Read variable value from a .py file."""
+    file_content = None
+    with open(file_path, encoding="utf-8") as f:
+        file_content = f.read()
+
+    variable_values_set = set()
+
+    start_pos = 0
+    while True:
+        file_content = file_content[start_pos:]
+        match = re.search(
+            rf'{var_name} *[:=] *["\'](.+)["\']',
+            file_content,
+        )
+        if match is None:
+            break
+
+        variable_values_set.add(match.group(1))
+        start_pos = match.end()
+
+    return variable_values_set
+
+
+def get_variable_from_toml_file(file_path: Path, var_name: str):
+    """Read variable value from a .toml file."""
+
+    toml_content = None
+    with open(file_path, encoding="utf-8") as f:
+        toml_content = tomlkit.loads(f.read())
+
+    toml_keys = var_name.split(".")
+    current_content = toml_content
+    for toml_key in toml_keys:
+        current_content = current_content[toml_key]
+
+    return current_content
+
+
+def check_version(args):
+    """check-version command entry point."""
+
+    version_str_set = set()
+
+    file_vars_set = load_file_vars_set(args.pyproject_file, args.file_vars)
+
+    for file_var_str in sorted(file_vars_set):
+        print(f"Processing {file_var_str}")
+        file, var_name = file_var_str.split(":", 1)
+        file_path = Path(file).resolve()
+
+        if file_path.suffix == ".py":
+            version_str_set.update(get_variable_from_py_file(file_path, var_name))
+        elif file_path.suffix == ".toml":
+            version_str_set.add(get_variable_from_toml_file(file_path, var_name))
+        else:
+            raise RuntimeError(f"Unsupported file extension: {file_path.suffix}")
+
+    if len(version_str_set) == 0:
+        raise RuntimeError(f"No versions found in {', '.join(sorted(file_vars_set))}")
+    if len(version_str_set) > 1:
+        raise RuntimeError(
+            f"Found more than one version: {', '.join(sorted(version_str_set))}\n"
+            "Re-run make set-version"
+        )
+    # Now version_str_set len == 1
+    if not VersionInfo.isvalid((version := next(iter(version_str_set)))):
+        raise RuntimeError(f"Unable to validate version: {version}")
+
+    print(f"Found version {version} in all processed locations.")
+
+
+def main(args):
+    """Entry point"""
+    args.entry_point(args)
+
+
+if __name__ == "__main__":
+    main_parser = argparse.ArgumentParser("Version utils", allow_abbrev=False)
+
+    sub_parsers = main_parser.add_subparsers(dest="sub-command", required=True)
+
+    parser_islatest = sub_parsers.add_parser("islatest")
+    parser_islatest.add_argument(
+        "--new-version", type=str, required=True, help="The new version to compare"
+    )
+    parser_islatest.add_argument(
+        "--existing-versions",
+        type=str,
+        nargs="+",
+        required=True,
+        help="The list of existing versions",
+    )
+    parser_islatest.set_defaults(entry_point=islatest)
+
+    parser_set_version = sub_parsers.add_parser("set-version")
+    parser_set_version.add_argument("--version", type=str, required=True, help="The version to set")
+    parser_set_version.add_argument(
+        "--pyproject-file",
+        type=str,
+        default="pyproject.toml",
+        help="The path to a project's pyproject.toml file, defaults to $pwd/pyproject.toml",
+    )
+    parser_set_version.add_argument(
+        "--file-vars",
+        type=str,
+        nargs="+",
+        help=(
+            "A space separated list of file/path.{py, toml}:variable to update with the new version"
+        ),
+    )
+    parser_set_version.set_defaults(entry_point=set_version)
+
+    parser_check_version = sub_parsers.add_parser("check-version")
+    parser_check_version.add_argument(
+        "--pyproject-file",
+        type=str,
+        default="pyproject.toml",
+        help="The path to a project's pyproject.toml file, defaults to $pwd/pyproject.toml",
+    )
+    parser_check_version.add_argument(
+        "--file-vars",
+        type=str,
+        nargs="+",
+        help=(
+            "A space separated list of file/path.{py, toml}:variable to update with the new version"
+        ),
+    )
+    parser_check_version.set_defaults(entry_point=check_version)
+
+    cli_args = main_parser.parse_args()
+
+    main(cli_args)