docs(ci): manage versions.html

- create generate_versions_html.py
- update workflow to be able to push pre-releases on preprod server

closes #738

.github/workflows/continuous-integration.yaml

@@ -598,6 +598,49 @@ jobs:
         with:
           name: changelog
           path: ${{ env.ARTIFACTS_RAW_DIR }}/changelog/
+      - name: Prepare docs push
+        id: docs-push-infos
+        run: |
+          if [[ ${{ secrets.AWS_REPO_PREPROD_DOCUMENTATION_BUCKET_NAME }} != "" ]] && \
+             [[ ${{ secrets.AWS_REPO_PREPROD_DOCUMENTATION_DISTRIBUTION_ID }} != "" ]] && \
+             [[ "${IS_PRERELEASE}" == "true" ]]; then
+            echo "::set-output name=aws-bucket::${{ secrets.AWS_REPO_PREPROD_DOCUMENTATION_BUCKET_NAME }}"
+            echo "::set-output name=aws-distribution::${{ secrets.AWS_REPO_PREPROD_DOCUMENTATION_DISTRIBUTION_ID }}"
+          else
+            echo "::set-output name=aws-bucket::${{ secrets.AWS_REPO_DOCUMENTATION_BUCKET_NAME }}"
+            echo "::set-output name=aws-distribution::${{ secrets.AWS_REPO_DOCUMENTATION_DISTRIBUTION_ID }}"
+          fi
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@0d9a5be0dceea74e09396820e1e522ba4a110d2f
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Update versions.html for docs
+        if: ${{ success() && !cancelled() }}
+        run: |
+          DOWNLOADED_HTML_FILE=$(mktemp --suffix=.html)
+          OUTPUT_HTML_FILE=$(mktemp --suffix=.html)
+
+          aws s3api get-object \
+          --bucket ${{ steps.docs-push-infos.outputs.aws-bucket }} \
+          --key concretefhe/versions.html "${DOWNLOADED_HTML_FILE}"
+
+          python3 ./script/actions_utils/generate_versions_html.py \
+          --add-versions "${PROJECT_VERSION}" \
+          --versions-html-file "${DOWNLOADED_HTML_FILE}" \
+          --output-html "${OUTPUT_HTML_FILE}"
+
+          aws s3 cp "${OUTPUT_HTML_FILE}" \
+          s3://${{ steps.docs-push-infos.outputs.aws-bucket }}/concretefhe/versions.html \
+          --acl public-read
+
+          aws cloudfront create-invalidation \
+          --distribution-id ${{ steps.docs-push-infos.outputs.aws-distribution }} \
+          --paths /concretefhe/versions.html
+
+          # Copy to docs to keep a version in docs artifacts
+          cp "${OUTPUT_HTML_FILE}" "${RAW_DOCS_DIR}"/versions.html
       - name: Create ready to upload/packaged artifacts and release body
         if: ${{ success() && !cancelled() }}
         env:
@@ -607,6 +650,8 @@ jobs:
           pushd "${RAW_DOCS_DIR}"
           zip -r "${ARTIFACTS_PACKAGED_DIR}/html-docs.zip" ./*
           tar -cvzf "${ARTIFACTS_PACKAGED_DIR}/html-docs.tar.gz" ./*
+          # Remove the versions.html to avoid pushing it to S3 but have it in release artifacts
+          rm versions.html
           popd
           cp "${RAW_CHANGELOG_DIR}"/* "${ARTIFACTS_PACKAGED_DIR}"
           ls -a "${ARTIFACTS_PACKAGED_DIR}"
@@ -615,26 +660,20 @@ jobs:
           echo "RELEASE_BODY_FILE=${RELEASE_BODY_FILE}" >> "$GITHUB_ENV"
 
           cp ./script/actions_utils/RELEASE_TEMPLATE.md "${RELEASE_BODY_FILE}"
-          echo "Docker Image: ${RELEASE_IMG_GIT_TAG}" >> "${RELEASE_BODY_FILE}"
-          if [[ "${IS_PRERELEASE}" == "false" ]]; then
-            echo "Documentation: https://docs.zama.ai/concretefhe/${PROJECT_VERSION}" >> "${RELEASE_BODY_FILE}"
-          fi
-          echo "" >> "${RELEASE_BODY_FILE}"
+          {
+            echo "Docker Image: ${RELEASE_IMG_GIT_TAG}";
+            echo "Documentation: https://${{ steps.docs-push-infos.outputs.aws-bucket }}/concretefhe/${PROJECT_VERSION}";
+            echo "";
+          } >> "${RELEASE_BODY_FILE}"
           cat "${RAW_CHANGELOG_DIR}"/* >> "${RELEASE_BODY_FILE}"
       - name: Push release docker image
         if: ${{ success() && !cancelled() }}
         run: |
           docker image push --all-tags "${RELEASE_IMAGE_BASE}"
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@0d9a5be0dceea74e09396820e1e522ba4a110d2f
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ secrets.AWS_REGION }}
       - name: Push release documentation
-        if: ${{ success() && !cancelled() && !fromJSON(env.IS_PRERELEASE) }}
+        if: ${{ success() && !cancelled() }}
         env:
-          AWS_S3_BUCKET: ${{ secrets.AWS_REPO_DOCUMENTATION_BUCKET_NAME }}
+          AWS_S3_BUCKET: ${{ steps.docs-push-infos.outputs.aws-bucket }}
           SOURCE_DIR: ${{ steps.download-docs.outputs.download-path }}
           DEST_DIR: 'concretefhe/${{ env.PROJECT_VERSION }}'
         run: |
@@ -643,7 +682,7 @@ jobs:
       - name: Push release documentation as stable
         if: ${{ success() && !cancelled() && !fromJSON(env.IS_PRERELEASE) && fromJSON(env.IS_LATEST) }}
         env:
-          AWS_S3_BUCKET: ${{ secrets.AWS_REPO_DOCUMENTATION_BUCKET_NAME }}
+          AWS_S3_BUCKET: ${{ steps.docs-push-infos.outputs.aws-bucket }}
           SOURCE_DIR: ${{ steps.download-docs.outputs.download-path }}
           DEST_DIR: 'concretefhe/stable'
         run: |
@@ -652,7 +691,7 @@ jobs:
         if: ${{ success() && !fromJSON(env.IS_PRERELEASE) && fromJSON(env.IS_LATEST) }}
         env:
           SOURCE_PATH: "/concretefhe/stable/*"
-          DISTRIBUTION_ID: ${{ secrets.AWS_REPO_DOCUMENTATION_DISTRIBUTION_ID }}
+          DISTRIBUTION_ID: ${{ steps.docs-push-infos.outputs.aws-distribution }}
         run: |
           aws cloudfront create-invalidation \
           --distribution-id "${DISTRIBUTION_ID}" \

docs/_templates/versions.html

@@ -0,0 +1,28 @@
+<html>
+ <head>
+  <meta charset="utf-8"/>
+  <link href="stable/_static/css/zama.css" rel="stylesheet" type="text/css"/>
+  <link href="_static/favicon.ico" rel="icon"/>
+ </head>
+ <body>
+  <div class="wy-nav-content">
+   <div class="rst-content">
+    <h1>
+     ConcreteFHE Documentation
+    </h1>
+    <div class="toctree-wrapper compound">
+     <p class="caption">
+      <span class="caption-text">
+       Pick a version
+      </span>
+     </p>
+     <ul id="versions-list">
+      <li class="toctree-l1">
+       <a class="reference internal" href="stable/">stable</a>
+      </li>
+     </ul>
+    </div>
+   </div>
+  </div>
+ </body>
+</html>

docs/versions.html

@@ -1,23 +0,0 @@
-
-<html>
-  <head>
-    <meta charset="utf-8">
-    <link rel="stylesheet" href="stable/_static/css/zama.css" type="text/css" />
-    <link rel="icon" href="_static/favicon.ico" />
-  </head>
-  <body>
-    <div class="wy-nav-content">
-      <div class="rst-content">
-      <h1> ConcreteFHE Documentation </h1>
-        <div class="toctree-wrapper compound">
-          <p class="caption"><span class="caption-text">Pick a version</span></p>
-          <ul>
-            <li class="toctree-l1">
-              <a class="reference internal" href="stable/">stable</a>
-            </li>
-          </ul>
-        </div>
-      </div>
-    </div>
-  </body>
-</html>

script/actions_utils/generate_versions_html.py

@@ -0,0 +1,160 @@
+"""Tool to manage the versions.html file at the root of our docs sites."""
+
+import argparse
+from pathlib import Path
+
+from bs4 import BeautifulSoup
+from bs4.element import Tag
+from semver import VersionInfo
+
+VERSIONS_LIST_ID = "versions-list"
+
+
+def strip_leading_v(version_str: str):
+    """Strip leading v of a version which is not SemVer compatible."""
+    return version_str[1:] if version_str.startswith("v") else version_str
+
+
+def create_list_element(soup: BeautifulSoup, contents: Tag) -> Tag:
+    """Create a list element for links.
+
+    Args:
+        soup (BeautifulSoup): The soup to use to create the tag.
+
+    Returns:
+        Tag: tag containing <li class="toctree-l1"></li>.
+    """
+    new_list_element = soup.new_tag("li", **{"class": "toctree-l1"})
+    new_list_element.contents.append(contents)
+    return new_list_element
+
+
+def create_link_tag_set_string(soup: BeautifulSoup, version_string: str) -> Tag:
+    """Create a link tag on the given soup to version specified by version_string.
+
+    Args:
+        soup (BeautifulSoup): The soup to use to create the tag.
+        version_string (str): The version string to use.
+
+    Returns:
+        Tag: tag containing <a class="reference internal" href="0.1.0/">{version_string}</a>.
+    """
+    new_tag = soup.new_tag(
+        "a",
+        **{
+            "href": f"{version_string}/",
+            "class": "reference internal",
+        },
+    )
+
+    new_tag.string = version_string
+    return new_tag
+
+
+def main(args):
+    """Entry point."""
+
+    invalid_versions = [
+        version
+        for version in args.add_versions
+        if not VersionInfo.isvalid(strip_leading_v(version))
+    ]
+    if len(invalid_versions) > 0:
+        raise RuntimeError(f"Found invalid versions: {invalid_versions}")
+
+    version_html = None
+    version_html_file_path = Path(args.versions_html_file).resolve()
+    with open(version_html_file_path, "r", encoding="utf-8") as f:
+        version_html = BeautifulSoup(f, "html.parser")
+
+    if version_html is None:
+        raise RuntimeError(f"An error occured while trying to load {str(version_html_file_path)}")
+
+    print(version_html)
+
+    version_list = version_html.find(id=VERSIONS_LIST_ID)
+    if version_list is None or version_list.name != "ul":
+        raise RuntimeError(f"Could not find <ul> tag with id {VERSIONS_LIST_ID}")
+
+    non_semver_versions = {}
+    semver_versions = {}
+    for list_entry in version_list.find_all("li"):
+        version_tags = []
+        version_is_valid_semver = False
+        for potential_version_tag in list_entry.contents:
+            if not isinstance(potential_version_tag, Tag):
+                continue
+            version_is_valid_semver = VersionInfo.isvalid(
+                strip_leading_v(potential_version_tag.string)
+            )
+            version_tags.append(potential_version_tag.string)
+
+        assert (
+            num_version_tags := len(version_tags) == 1
+        ), f"Can only have 1 version tag, got {num_version_tags}"
+
+        version_tag = version_tags[0]
+
+        if version_is_valid_semver:
+            semver_versions[version_tag.string] = list_entry
+        else:
+            non_semver_versions[version_tag.string] = list_entry
+
+    parsed_versions = [VersionInfo.parse(version) for version in args.add_versions]
+
+    versions_already_in_html = set(parsed_versions).intersection(semver_versions.keys())
+    if len(versions_already_in_html) > 0:
+        raise RuntimeError(
+            "The following versions are already in the html: "
+            f"{', '.join(str(ver) for ver in sorted(versions_already_in_html))}"
+        )
+
+    semver_versions.update(
+        (
+            parsed_version,
+            create_list_element(
+                version_html, create_link_tag_set_string(version_html, str(parsed_version))
+            ),
+        )
+        for parsed_version in parsed_versions
+    )
+
+    version_list.contents = []
+    for sorted_non_semver_version in sorted(non_semver_versions.keys()):
+        version_list.contents.append(non_semver_versions[sorted_non_semver_version])
+
+    # We want the most recent versions at the top
+    for sorted_semver_version in sorted(semver_versions.keys(), reverse=True):
+        version_list.contents.append(semver_versions[sorted_semver_version])
+
+    pretty_output = version_html.prettify()
+    print(pretty_output)
+
+    output_html_path = Path(args.output_html).resolve()
+    with open(output_html_path, "w", encoding="utf-8") as f:
+        f.write(pretty_output)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("versions.html generator", allow_abbrev=False)
+
+    parser.add_argument(
+        "--add-versions",
+        type=str,
+        required=True,
+        nargs="+",
+        help="A list of versions to add to versions.html. "
+        "The links will be sorted by versions with stable/main as the first entry. "
+        "The link will point to '$VERSION/' and will have text '$VERSION'.",
+    )
+    parser.add_argument(
+        "--versions-html-file",
+        type=str,
+        required=True,
+        help="Path to the versions.html to update. "
+        'It must have a <li> tag with id="versions-list".',
+    )
+    parser.add_argument("--output-html", type=str, required=True, help="Output file path.")
+
+    cli_args = parser.parse_args()
+    main(cli_args)