#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-or-later # SPDX-FileCopyrightText: 2011-2024 Blender Authors # import argparse import json import os import pathlib import re import sys import time from collections import OrderedDict from datetime import timedelta from typing import Optional, Sequence sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) import worker.configure import worker.utils class ManualBuilder(worker.utils.Builder): def __init__(self, args: argparse.Namespace): super().__init__(args, "blender", "blender-manual") self.needs_all_locales = args.needs_all_locales self.needs_package_delivery = args.needs_package_delivery self.doc_format = args.doc_format self.build_path = self.track_path / "build" self.setup_track_path() def get_locales(self) -> Sequence[str]: locales = ["en"] if self.needs_all_locales: locale_path = self.code_path / "locale" locales += [ item.name for item in locale_path.iterdir() if not item.name.startswith(".") ] return locales def update(builder: ManualBuilder) -> None: builder.update_source() if builder.needs_all_locales: worker.utils.update_source( "blender", "blender-manual-translations", builder.code_path / "locale" ) def check(builder: ManualBuilder) -> None: os.chdir(builder.track_path) worker.utils.call_pipenv( ["install", "--pre", "--requirements", builder.code_path / "requirements.txt"] ) os.chdir(builder.code_path) make_cmd = "make.bat" if builder.platform == "windows" else "make" worker.utils.call_pipenv(["run", make_cmd, "check_structure"]) # worker.utils.call_pipenv(["run", make_cmd, "check_syntax"]) # worker.utils.call_pipenv(["run", make_cmd, "check_spelling"]) def compile_doc(builder: ManualBuilder) -> None: # Install requirements. os.chdir(builder.track_path) worker.utils.call_pipenv( ["install", "--pre", "--requirements", builder.code_path / "requirements.txt"] ) # Determine format and locales locales = builder.get_locales() doc_format = builder.doc_format # Clean build folder worker.utils.remove_dir(builder.build_path) os.makedirs(builder.build_path, exist_ok=True) os.chdir(builder.code_path) branches_config = builder.get_branches_config() # Check manual version matches track. conf_file_path = builder.code_path / "manual" / "conf.py" conf_text = conf_file_path.read_text() match = re.search(r"blender_version\s*=\s*['\"](.*)['\"]", conf_text) expected_version = branches_config.track_major_minor_versions[builder.track_id] found_version = match.groups(0)[0] if match else "nothing" if found_version != expected_version: raise Exception( f"Expected blender_version {expected_version}, but found {found_version} in manual/conf.py" ) def filter_output(line: str) -> Optional[str]: if line.find("WARNING: unknown mimetype for .doctrees") != -1: return None elif line.find("copying images...") != -1: return None return line # Generate manual for locale in locales: start_timestamp = time.time() worker.utils.info(f"Generating {locale} in {doc_format}") num_threads = worker.configure.get_thread_count(thread_memory_in_GB=1.25) os.chdir(builder.code_path) build_output_path = builder.build_path / doc_format / locale worker.utils.call_pipenv( [ "run", "sphinx-build", "-b", doc_format, "-j", str(num_threads), "-D", f"language={locale}", "./manual", build_output_path, ], filter_output=filter_output, ) if doc_format == "epub": if not build_output_path.rglob("*.epub"): raise Exception(f"Expected epub files missing in {build_output_path}") # Hack appropriate versions.json URL into version_switch.js worker.utils.info("Replacing URL in version_switch.js") version_switch_file_path = ( build_output_path / "_static" / "js" / "version_switch.js" ) versions_file_url = ( f"https://docs.blender.org/{builder.service_env_id}/versions.json" ) version_switch_text = version_switch_file_path.read_text() version_switch_text = version_switch_text.replace( "https://docs.blender.org/versions.json", versions_file_url ) version_switch_text = version_switch_text.replace( "https://docs.blender.org/PROD/versions.json", versions_file_url ) version_switch_text = version_switch_text.replace( "https://docs.blender.org/UATEST/versions.json", versions_file_url ) version_switch_file_path.write_text(version_switch_text) time_total = time.time() - start_timestamp time_delta = str(timedelta(seconds=time_total)) worker.utils.info(f"Generated {locale} in {doc_format} in {time_delta}") def package(builder: ManualBuilder) -> None: if not builder.needs_package_delivery: worker.utils.info("No package delivery needed, skipping packaging") return locales = builder.get_locales() doc_format = builder.doc_format os.chdir(builder.build_path) compression_option = "" # "-mx=9" package_file_name = f"blender_manual_{doc_format}.zip" build_package_path = builder.build_path / "package" for locale in locales: package_file_path = build_package_path / locale / package_file_name worker.utils.remove_file(package_file_path) source_path = f"{doc_format}/{locale}" cmd = [ "7z", "a", "-tzip", package_file_path, source_path, "-r", "-xr!.doctrees", compression_option, ] worker.utils.call(cmd) cmd = [ "7z", "rn", package_file_path, source_path, f"blender_manual_{builder.track_id}_{locale}.{doc_format}", ] worker.utils.call(cmd) def deliver(builder: ManualBuilder) -> None: locales = builder.get_locales() doc_format = builder.doc_format # Get versions branches_config = builder.get_branches_config() version = branches_config.track_major_minor_versions[builder.track_id] dev_version = branches_config.track_major_minor_versions["vdev"] latest_version = branches_config.doc_stable_major_minor_version # Get remote paths worker_config = builder.get_worker_config() connect_id = f"{worker_config.docs_user}@{worker_config.docs_machine}" docs_remote_path = ( pathlib.Path(worker_config.docs_folder) / "docs.blender.org" / "htdocs" / builder.service_env_id ) # Sync each locale for locale in locales: worker.utils.info(f"Syncing {locale}") # Create directory remote_path = docs_remote_path / "manual" / locale version_remote_path = remote_path / version worker.utils.call_ssh(connect_id, ["mkdir", "-p", version_remote_path]) if doc_format == "html": # Sync html files source_path = f"{builder.build_path}/{doc_format}/{locale}/" dest_path = f"{connect_id}:{version_remote_path}/" # Exclude packaged download files; these get synced with `needs_package_delivery`. worker.utils.rsync( source_path, dest_path, exclude_paths=[".doctrees", "blender_manual_*.zip"], delete=True, delete_path_check=str(version_remote_path), ) # Create links if builder.track_id == "vdev": worker.utils.info(f"Creating links for {locale}") worker.utils.call_ssh( connect_id, ["ln", "-svF", remote_path / dev_version, remote_path / "dev"], ) worker.utils.call_ssh( connect_id, [ "ln", "-svF", remote_path / latest_version, remote_path / "latest", ], ) if builder.needs_package_delivery: # Sync zip package worker.utils.info(f"Syncing package for {locale}") build_package_path = builder.build_path / "package" package_file_name = f"blender_manual_{doc_format}.zip" source_path = build_package_path / locale / package_file_name dest_path = f"{connect_id}:{version_remote_path}/{package_file_name}" worker.utils.rsync(source_path, dest_path, exclude_paths=[".doctrees"]) # Create and sync versions.json worker.utils.info("Creating and syncing versions.json") doc_version_labels = branches_config.doc_manual_version_labels versions_path = builder.build_path / "versions.json" versions_path.write_text(json.dumps(doc_version_labels, indent=2)) worker.utils.info(versions_path.read_text()) dest_path = f"{connect_id}:{docs_remote_path}/versions.json" worker.utils.rsync(versions_path, dest_path) def clean(builder: ManualBuilder) -> None: worker.utils.remove_dir(builder.build_path) if __name__ == "__main__": steps: worker.utils.BuilderSteps = OrderedDict() steps["configure-machine"] = worker.configure.configure_machine steps["update"] = update steps["check"] = check steps["compile"] = compile_doc steps["package"] = package steps["deliver"] = deliver steps["clean"] = clean parser = worker.utils.create_argument_parser(steps=steps) parser.add_argument("--needs-all-locales", action="store_true", required=False) parser.add_argument("--needs-package-delivery", action="store_true", required=False) parser.add_argument( "--doc-format", default="html", type=str, required=False, choices=["html", "epub"], ) args = parser.parse_args() builder = ManualBuilder(args) builder.run(args.step, steps)