#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-or-later # SPDX-FileCopyrightText: 2011-2024 Blender Authors # import argparse import os import pathlib import sys from collections import OrderedDict sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) import worker.configure import worker.utils import worker.blender import worker.blender.compile import worker.blender.update import worker.blender.version class DocApiBuilder(worker.blender.CodeBuilder): def __init__(self, args: argparse.Namespace): super().__init__(args) self.needs_package_delivery = args.needs_package_delivery self.setup_track_path() def download_api_dump_test_data(local_delivery_path: pathlib.Path) -> None: import urllib.request import json api_base_url = "https://docs.blender.org/api" api_dump_index_url = f"{api_base_url}/api_dump_index.json" request = urllib.request.Request( api_dump_index_url, headers={"User-Agent": "Mozilla"} ) response = urllib.request.urlopen(request, timeout=5.0) api_dump_index_text = response.read().decode("utf-8", "ignore") api_dump_index_path = local_delivery_path / "api_dump_index.json" os.makedirs(api_dump_index_path.parent, exist_ok=True) api_dump_index_path.write_text(api_dump_index_text) api_dump_index = json.loads(api_dump_index_text) for version in api_dump_index.keys(): api_dump_url = f"{api_base_url}/{version}/api_dump.json" worker.utils.info(f"Download {api_dump_url}") request = urllib.request.Request( api_dump_url, headers={"User-Agent": "Mozilla"} ) response = urllib.request.urlopen(request, timeout=5.0) api_dump_text = response.read().decode("utf-8", "ignore") api_dump_path = local_delivery_path / version / "api_dump.json" os.makedirs(api_dump_path.parent, exist_ok=True) api_dump_path.write_text(api_dump_text) def compile_doc(builder: DocApiBuilder) -> None: # Install requirements os.chdir(builder.track_path) doc_api_script_path = builder.code_path / "doc" / "python_api" worker.utils.call_pipenv( ["install", "--requirements", doc_api_script_path / "requirements.txt"] ) # Clean build directory worker.utils.remove_dir(builder.build_doc_path) os.makedirs(builder.build_doc_path, exist_ok=True) os.chdir(doc_api_script_path) # Get API dumps data from server. api_dump_build_path = builder.build_doc_path / "api_dump" os.makedirs(api_dump_build_path, exist_ok=True) api_dump_include_paths = ["api_dump_index.json", "*/", "api_dump.json"] api_dump_build_path_index = api_dump_build_path / "api_dump_index.json" worker_config = builder.get_worker_config() connect_id = f"{worker_config.docs_user}@{worker_config.docs_machine}" remote_path = ( pathlib.Path(worker_config.docs_folder) / "docs.blender.org" / "htdocs" / builder.service_env_id / "api" ) # Get data from docs.blender.org for local testing. if builder.service_env_id == "LOCAL": worker.utils.info("Downloading API dump data from docs.blender.org for testing") download_api_dump_test_data(remote_path) source_path = f"{connect_id}:{remote_path}/" dest_path = api_dump_build_path worker.utils.rsync( source_path, dest_path, include_paths=api_dump_include_paths, exclude_paths=["*"], ) version = worker.blender.version.VersionInfo(builder).short_version api_dump_build_path_current_version = api_dump_build_path / version os.makedirs(api_dump_build_path_current_version, exist_ok=True) # Generate API docs cmd = [ builder.blender_command_path(), "--background", "--factory-startup", "-noaudio", "--python", doc_api_script_path / "sphinx_doc_gen.py", "--", "--output", builder.build_doc_path, "--api-changelog-generate", "--api-dump-index-path", api_dump_build_path_index, ] worker.utils.call(cmd) num_threads = worker.configure.get_thread_count(thread_memory_in_GB=1.25) in_path = builder.build_doc_path / "sphinx-in" out_path = builder.build_doc_path / "sphinx-out-html" worker.utils.call( ["sphinx-build", "-b", "html", "-j", str(num_threads), in_path, out_path] ) def package(builder: DocApiBuilder) -> None: os.chdir(builder.build_doc_path) version = worker.blender.version.VersionInfo(builder).short_version version_file_label = version.replace(".", "_") package_name = f"blender_python_reference_{version_file_label}" package_file_name = f"{package_name}.zip" cmd = ["7z", "a", "-tzip", package_file_name, "./sphinx-out-html", "-r"] worker.utils.call(cmd) cmd = ["7z", "rn", package_file_name, "sphinx-out-html", package_name] worker.utils.call(cmd) def deliver(builder: DocApiBuilder) -> None: # Get versions branches_config = builder.get_branches_config() version = worker.blender.version.VersionInfo(builder).short_version dev_version = branches_config.track_major_minor_versions["vdev"] latest_version = branches_config.doc_stable_major_minor_version # Get remote path worker_config = builder.get_worker_config() connect_id = f"{worker_config.docs_user}@{worker_config.docs_machine}" remote_path = ( pathlib.Path(worker_config.docs_folder) / "docs.blender.org" / "htdocs" / builder.service_env_id / "api" ) version_remote_path = remote_path / version worker.utils.call_ssh(connect_id, ["mkdir", "-p", version_remote_path]) change_modes = ["D0755", "F0644"] # Sync HTML files source_path = f"{builder.build_doc_path}/sphinx-out-html/" dest_path = f"{connect_id}:{version_remote_path}/" worker.utils.rsync( source_path, dest_path, exclude_paths=[".doctrees"], change_modes=change_modes ) # Put API dumps data on the server. api_dump_build_path = f"{builder.build_doc_path}/api_dump/" api_dump_dest_path = f"{connect_id}:{remote_path}/" worker.utils.rsync( api_dump_build_path, api_dump_dest_path, change_modes=change_modes ) # Sync zip package if builder.needs_package_delivery: version_file_label = version.replace(".", "_") package_name = f"blender_python_reference_{version_file_label}" package_file_name = f"{package_name}.zip" source_file_path = builder.build_doc_path / package_file_name dest_file_path = f"{connect_id}:{version_remote_path}/{package_file_name}" worker.utils.rsync( source_file_path, dest_file_path, exclude_paths=[".doctrees"], change_modes=change_modes, ) # Create links if builder.track_id == "vdev": worker.utils.call_ssh( connect_id, ["ln", "-svF", remote_path / dev_version, remote_path / "dev"] ) worker.utils.call_ssh( connect_id, ["ln", "-svF", remote_path / dev_version, remote_path / "master"], ) worker.utils.call_ssh( connect_id, ["ln", "-svF", remote_path / dev_version, remote_path / "main"] ) worker.utils.call_ssh( connect_id, ["ln", "-svF", remote_path / latest_version, remote_path / "latest"], ) worker.utils.call_ssh( connect_id, ["ln", "-svF", remote_path / latest_version, remote_path / "current"], ) if __name__ == "__main__": steps: worker.utils.BuilderSteps = OrderedDict() steps["configure-machine"] = worker.configure.configure_machine steps["update-code"] = worker.blender.update.update steps["compile-code"] = worker.blender.compile.compile_code steps["compile-install"] = worker.blender.compile.compile_install steps["compile"] = compile_doc steps["package"] = package steps["deliver"] = deliver steps["clean"] = worker.blender.CodeBuilder.clean parser = worker.blender.create_argument_parser(steps=steps) parser.add_argument("--needs-package-delivery", action="store_true", required=False) args = parser.parse_args() builder = DocApiBuilder(args) builder.run(args.step, steps)