310 lines
10 KiB
Python
Executable file
310 lines
10 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-2.0-or-later
|
|
# SPDX-FileCopyrightText: 2011-2024 Blender Authors
|
|
# <pep8 compliant>
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
import time
|
|
|
|
from collections import OrderedDict
|
|
from datetime import timedelta
|
|
from typing import Optional, Sequence
|
|
|
|
sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent))
|
|
|
|
import worker.configure
|
|
import worker.utils
|
|
|
|
|
|
class ManualBuilder(worker.utils.Builder):
|
|
def __init__(self, args: argparse.Namespace):
|
|
super().__init__(args, "blender", "blender-manual")
|
|
self.needs_all_locales = args.needs_all_locales
|
|
self.needs_package_delivery = args.needs_package_delivery
|
|
self.doc_format = args.doc_format
|
|
self.build_path = self.track_path / "build"
|
|
self.setup_track_path()
|
|
|
|
def get_locales(self) -> Sequence[str]:
|
|
locales = ["en"]
|
|
if self.needs_all_locales:
|
|
locale_path = self.code_path / "locale"
|
|
locales += [
|
|
item.name
|
|
for item in locale_path.iterdir()
|
|
if not item.name.startswith(".")
|
|
]
|
|
return locales
|
|
|
|
|
|
def update(builder: ManualBuilder) -> None:
|
|
builder.update_source()
|
|
if builder.needs_all_locales:
|
|
worker.utils.update_source(
|
|
"blender", "blender-manual-translations", builder.code_path / "locale"
|
|
)
|
|
|
|
|
|
def check(builder: ManualBuilder) -> None:
|
|
os.chdir(builder.track_path)
|
|
worker.utils.call_pipenv(
|
|
["install", "--pre", "--requirements", builder.code_path / "requirements.txt"]
|
|
)
|
|
|
|
os.chdir(builder.code_path)
|
|
|
|
make_cmd = "make.bat" if builder.platform == "windows" else "make"
|
|
worker.utils.call_pipenv(["run", make_cmd, "check_structure"])
|
|
# worker.utils.call_pipenv(["run", make_cmd, "check_syntax"])
|
|
# worker.utils.call_pipenv(["run", make_cmd, "check_spelling"])
|
|
|
|
|
|
def compile_doc(builder: ManualBuilder) -> None:
|
|
# Install requirements.
|
|
os.chdir(builder.track_path)
|
|
worker.utils.call_pipenv(
|
|
["install", "--pre", "--requirements", builder.code_path / "requirements.txt"]
|
|
)
|
|
|
|
# Determine format and locales
|
|
locales = builder.get_locales()
|
|
doc_format = builder.doc_format
|
|
|
|
# Clean build folder
|
|
worker.utils.remove_dir(builder.build_path)
|
|
os.makedirs(builder.build_path, exist_ok=True)
|
|
os.chdir(builder.code_path)
|
|
|
|
branches_config = builder.get_branches_config()
|
|
|
|
# Check manual version matches track.
|
|
conf_file_path = builder.code_path / "manual" / "conf.py"
|
|
conf_text = conf_file_path.read_text()
|
|
match = re.search(r"blender_version\s*=\s*['\"](.*)['\"]", conf_text)
|
|
expected_version = branches_config.track_major_minor_versions[builder.track_id]
|
|
found_version = match.groups(0)[0] if match else "nothing"
|
|
if found_version != expected_version:
|
|
raise Exception(
|
|
f"Expected blender_version {expected_version}, but found {found_version} in manual/conf.py"
|
|
)
|
|
|
|
def filter_output(line: str) -> Optional[str]:
|
|
if line.find("WARNING: unknown mimetype for .doctrees") != -1:
|
|
return None
|
|
elif line.find("copying images...") != -1:
|
|
return None
|
|
return line
|
|
|
|
# Generate manual
|
|
for locale in locales:
|
|
start_timestamp = time.time()
|
|
worker.utils.info(f"Generating {locale} in {doc_format}")
|
|
|
|
num_threads = worker.configure.get_thread_count(thread_memory_in_GB=1.25)
|
|
|
|
os.chdir(builder.code_path)
|
|
build_output_path = builder.build_path / doc_format / locale
|
|
|
|
worker.utils.call_pipenv(
|
|
[
|
|
"run",
|
|
"sphinx-build",
|
|
"-b",
|
|
doc_format,
|
|
"-j",
|
|
str(num_threads),
|
|
"-D",
|
|
f"language={locale}",
|
|
"./manual",
|
|
build_output_path,
|
|
],
|
|
filter_output=filter_output,
|
|
)
|
|
|
|
if doc_format == "epub":
|
|
if not build_output_path.rglob("*.epub"):
|
|
raise Exception(f"Expected epub files missing in {build_output_path}")
|
|
|
|
# Hack appropriate versions.json URL into version_switch.js
|
|
worker.utils.info("Replacing URL in version_switch.js")
|
|
|
|
version_switch_file_path = (
|
|
build_output_path / "_static" / "js" / "version_switch.js"
|
|
)
|
|
versions_file_url = (
|
|
f"https://docs.blender.org/{builder.service_env_id}/versions.json"
|
|
)
|
|
|
|
version_switch_text = version_switch_file_path.read_text()
|
|
version_switch_text = version_switch_text.replace(
|
|
"https://docs.blender.org/versions.json", versions_file_url
|
|
)
|
|
version_switch_text = version_switch_text.replace(
|
|
"https://docs.blender.org/PROD/versions.json", versions_file_url
|
|
)
|
|
version_switch_text = version_switch_text.replace(
|
|
"https://docs.blender.org/UATEST/versions.json", versions_file_url
|
|
)
|
|
version_switch_file_path.write_text(version_switch_text)
|
|
|
|
time_total = time.time() - start_timestamp
|
|
time_delta = str(timedelta(seconds=time_total))
|
|
worker.utils.info(f"Generated {locale} in {doc_format} in {time_delta}")
|
|
|
|
|
|
def package(builder: ManualBuilder) -> None:
|
|
if not builder.needs_package_delivery:
|
|
worker.utils.info("No package delivery needed, skipping packaging")
|
|
return
|
|
|
|
locales = builder.get_locales()
|
|
doc_format = builder.doc_format
|
|
|
|
os.chdir(builder.build_path)
|
|
|
|
compression_option = "" # "-mx=9"
|
|
package_file_name = f"blender_manual_{doc_format}.zip"
|
|
|
|
build_package_path = builder.build_path / "package"
|
|
|
|
for locale in locales:
|
|
package_file_path = build_package_path / locale / package_file_name
|
|
worker.utils.remove_file(package_file_path)
|
|
|
|
source_path = f"{doc_format}/{locale}"
|
|
|
|
cmd = [
|
|
"7z",
|
|
"a",
|
|
"-tzip",
|
|
package_file_path,
|
|
source_path,
|
|
"-r",
|
|
"-xr!.doctrees",
|
|
compression_option,
|
|
]
|
|
worker.utils.call(cmd)
|
|
|
|
cmd = [
|
|
"7z",
|
|
"rn",
|
|
package_file_path,
|
|
source_path,
|
|
f"blender_manual_{builder.track_id}_{locale}.{doc_format}",
|
|
]
|
|
worker.utils.call(cmd)
|
|
|
|
|
|
def deliver(builder: ManualBuilder) -> None:
|
|
locales = builder.get_locales()
|
|
doc_format = builder.doc_format
|
|
|
|
# Get versions
|
|
branches_config = builder.get_branches_config()
|
|
version = branches_config.track_major_minor_versions[builder.track_id]
|
|
dev_version = branches_config.track_major_minor_versions["vdev"]
|
|
latest_version = branches_config.doc_stable_major_minor_version
|
|
|
|
# Get remote paths
|
|
worker_config = builder.get_worker_config()
|
|
connect_id = f"{worker_config.docs_user}@{worker_config.docs_machine}"
|
|
docs_remote_path = (
|
|
pathlib.Path(worker_config.docs_folder)
|
|
/ "docs.blender.org"
|
|
/ "htdocs"
|
|
/ builder.service_env_id
|
|
)
|
|
|
|
# Sync each locale
|
|
for locale in locales:
|
|
worker.utils.info(f"Syncing {locale}")
|
|
|
|
# Create directory
|
|
remote_path = docs_remote_path / "manual" / locale
|
|
version_remote_path = remote_path / version
|
|
worker.utils.call_ssh(connect_id, ["mkdir", "-p", version_remote_path])
|
|
|
|
if doc_format == "html":
|
|
# Sync html files
|
|
source_path = f"{builder.build_path}/{doc_format}/{locale}/"
|
|
dest_path = f"{connect_id}:{version_remote_path}/"
|
|
# Exclude packaged download files; these get synced with `needs_package_delivery`.
|
|
worker.utils.rsync(
|
|
source_path,
|
|
dest_path,
|
|
exclude_paths=[".doctrees", "blender_manual_*.zip"],
|
|
delete=True,
|
|
delete_path_check=str(version_remote_path),
|
|
)
|
|
|
|
# Create links
|
|
if builder.track_id == "vdev":
|
|
worker.utils.info(f"Creating links for {locale}")
|
|
worker.utils.call_ssh(
|
|
connect_id,
|
|
["ln", "-svF", remote_path / dev_version, remote_path / "dev"],
|
|
)
|
|
worker.utils.call_ssh(
|
|
connect_id,
|
|
[
|
|
"ln",
|
|
"-svF",
|
|
remote_path / latest_version,
|
|
remote_path / "latest",
|
|
],
|
|
)
|
|
|
|
if builder.needs_package_delivery:
|
|
# Sync zip package
|
|
worker.utils.info(f"Syncing package for {locale}")
|
|
build_package_path = builder.build_path / "package"
|
|
package_file_name = f"blender_manual_{doc_format}.zip"
|
|
source_path = build_package_path / locale / package_file_name
|
|
dest_path = f"{connect_id}:{version_remote_path}/{package_file_name}"
|
|
worker.utils.rsync(source_path, dest_path, exclude_paths=[".doctrees"])
|
|
|
|
# Create and sync versions.json
|
|
worker.utils.info("Creating and syncing versions.json")
|
|
|
|
doc_version_labels = branches_config.doc_manual_version_labels
|
|
versions_path = builder.build_path / "versions.json"
|
|
versions_path.write_text(json.dumps(doc_version_labels, indent=2))
|
|
worker.utils.info(versions_path.read_text())
|
|
|
|
dest_path = f"{connect_id}:{docs_remote_path}/versions.json"
|
|
worker.utils.rsync(versions_path, dest_path)
|
|
|
|
|
|
def clean(builder: ManualBuilder) -> None:
|
|
worker.utils.remove_dir(builder.build_path)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
steps: worker.utils.BuilderSteps = OrderedDict()
|
|
steps["configure-machine"] = worker.configure.configure_machine
|
|
steps["update"] = update
|
|
steps["check"] = check
|
|
steps["compile"] = compile_doc
|
|
steps["package"] = package
|
|
steps["deliver"] = deliver
|
|
steps["clean"] = clean
|
|
|
|
parser = worker.utils.create_argument_parser(steps=steps)
|
|
parser.add_argument("--needs-all-locales", action="store_true", required=False)
|
|
parser.add_argument("--needs-package-delivery", action="store_true", required=False)
|
|
parser.add_argument(
|
|
"--doc-format",
|
|
default="html",
|
|
type=str,
|
|
required=False,
|
|
choices=["html", "epub"],
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
builder = ManualBuilder(args)
|
|
builder.run(args.step, steps)
|