Get back to original
This commit is contained in:
Bart van der Braak
parent
77ae214d24
commit
5cc9d7b0e9
68 changed files with 83 additions and 42 deletions
310
buildbot/config/worker/doc_manual.py
Executable file
310
buildbot/config/worker/doc_manual.py
Executable file
|
|
@ -0,0 +1,310 @@
|
|||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||
# SPDX-FileCopyrightText: 2011-2024 Blender Authors
|
||||
# <pep8 compliant>
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
from collections import OrderedDict
|
||||
from datetime import timedelta
|
||||
from typing import Optional, Sequence
|
||||
|
||||
sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent))
|
||||
|
||||
import worker.configure
|
||||
import worker.utils
|
||||
|
||||
|
||||
class ManualBuilder(worker.utils.Builder):
|
||||
def __init__(self, args: argparse.Namespace):
|
||||
super().__init__(args, "blender", "blender-manual")
|
||||
self.needs_all_locales = args.needs_all_locales
|
||||
self.needs_package_delivery = args.needs_package_delivery
|
||||
self.doc_format = args.doc_format
|
||||
self.build_path = self.track_path / "build"
|
||||
self.setup_track_path()
|
||||
|
||||
def get_locales(self) -> Sequence[str]:
|
||||
locales = ["en"]
|
||||
if self.needs_all_locales:
|
||||
locale_path = self.code_path / "locale"
|
||||
locales += [
|
||||
item.name
|
||||
for item in locale_path.iterdir()
|
||||
if not item.name.startswith(".")
|
||||
]
|
||||
return locales
|
||||
|
||||
|
||||
def update(builder: ManualBuilder) -> None:
|
||||
builder.update_source()
|
||||
if builder.needs_all_locales:
|
||||
worker.utils.update_source(
|
||||
"blender", "blender-manual-translations", builder.code_path / "locale"
|
||||
)
|
||||
|
||||
|
||||
def check(builder: ManualBuilder) -> None:
|
||||
os.chdir(builder.track_path)
|
||||
worker.utils.call_pipenv(
|
||||
["install", "--pre", "--requirements", builder.code_path / "requirements.txt"]
|
||||
)
|
||||
|
||||
os.chdir(builder.code_path)
|
||||
|
||||
make_cmd = "make.bat" if builder.platform == "windows" else "make"
|
||||
worker.utils.call_pipenv(["run", make_cmd, "check_structure"])
|
||||
# worker.utils.call_pipenv(["run", make_cmd, "check_syntax"])
|
||||
# worker.utils.call_pipenv(["run", make_cmd, "check_spelling"])
|
||||
|
||||
|
||||
def compile_doc(builder: ManualBuilder) -> None:
|
||||
# Install requirements.
|
||||
os.chdir(builder.track_path)
|
||||
worker.utils.call_pipenv(
|
||||
["install", "--pre", "--requirements", builder.code_path / "requirements.txt"]
|
||||
)
|
||||
|
||||
# Determine format and locales
|
||||
locales = builder.get_locales()
|
||||
doc_format = builder.doc_format
|
||||
|
||||
# Clean build folder
|
||||
worker.utils.remove_dir(builder.build_path)
|
||||
os.makedirs(builder.build_path, exist_ok=True)
|
||||
os.chdir(builder.code_path)
|
||||
|
||||
branches_config = builder.get_branches_config()
|
||||
|
||||
# Check manual version matches track.
|
||||
conf_file_path = builder.code_path / "manual" / "conf.py"
|
||||
conf_text = conf_file_path.read_text()
|
||||
match = re.search(r"blender_version\s*=\s*['\"](.*)['\"]", conf_text)
|
||||
expected_version = branches_config.track_major_minor_versions[builder.track_id]
|
||||
found_version = match.groups(0)[0] if match else "nothing"
|
||||
if found_version != expected_version:
|
||||
raise Exception(
|
||||
f"Expected blender_version {expected_version}, but found {found_version} in manual/conf.py"
|
||||
)
|
||||
|
||||
def filter_output(line: str) -> Optional[str]:
|
||||
if line.find("WARNING: unknown mimetype for .doctrees") != -1:
|
||||
return None
|
||||
elif line.find("copying images...") != -1:
|
||||
return None
|
||||
return line
|
||||
|
||||
# Generate manual
|
||||
for locale in locales:
|
||||
start_timestamp = time.time()
|
||||
worker.utils.info(f"Generating {locale} in {doc_format}")
|
||||
|
||||
num_threads = worker.configure.get_thread_count(thread_memory_in_GB=1.25)
|
||||
|
||||
os.chdir(builder.code_path)
|
||||
build_output_path = builder.build_path / doc_format / locale
|
||||
|
||||
worker.utils.call_pipenv(
|
||||
[
|
||||
"run",
|
||||
"sphinx-build",
|
||||
"-b",
|
||||
doc_format,
|
||||
"-j",
|
||||
str(num_threads),
|
||||
"-D",
|
||||
f"language={locale}",
|
||||
"./manual",
|
||||
build_output_path,
|
||||
],
|
||||
filter_output=filter_output,
|
||||
)
|
||||
|
||||
if doc_format == "epub":
|
||||
if not build_output_path.rglob("*.epub"):
|
||||
raise Exception(f"Expected epub files missing in {build_output_path}")
|
||||
|
||||
# Hack appropriate versions.json URL into version_switch.js
|
||||
worker.utils.info("Replacing URL in version_switch.js")
|
||||
|
||||
version_switch_file_path = (
|
||||
build_output_path / "_static" / "js" / "version_switch.js"
|
||||
)
|
||||
versions_file_url = (
|
||||
f"https://docs.blender.org/{builder.service_env_id}/versions.json"
|
||||
)
|
||||
|
||||
version_switch_text = version_switch_file_path.read_text()
|
||||
version_switch_text = version_switch_text.replace(
|
||||
"https://docs.blender.org/versions.json", versions_file_url
|
||||
)
|
||||
version_switch_text = version_switch_text.replace(
|
||||
"https://docs.blender.org/PROD/versions.json", versions_file_url
|
||||
)
|
||||
version_switch_text = version_switch_text.replace(
|
||||
"https://docs.blender.org/UATEST/versions.json", versions_file_url
|
||||
)
|
||||
version_switch_file_path.write_text(version_switch_text)
|
||||
|
||||
time_total = time.time() - start_timestamp
|
||||
time_delta = str(timedelta(seconds=time_total))
|
||||
worker.utils.info(f"Generated {locale} in {doc_format} in {time_delta}")
|
||||
|
||||
|
||||
def package(builder: ManualBuilder) -> None:
|
||||
if not builder.needs_package_delivery:
|
||||
worker.utils.info("No package delivery needed, skipping packaging")
|
||||
return
|
||||
|
||||
locales = builder.get_locales()
|
||||
doc_format = builder.doc_format
|
||||
|
||||
os.chdir(builder.build_path)
|
||||
|
||||
compression_option = "" # "-mx=9"
|
||||
package_file_name = f"blender_manual_{doc_format}.zip"
|
||||
|
||||
build_package_path = builder.build_path / "package"
|
||||
|
||||
for locale in locales:
|
||||
package_file_path = build_package_path / locale / package_file_name
|
||||
worker.utils.remove_file(package_file_path)
|
||||
|
||||
source_path = f"{doc_format}/{locale}"
|
||||
|
||||
cmd = [
|
||||
"7z",
|
||||
"a",
|
||||
"-tzip",
|
||||
package_file_path,
|
||||
source_path,
|
||||
"-r",
|
||||
"-xr!.doctrees",
|
||||
compression_option,
|
||||
]
|
||||
worker.utils.call(cmd)
|
||||
|
||||
cmd = [
|
||||
"7z",
|
||||
"rn",
|
||||
package_file_path,
|
||||
source_path,
|
||||
f"blender_manual_{builder.track_id}_{locale}.{doc_format}",
|
||||
]
|
||||
worker.utils.call(cmd)
|
||||
|
||||
|
||||
def deliver(builder: ManualBuilder) -> None:
|
||||
locales = builder.get_locales()
|
||||
doc_format = builder.doc_format
|
||||
|
||||
# Get versions
|
||||
branches_config = builder.get_branches_config()
|
||||
version = branches_config.track_major_minor_versions[builder.track_id]
|
||||
dev_version = branches_config.track_major_minor_versions["vdev"]
|
||||
latest_version = branches_config.doc_stable_major_minor_version
|
||||
|
||||
# Get remote paths
|
||||
worker_config = builder.get_worker_config()
|
||||
connect_id = f"{worker_config.docs_user}@{worker_config.docs_machine}"
|
||||
docs_remote_path = (
|
||||
pathlib.Path(worker_config.docs_folder)
|
||||
/ "docs.blender.org"
|
||||
/ "htdocs"
|
||||
/ builder.service_env_id
|
||||
)
|
||||
|
||||
# Sync each locale
|
||||
for locale in locales:
|
||||
worker.utils.info(f"Syncing {locale}")
|
||||
|
||||
# Create directory
|
||||
remote_path = docs_remote_path / "manual" / locale
|
||||
version_remote_path = remote_path / version
|
||||
worker.utils.call_ssh(connect_id, ["mkdir", "-p", version_remote_path])
|
||||
|
||||
if doc_format == "html":
|
||||
# Sync html files
|
||||
source_path = f"{builder.build_path}/{doc_format}/{locale}/"
|
||||
dest_path = f"{connect_id}:{version_remote_path}/"
|
||||
# Exclude packaged download files; these get synced with `needs_package_delivery`.
|
||||
worker.utils.rsync(
|
||||
source_path,
|
||||
dest_path,
|
||||
exclude_paths=[".doctrees", "blender_manual_*.zip"],
|
||||
delete=True,
|
||||
delete_path_check=str(version_remote_path),
|
||||
)
|
||||
|
||||
# Create links
|
||||
if builder.track_id == "vdev":
|
||||
worker.utils.info(f"Creating links for {locale}")
|
||||
worker.utils.call_ssh(
|
||||
connect_id,
|
||||
["ln", "-svF", remote_path / dev_version, remote_path / "dev"],
|
||||
)
|
||||
worker.utils.call_ssh(
|
||||
connect_id,
|
||||
[
|
||||
"ln",
|
||||
"-svF",
|
||||
remote_path / latest_version,
|
||||
remote_path / "latest",
|
||||
],
|
||||
)
|
||||
|
||||
if builder.needs_package_delivery:
|
||||
# Sync zip package
|
||||
worker.utils.info(f"Syncing package for {locale}")
|
||||
build_package_path = builder.build_path / "package"
|
||||
package_file_name = f"blender_manual_{doc_format}.zip"
|
||||
source_path = build_package_path / locale / package_file_name
|
||||
dest_path = f"{connect_id}:{version_remote_path}/{package_file_name}"
|
||||
worker.utils.rsync(source_path, dest_path, exclude_paths=[".doctrees"])
|
||||
|
||||
# Create and sync versions.json
|
||||
worker.utils.info("Creating and syncing versions.json")
|
||||
|
||||
doc_version_labels = branches_config.doc_manual_version_labels
|
||||
versions_path = builder.build_path / "versions.json"
|
||||
versions_path.write_text(json.dumps(doc_version_labels, indent=2))
|
||||
worker.utils.info(versions_path.read_text())
|
||||
|
||||
dest_path = f"{connect_id}:{docs_remote_path}/versions.json"
|
||||
worker.utils.rsync(versions_path, dest_path)
|
||||
|
||||
|
||||
def clean(builder: ManualBuilder) -> None:
|
||||
worker.utils.remove_dir(builder.build_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
steps: worker.utils.BuilderSteps = OrderedDict()
|
||||
steps["configure-machine"] = worker.configure.configure_machine
|
||||
steps["update"] = update
|
||||
steps["check"] = check
|
||||
steps["compile"] = compile_doc
|
||||
steps["package"] = package
|
||||
steps["deliver"] = deliver
|
||||
steps["clean"] = clean
|
||||
|
||||
parser = worker.utils.create_argument_parser(steps=steps)
|
||||
parser.add_argument("--needs-all-locales", action="store_true", required=False)
|
||||
parser.add_argument("--needs-package-delivery", action="store_true", required=False)
|
||||
parser.add_argument(
|
||||
"--doc-format",
|
||||
default="html",
|
||||
type=str,
|
||||
required=False,
|
||||
choices=["html", "epub"],
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
builder = ManualBuilder(args)
|
||||
builder.run(args.step, steps)
|
||||
Loading…
Add table
Add a link
Reference in a new issue