199 lines
7.5 KiB
Python
199 lines
7.5 KiB
Python
# SPDX-License-Identifier: GPL-2.0-or-later
|
|
# SPDX-FileCopyrightText: 2011-2024 Blender Authors
|
|
# <pep8 compliant>
|
|
|
|
import os
|
|
import pathlib
|
|
import platform
|
|
import psutil
|
|
import shutil
|
|
|
|
from typing import List, Tuple
|
|
|
|
import worker.utils
|
|
|
|
|
|
def get_os_release() -> str:
|
|
if platform.system() == "Darwin":
|
|
return "macOS " + platform.mac_ver()[0]
|
|
else:
|
|
return platform.version()
|
|
|
|
|
|
def get_cpu_info() -> str:
|
|
if platform.system() == "Darwin":
|
|
return worker.utils.check_output(["/usr/sbin/sysctl", "-n", "machdep.cpu.brand_string"])
|
|
elif platform.system() == "Linux":
|
|
cpuinfo = pathlib.Path("/proc/cpuinfo").read_text()
|
|
for line in cpuinfo.splitlines():
|
|
if line.find("model name") != -1:
|
|
return line.split(":")[1].strip()
|
|
|
|
return platform.processor()
|
|
|
|
|
|
def disk_free_in_gb(builder: worker.utils.Builder) -> float:
|
|
_, _, disk_free = shutil.disk_usage(builder.track_path)
|
|
return disk_free / (1024.0**3)
|
|
|
|
|
|
def get_thread_count(thread_memory_in_GB: float) -> int:
|
|
num_threads = psutil.cpu_count()
|
|
memory_in_GB = psutil.virtual_memory().total / (1024**3)
|
|
|
|
return min(int(memory_in_GB / thread_memory_in_GB), num_threads)
|
|
|
|
|
|
def clean(builder: worker.utils.Builder) -> None:
|
|
# Remove build folders to make space.
|
|
delete_paths: List[pathlib.Path] = []
|
|
optional_delete_paths: List[pathlib.Path] = []
|
|
|
|
branches_config = builder.get_branches_config()
|
|
tracks = branches_config.track_major_minor_versions.keys()
|
|
|
|
# TODO: don't hardcode these folder and track names
|
|
for track in tracks:
|
|
track_path = builder.tracks_root_path / ("blender-manual-" + track)
|
|
optional_delete_paths += [track_path / "build"]
|
|
|
|
for track in tracks:
|
|
track_path = builder.tracks_root_path / ("blender-" + track)
|
|
delete_paths += [track_path / "build_download"]
|
|
delete_paths += [track_path / "build_linux"]
|
|
delete_paths += [track_path / "build_darwin"]
|
|
delete_paths += [track_path / "build_package"]
|
|
delete_paths += [track_path / "build_source"]
|
|
delete_paths += [track_path / "build_debug"]
|
|
delete_paths += [track_path / "build_arm64_debug"]
|
|
delete_paths += [track_path / "build_x86_64_debug"]
|
|
delete_paths += [track_path / "build_sanitizer"]
|
|
delete_paths += [track_path / "build_arm64_sanitizer"]
|
|
delete_paths += [track_path / "build_x86_64_sanitizer"]
|
|
delete_paths += [track_path / "install_release"]
|
|
delete_paths += [track_path / "install_asserts"]
|
|
delete_paths += [track_path / "install_sanitizer"]
|
|
delete_paths += [track_path / "install_debug"]
|
|
delete_paths += [track_path / "benchmark"]
|
|
optional_delete_paths += [track_path / "build_release"]
|
|
optional_delete_paths += [track_path / "build_arm64_release"]
|
|
optional_delete_paths += [track_path / "build_x86_64_release"]
|
|
optional_delete_paths += [track_path / "build_asserts"]
|
|
optional_delete_paths += [track_path / "build_arm64_asserts"]
|
|
optional_delete_paths += [track_path / "build_x86_64_asserts"]
|
|
|
|
for delete_path in delete_paths:
|
|
worker.utils.remove_dir(delete_path)
|
|
|
|
# Cached build folders only if we are low on disk space
|
|
if builder.platform == "darwin":
|
|
# On macOS APFS this is not reliable, it makes space on demand.
|
|
# This should be ok still.
|
|
required_space_gb = 12.0
|
|
else:
|
|
required_space_gb = 25.0
|
|
|
|
free_space_gb = disk_free_in_gb(builder)
|
|
if free_space_gb < required_space_gb:
|
|
worker.utils.warning(
|
|
f"Trying to delete cached builds for disk space (free {free_space_gb:.2f} GB)"
|
|
)
|
|
sorted_paths: List[Tuple[float, pathlib.Path]] = []
|
|
for delete_path in optional_delete_paths:
|
|
try:
|
|
sorted_paths += [(os.path.getmtime(delete_path), delete_path)]
|
|
except:
|
|
pass
|
|
|
|
for _, delete_path in sorted(sorted_paths):
|
|
worker.utils.remove_dir(delete_path)
|
|
if disk_free_in_gb(builder) >= required_space_gb:
|
|
break
|
|
|
|
# Might be left over from git command hanging
|
|
stack_dump_file_path = builder.code_path / "sh.exe.stackdump"
|
|
worker.utils.remove_file(stack_dump_file_path)
|
|
|
|
|
|
def configure_machine(builder: worker.utils.Builder) -> None:
|
|
worker_config = builder.get_worker_config()
|
|
|
|
clean(builder)
|
|
|
|
# Print system information.
|
|
processor = get_cpu_info()
|
|
|
|
worker.utils.info("System information")
|
|
print(f"System: {platform.system()}")
|
|
print(f"Release: {get_os_release()}")
|
|
print(f"Version: {platform.version()}")
|
|
print(f"Processor: {processor}")
|
|
print(f"Cores: {psutil.cpu_count()} logical, {psutil.cpu_count(logical=False)} physical")
|
|
print(f"Total Memory: {psutil.virtual_memory().total / (1024**3):.2f} GB")
|
|
print(f"Available Memory: {psutil.virtual_memory().available / (1024**3):.2f} GB")
|
|
|
|
disk_total, disk_used, disk_free = shutil.disk_usage(builder.track_path)
|
|
print(
|
|
f"Disk: total {disk_total / (1024**3):.2f} GB, "
|
|
f"used {disk_used / (1024**3):.2f} GB, "
|
|
f"free {disk_free / (1024**3):.2f} GB"
|
|
)
|
|
|
|
# Check dependencies and provision
|
|
worker.utils.info("Checking installable software cache")
|
|
avilable_software_artifacts = worker_config.software_cache_path.glob("*/*")
|
|
for artifact in avilable_software_artifacts:
|
|
print(artifact)
|
|
|
|
# Check packages
|
|
if builder.platform == "linux":
|
|
etc_rocky = pathlib.Path("/etc/rocky-release")
|
|
|
|
if etc_rocky.exists():
|
|
worker.utils.call(["yum", "updateinfo"])
|
|
worker.utils.call(["yum", "list", "updates"])
|
|
else:
|
|
worker.utils.call(["apt", "list", "--upgradable"])
|
|
|
|
elif builder.platform == "windows":
|
|
choco_version_str = worker.utils.check_output(["choco", "--version"])
|
|
choco_version = [int(x) for x in choco_version_str.split(".")]
|
|
if choco_version[0] >= 2:
|
|
# In the newer Chocolatey versions `choco list` behavior got changed
|
|
# to only list installed package, and the --localonly flag has been
|
|
# removed.
|
|
worker.utils.call(["choco", "list"])
|
|
else:
|
|
worker.utils.call(["choco", "list", "--lo"])
|
|
worker.utils.call(["choco", "outdated"])
|
|
|
|
# Not an actual command, disabled for now.
|
|
# worker.utils.call(["scoop", "list"])
|
|
# worker.utils.call(["scoop", "status"])
|
|
|
|
elif builder.platform == "darwin":
|
|
worker.utils.call(["brew", "update"])
|
|
worker.utils.call(["brew", "outdated", "--cask"])
|
|
worker.utils.call(["xcrun", "--show-sdk-path"])
|
|
|
|
# XXX Windows builder debug code
|
|
if builder.platform == "windows":
|
|
# Ensure the idiff.exe process is stopped.
|
|
# It might be hanging there since the previously failed build and it will
|
|
# prevent removal of the install directory for the new build (due to held
|
|
# open DLLs).
|
|
worker.utils.info("Stopping idiff.exe if running")
|
|
|
|
dump_folder = pathlib.Path("C:\\tmp\\dump\\")
|
|
os.makedirs(dump_folder, exist_ok=True)
|
|
|
|
worker.utils.call(["procdump", "idiff.exe", dump_folder], exit_on_error=False)
|
|
|
|
for proc in psutil.process_iter():
|
|
if proc.name() == "idiff.exe":
|
|
proc.kill()
|
|
|
|
for proc in psutil.process_iter():
|
|
if proc.name().lower() in ["blender", "blender.exe", "blender_test", "blender_test.exe"]:
|
|
worker.utils.warning("Killing stray Blender process")
|
|
proc.kill()
|