# SPDX-License-Identifier: GPL-2.0-or-later # SPDX-FileCopyrightText: 2011-2024 Blender Authors # import os import pathlib import platform import psutil import shutil from typing import List, Tuple import worker.utils def get_os_release() -> str: if platform.system() == "Darwin": return "macOS " + platform.mac_ver()[0] else: return platform.version() def get_cpu_info() -> str: if platform.system() == "Darwin": return worker.utils.check_output(["/usr/sbin/sysctl", "-n", "machdep.cpu.brand_string"]) elif platform.system() == "Linux": cpuinfo = pathlib.Path("/proc/cpuinfo").read_text() for line in cpuinfo.splitlines(): if line.find("model name") != -1: return line.split(":")[1].strip() return platform.processor() def disk_free_in_gb(builder: worker.utils.Builder) -> float: _, _, disk_free = shutil.disk_usage(builder.track_path) return disk_free / (1024.0**3) def get_thread_count(thread_memory_in_GB: float) -> int: num_threads = psutil.cpu_count() memory_in_GB = psutil.virtual_memory().total / (1024**3) return min(int(memory_in_GB / thread_memory_in_GB), num_threads) def clean(builder: worker.utils.Builder) -> None: # Remove build folders to make space. delete_paths: List[pathlib.Path] = [] optional_delete_paths: List[pathlib.Path] = [] branches_config = builder.get_branches_config() tracks = branches_config.track_major_minor_versions.keys() # TODO: don't hardcode these folder and track names for track in tracks: track_path = builder.tracks_root_path / ("blender-manual-" + track) optional_delete_paths += [track_path / "build"] for track in tracks: track_path = builder.tracks_root_path / ("blender-" + track) delete_paths += [track_path / "build_download"] delete_paths += [track_path / "build_linux"] delete_paths += [track_path / "build_darwin"] delete_paths += [track_path / "build_package"] delete_paths += [track_path / "build_source"] delete_paths += [track_path / "build_debug"] delete_paths += [track_path / "build_arm64_debug"] delete_paths += [track_path / "build_x86_64_debug"] delete_paths += [track_path / "build_sanitizer"] delete_paths += [track_path / "build_arm64_sanitizer"] delete_paths += [track_path / "build_x86_64_sanitizer"] delete_paths += [track_path / "install_release"] delete_paths += [track_path / "install_asserts"] delete_paths += [track_path / "install_sanitizer"] delete_paths += [track_path / "install_debug"] delete_paths += [track_path / "benchmark"] optional_delete_paths += [track_path / "build_release"] optional_delete_paths += [track_path / "build_arm64_release"] optional_delete_paths += [track_path / "build_x86_64_release"] optional_delete_paths += [track_path / "build_asserts"] optional_delete_paths += [track_path / "build_arm64_asserts"] optional_delete_paths += [track_path / "build_x86_64_asserts"] for delete_path in delete_paths: worker.utils.remove_dir(delete_path) # Cached build folders only if we are low on disk space if builder.platform == "darwin": # On macOS APFS this is not reliable, it makes space on demand. # This should be ok still. required_space_gb = 12.0 else: required_space_gb = 25.0 free_space_gb = disk_free_in_gb(builder) if free_space_gb < required_space_gb: worker.utils.warning( f"Trying to delete cached builds for disk space (free {free_space_gb:.2f} GB)" ) sorted_paths: List[Tuple[float, pathlib.Path]] = [] for delete_path in optional_delete_paths: try: sorted_paths += [(os.path.getmtime(delete_path), delete_path)] except: pass for _, delete_path in sorted(sorted_paths): worker.utils.remove_dir(delete_path) if disk_free_in_gb(builder) >= required_space_gb: break # Might be left over from git command hanging stack_dump_file_path = builder.code_path / "sh.exe.stackdump" worker.utils.remove_file(stack_dump_file_path) def configure_machine(builder: worker.utils.Builder) -> None: worker_config = builder.get_worker_config() clean(builder) # Print system information. processor = get_cpu_info() worker.utils.info("System information") print(f"System: {platform.system()}") print(f"Release: {get_os_release()}") print(f"Version: {platform.version()}") print(f"Processor: {processor}") print(f"Cores: {psutil.cpu_count()} logical, {psutil.cpu_count(logical=False)} physical") print(f"Total Memory: {psutil.virtual_memory().total / (1024**3):.2f} GB") print(f"Available Memory: {psutil.virtual_memory().available / (1024**3):.2f} GB") disk_total, disk_used, disk_free = shutil.disk_usage(builder.track_path) print( f"Disk: total {disk_total / (1024**3):.2f} GB, " f"used {disk_used / (1024**3):.2f} GB, " f"free {disk_free / (1024**3):.2f} GB" ) # Check dependencies and provision worker.utils.info("Checking installable software cache") avilable_software_artifacts = worker_config.software_cache_path.glob("*/*") for artifact in avilable_software_artifacts: print(artifact) # Check packages if builder.platform == "linux": etc_rocky = pathlib.Path("/etc/rocky-release") if etc_rocky.exists(): worker.utils.call(["yum", "updateinfo"]) worker.utils.call(["yum", "list", "updates"]) else: worker.utils.call(["apt", "list", "--upgradable"]) elif builder.platform == "windows": choco_version_str = worker.utils.check_output(["choco", "--version"]) choco_version = [int(x) for x in choco_version_str.split(".")] if choco_version[0] >= 2: # In the newer Chocolatey versions `choco list` behavior got changed # to only list installed package, and the --localonly flag has been # removed. worker.utils.call(["choco", "list"]) else: worker.utils.call(["choco", "list", "--lo"]) worker.utils.call(["choco", "outdated"]) # Not an actual command, disabled for now. # worker.utils.call(["scoop", "list"]) # worker.utils.call(["scoop", "status"]) elif builder.platform == "darwin": worker.utils.call(["brew", "update"]) worker.utils.call(["brew", "outdated", "--cask"]) worker.utils.call(["xcrun", "--show-sdk-path"]) # XXX Windows builder debug code if builder.platform == "windows": # Ensure the idiff.exe process is stopped. # It might be hanging there since the previously failed build and it will # prevent removal of the install directory for the new build (due to held # open DLLs). worker.utils.info("Stopping idiff.exe if running") dump_folder = pathlib.Path("C:\\tmp\\dump\\") os.makedirs(dump_folder, exist_ok=True) worker.utils.call(["procdump", "idiff.exe", dump_folder], exit_on_error=False) for proc in psutil.process_iter(): if proc.name() == "idiff.exe": proc.kill() for proc in psutil.process_iter(): if proc.name().lower() in ["blender", "blender.exe", "blender_test", "blender_test.exe"]: worker.utils.warning("Killing stray Blender process") proc.kill()