# SPDX-License-Identifier: GPL-2.0-or-later # SPDX-FileCopyrightText: 2011-2024 Blender Authors # # Runs on buildbot worker, creating a release package using the build # system and zipping it into buildbot_upload.zip. This is then uploaded # to the master in the next buildbot step. import hashlib import json import os import sys import pathlib import tarfile import worker.blender import worker.utils import worker.blender.sign import worker.blender.bundle_dmg import worker.blender.version # SemVer based file naming def get_package_name(builder: worker.blender.CodeBuilder) -> str: version_info = worker.blender.version.VersionInfo(builder) # For release branch we will trim redundant info branch_id = ( builder.branch_id.replace("/", "-") .replace(".", "") .replace("blender-", "") .replace("-release", "") ) package_name = "bpy" if builder.python_module else "blender" package_name += f"-{version_info.version}" package_name += f"-{version_info.risk_id}" package_name += f"+{branch_id}" if builder.patch_id: if builder.patch_id.startswith("D"): package_name += f"-{builder.patch_id}" else: package_name += f"-PR{builder.patch_id}" package_name += f".{version_info.hash}" package_name += f"-{builder.platform}" package_name += f".{builder.architecture}" package_name += f"-{builder.build_configuration}" return package_name # Generate .sha256 file next to packge def generate_file_hash(package_file_path: pathlib.Path) -> None: hash_algorithm = hashlib.sha256() mem_array = bytearray(128 * 1024) mem_view = memoryview(mem_array) with open(package_file_path, "rb", buffering=0) as f: while 1: # https://github.com/python/typeshed/issues/2166 n = f.readinto(mem_view) # type: ignore if not n: break hash_algorithm.update(mem_view[:n]) hash_file_path = (package_file_path.parent) / (package_file_path.name + ".sha256") hash_text = hash_algorithm.hexdigest() hash_file_path.write_text(hash_text) worker.utils.info(f"Generated hash [{hash_file_path}]") print(hash_text) # tar cf archive.tar test.c --owner=0 --group=0 def create_tar_xz(src: pathlib.Path, dest: pathlib.Path, package_name: str) -> None: # One extra to remove leading os.sep when cleaning root for package_root ln = len(str(src)) + 1 flist = list() # Create list of tuples containing file and archive name for root, dirs, files in os.walk(src): package_root = os.path.join(package_name, root[ln:]) flist.extend( [ (os.path.join(root, file), os.path.join(package_root, file)) for file in files ] ) # Set UID/GID of archived files to 0, otherwise they'd be owned by whatever # user compiled the package. If root then unpacks it to /usr/local/ you get # a security issue. def _fakeroot(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo: tarinfo.gid = 0 tarinfo.gname = "root" tarinfo.uid = 0 tarinfo.uname = "root" return tarinfo # Silence false positive mypy error. package = tarfile.open(dest, "w:xz", preset=6) # type: ignore[call-arg] for entry in flist: worker.utils.info(f"Adding [{entry[0]}] to archive [{entry[1]}]") package.add(entry[0], entry[1], recursive=False, filter=_fakeroot) package.close() def cleanup_files(dirpath: pathlib.Path, extension: str) -> None: if dirpath.exists(): for filename in os.listdir(dirpath): filepath = pathlib.Path(os.path.join(dirpath, filename)) if filepath.is_file() and filename.endswith(extension): worker.utils.remove_file(filepath) def pack_mac(builder: worker.blender.CodeBuilder) -> None: worker.blender.version.VersionInfo(builder) os.chdir(builder.build_dir) cleanup_files(builder.package_dir, ".dmg") package_name = get_package_name(builder) package_file_name = package_name + ".dmg" package_file_path = builder.package_dir / package_file_name applescript_file_path = ( pathlib.Path(__file__).parent.resolve() / "blender.applescript" ) background_image_file_path = ( builder.blender_dir / "release" / "darwin" / "background.tif" ) worker.blender.bundle_dmg.bundle( builder.install_dir, package_file_path, applescript_file_path, background_image_file_path, ) # Sign worker.blender.sign.sign_darwin_files( builder, [package_file_path], "entitlements.plist" ) # Notarize worker_config = builder.get_worker_config() team_id = worker_config.sign_code_darwin_team_id apple_id = worker_config.sign_code_darwin_apple_id keychain_profile = worker_config.sign_code_darwin_keychain_profile timeout = "30m" if builder.service_env_id == "LOCAL" and not apple_id: worker.utils.info("Skipping notarization without Apple ID in local build") return # Upload file and wait for completion. notarize_cmd = [ "xcrun", "notarytool", "submit", package_file_path, "--apple-id", worker.utils.HiddenArgument(apple_id), "--keychain-profile", worker.utils.HiddenArgument(keychain_profile), "--team-id", worker.utils.HiddenArgument(team_id), "--timeout", timeout, "--wait", "--output-format", "json", ] request = worker.utils.check_output(notarize_cmd) request_data = json.loads(request) request_id = request_data["id"] request_status = request_data["status"] # Show logs worker.utils.call( [ "xcrun", "notarytool", "log", "--keychain-profile", keychain_profile, request_id, ], retry_count=5, retry_wait_time=10.0, ) # Failed? if request_status != "Accepted": raise Exception("Notarization failed, aborting") # Staple it worker.utils.call(["xcrun", "stapler", "staple", package_file_path]) generate_file_hash(package_file_path) def pack_win(builder: worker.blender.CodeBuilder, pack_format: str) -> None: os.chdir(builder.build_dir) if pack_format == "msi": cpack_type = "WIX" else: cpack_type = "ZIP" package_extension = pack_format cleanup_files(builder.package_dir, f".{package_extension}") script_folder_path = pathlib.Path(os.path.realpath(__file__)).parent # Will take care of codesigning and correct the folder name in zip # # Code signing is done as part of INSTALL target, which makes it possible to sign # files which are aimed into a bundle and coming from a non-signed source (such as # libraries SVN). # # This is achieved by specifying cpack_post.cmake as a post-install script run # by cpack. cpack_post.ps1 takes care of actual code signing. post_script_file_path = script_folder_path / "cpack_post.cmake" app_id = "Blender" final_package_name = get_package_name(builder) # MSI needs the app id for the Windows menu folder name # It will also fail if anything else. cpack_package_name = app_id if pack_format == "msi" else final_package_name cmake_cmd = [ "cmake", f"-DCPACK_PACKAGE_NAME:STRING={cpack_package_name}", f"-DCPACK_OVERRIDE_PACKAGENAME:STRING={cpack_package_name}", # Only works with ZIP, ignored by MSI # f'-DARCHIVE_FILE:STRING={package_name}', # f'-DCPACK_PACKAGE_FILE_NAME:STRING={cpack_package_name}', f"-DCMAKE_INSTALL_PREFIX:PATH={builder.install_dir}", f"-DPOSTINSTALL_SCRIPT:PATH={post_script_file_path}", ".", ] builder.call(cmake_cmd) worker.utils.info("Packaging Blender files") cpack_cmd = [ "cpack", "-G", cpack_type, # '--verbose', "--trace-expand", "-C", builder.build_configuration, "-B", str(builder.package_dir), # CPACK_PACKAGE_DIRECTORY "-P", cpack_package_name, ] builder.call(cpack_cmd) final_package_file_name = f"{final_package_name}.{package_extension}" final_package_file_path = builder.package_dir / final_package_file_name # HACK: Rename files correctly, packages are appended `-windows64` with no option to rename bogus_cpack_file_path = ( builder.package_dir / f"{cpack_package_name}-windows64.{package_extension}" ) if pack_format == "zip": if bogus_cpack_file_path.exists(): worker.utils.info(f"Removing bogus file [{bogus_cpack_file_path}]") worker.utils.remove_file(bogus_cpack_file_path) source_cpack_file_path = ( builder.package_dir / "_CPack_Packages" / "Windows" / "ZIP" / f"{final_package_file_name}" ) worker.utils.info( f"Moving [{source_cpack_file_path}] to [{final_package_file_path}]" ) os.rename(source_cpack_file_path, final_package_file_path) else: os.rename(bogus_cpack_file_path, final_package_file_path) version_info = worker.blender.version.VersionInfo(builder) description = f"Blender {version_info.version}" worker.blender.sign.sign_windows_files( builder.service_env_id, [final_package_file_path], description=description ) generate_file_hash(final_package_file_path) def pack_linux(builder: worker.blender.CodeBuilder) -> None: blender_executable = builder.install_dir / "blender" version_info = worker.blender.version.VersionInfo(builder) # Strip all unused symbols from the binaries worker.utils.info("Stripping binaries") builder.call(["strip", "--strip-all", blender_executable]) worker.utils.info("Stripping python") # This should work for 3.0, but for now it is in 3.00 py_target = builder.install_dir / version_info.short_version if not os.path.exists(py_target): # Support older format and current issue with 3.00 py_target = builder.install_dir / ( "%d.%02d" % (version_info.major, version_info.minor) ) worker.utils.call( ["find", py_target, "-iname", "*.so", "-exec", "strip", "-s", "{}", ";"] ) package_name = get_package_name(builder) package_file_name = f"{package_name}.tar.xz" package_file_path = builder.package_dir / package_file_name worker.utils.info(f"Creating [{package_file_path}] archive") os.makedirs(builder.package_dir, exist_ok=True) create_tar_xz(builder.install_dir, package_file_path, package_name) generate_file_hash(package_file_path) def pack_python_module(builder: worker.blender.CodeBuilder) -> None: cleanup_files(builder.package_dir, ".whl") cleanup_files(builder.package_dir, ".zip") package_name = get_package_name(builder) + ".zip" package_filepath = builder.package_dir / package_name pack_script = builder.blender_dir / "build_files" / "utils" / "make_bpy_wheel.py" # Make wheel worker.utils.info("Packaging Python Wheel") cmd = [sys.executable, pack_script, builder.install_dir] cmd += ["--build-dir", builder.build_dir] cmd += ["--output-dir", builder.package_dir] builder.call(cmd) # Pack wheel in zip, until pipeline and www can deal with .whl files. import zipfile with zipfile.ZipFile(package_filepath, "w") as zipf: for whl_name in os.listdir(builder.package_dir): if whl_name.endswith(".whl"): whl_filepath = builder.package_dir / whl_name zipf.write(whl_filepath, arcname=whl_name) cleanup_files(builder.package_dir, ".whl") generate_file_hash(package_filepath) def pack(builder: worker.blender.CodeBuilder) -> None: builder.setup_build_environment() # Create clean package directory worker.utils.remove_dir(builder.package_dir) os.makedirs(builder.package_dir, exist_ok=True) # Make sure install directory always exists os.makedirs(builder.install_dir, exist_ok=True) if builder.python_module: pack_python_module(builder) elif builder.platform == "darwin": pack_mac(builder) elif builder.platform == "windows": pack_win(builder, "zip") if builder.track_id not in ["vdev", "vexp"]: pack_win(builder, "msi") elif builder.platform == "linux": pack_linux(builder)