383 lines
12 KiB
Python
383 lines
12 KiB
Python
# SPDX-License-Identifier: GPL-2.0-or-later
|
|
# SPDX-FileCopyrightText: 2011-2024 Blender Authors
|
|
# <pep8 compliant>
|
|
|
|
# Runs on buildbot worker, creating a release package using the build
|
|
# system and zipping it into buildbot_upload.zip. This is then uploaded
|
|
# to the master in the next buildbot step.
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
import pathlib
|
|
import tarfile
|
|
|
|
import worker.blender
|
|
import worker.utils
|
|
|
|
import worker.blender.sign
|
|
import worker.blender.bundle_dmg
|
|
import worker.blender.version
|
|
|
|
|
|
# SemVer based file naming
|
|
def get_package_name(builder: worker.blender.CodeBuilder) -> str:
|
|
version_info = worker.blender.version.VersionInfo(builder)
|
|
|
|
# For release branch we will trim redundant info
|
|
branch_id = (
|
|
builder.branch_id.replace("/", "-")
|
|
.replace(".", "")
|
|
.replace("blender-", "")
|
|
.replace("-release", "")
|
|
)
|
|
package_name = "bpy" if builder.python_module else "blender"
|
|
package_name += f"-{version_info.version}"
|
|
package_name += f"-{version_info.risk_id}"
|
|
package_name += f"+{branch_id}"
|
|
if builder.patch_id:
|
|
if builder.patch_id.startswith("D"):
|
|
package_name += f"-{builder.patch_id}"
|
|
else:
|
|
package_name += f"-PR{builder.patch_id}"
|
|
|
|
package_name += f".{version_info.hash}"
|
|
package_name += f"-{builder.platform}"
|
|
package_name += f".{builder.architecture}"
|
|
package_name += f"-{builder.build_configuration}"
|
|
|
|
return package_name
|
|
|
|
|
|
# Generate .sha256 file next to packge
|
|
def generate_file_hash(package_file_path: pathlib.Path) -> None:
|
|
hash_algorithm = hashlib.sha256()
|
|
|
|
mem_array = bytearray(128 * 1024)
|
|
mem_view = memoryview(mem_array)
|
|
with open(package_file_path, "rb", buffering=0) as f:
|
|
while 1:
|
|
# https://github.com/python/typeshed/issues/2166
|
|
n = f.readinto(mem_view) # type: ignore
|
|
if not n:
|
|
break
|
|
hash_algorithm.update(mem_view[:n])
|
|
|
|
hash_file_path = (package_file_path.parent) / (package_file_path.name + ".sha256")
|
|
hash_text = hash_algorithm.hexdigest()
|
|
hash_file_path.write_text(hash_text)
|
|
|
|
worker.utils.info(f"Generated hash [{hash_file_path}]")
|
|
print(hash_text)
|
|
|
|
|
|
# tar cf archive.tar test.c --owner=0 --group=0
|
|
def create_tar_xz(src: pathlib.Path, dest: pathlib.Path, package_name: str) -> None:
|
|
# One extra to remove leading os.sep when cleaning root for package_root
|
|
ln = len(str(src)) + 1
|
|
flist = list()
|
|
|
|
# Create list of tuples containing file and archive name
|
|
for root, dirs, files in os.walk(src):
|
|
package_root = os.path.join(package_name, root[ln:])
|
|
flist.extend(
|
|
[
|
|
(os.path.join(root, file), os.path.join(package_root, file))
|
|
for file in files
|
|
]
|
|
)
|
|
|
|
# Set UID/GID of archived files to 0, otherwise they'd be owned by whatever
|
|
# user compiled the package. If root then unpacks it to /usr/local/ you get
|
|
# a security issue.
|
|
def _fakeroot(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo:
|
|
tarinfo.gid = 0
|
|
tarinfo.gname = "root"
|
|
tarinfo.uid = 0
|
|
tarinfo.uname = "root"
|
|
return tarinfo
|
|
|
|
# Silence false positive mypy error.
|
|
package = tarfile.open(dest, "w:xz", preset=6) # type: ignore[call-arg]
|
|
for entry in flist:
|
|
worker.utils.info(f"Adding [{entry[0]}] to archive [{entry[1]}]")
|
|
package.add(entry[0], entry[1], recursive=False, filter=_fakeroot)
|
|
package.close()
|
|
|
|
|
|
def cleanup_files(dirpath: pathlib.Path, extension: str) -> None:
|
|
if dirpath.exists():
|
|
for filename in os.listdir(dirpath):
|
|
filepath = pathlib.Path(os.path.join(dirpath, filename))
|
|
if filepath.is_file() and filename.endswith(extension):
|
|
worker.utils.remove_file(filepath)
|
|
|
|
|
|
def pack_mac(builder: worker.blender.CodeBuilder) -> None:
|
|
worker.blender.version.VersionInfo(builder)
|
|
|
|
os.chdir(builder.build_dir)
|
|
cleanup_files(builder.package_dir, ".dmg")
|
|
|
|
package_name = get_package_name(builder)
|
|
package_file_name = package_name + ".dmg"
|
|
package_file_path = builder.package_dir / package_file_name
|
|
|
|
applescript_file_path = (
|
|
pathlib.Path(__file__).parent.resolve() / "blender.applescript"
|
|
)
|
|
background_image_file_path = (
|
|
builder.blender_dir / "release" / "darwin" / "background.tif"
|
|
)
|
|
|
|
worker.blender.bundle_dmg.bundle(
|
|
builder.install_dir,
|
|
package_file_path,
|
|
applescript_file_path,
|
|
background_image_file_path,
|
|
)
|
|
|
|
# Sign
|
|
worker.blender.sign.sign_darwin_files(
|
|
builder, [package_file_path], "entitlements.plist"
|
|
)
|
|
|
|
# Notarize
|
|
worker_config = builder.get_worker_config()
|
|
team_id = worker_config.sign_code_darwin_team_id
|
|
apple_id = worker_config.sign_code_darwin_apple_id
|
|
keychain_profile = worker_config.sign_code_darwin_keychain_profile
|
|
timeout = "30m"
|
|
|
|
if builder.service_env_id == "LOCAL" and not apple_id:
|
|
worker.utils.info("Skipping notarization without Apple ID in local build")
|
|
return
|
|
|
|
# Upload file and wait for completion.
|
|
notarize_cmd = [
|
|
"xcrun",
|
|
"notarytool",
|
|
"submit",
|
|
package_file_path,
|
|
"--apple-id",
|
|
worker.utils.HiddenArgument(apple_id),
|
|
"--keychain-profile",
|
|
worker.utils.HiddenArgument(keychain_profile),
|
|
"--team-id",
|
|
worker.utils.HiddenArgument(team_id),
|
|
"--timeout",
|
|
timeout,
|
|
"--wait",
|
|
"--output-format",
|
|
"json",
|
|
]
|
|
|
|
request = worker.utils.check_output(notarize_cmd)
|
|
|
|
request_data = json.loads(request)
|
|
request_id = request_data["id"]
|
|
request_status = request_data["status"]
|
|
|
|
# Show logs
|
|
worker.utils.call(
|
|
[
|
|
"xcrun",
|
|
"notarytool",
|
|
"log",
|
|
"--keychain-profile",
|
|
keychain_profile,
|
|
request_id,
|
|
],
|
|
retry_count=5,
|
|
retry_wait_time=10.0,
|
|
)
|
|
|
|
# Failed?
|
|
if request_status != "Accepted":
|
|
raise Exception("Notarization failed, aborting")
|
|
|
|
# Staple it
|
|
worker.utils.call(["xcrun", "stapler", "staple", package_file_path])
|
|
|
|
generate_file_hash(package_file_path)
|
|
|
|
|
|
def pack_win(builder: worker.blender.CodeBuilder, pack_format: str) -> None:
|
|
os.chdir(builder.build_dir)
|
|
|
|
if pack_format == "msi":
|
|
cpack_type = "WIX"
|
|
else:
|
|
cpack_type = "ZIP"
|
|
|
|
package_extension = pack_format
|
|
cleanup_files(builder.package_dir, f".{package_extension}")
|
|
|
|
script_folder_path = pathlib.Path(os.path.realpath(__file__)).parent
|
|
|
|
# Will take care of codesigning and correct the folder name in zip
|
|
#
|
|
# Code signing is done as part of INSTALL target, which makes it possible to sign
|
|
# files which are aimed into a bundle and coming from a non-signed source (such as
|
|
# libraries SVN).
|
|
#
|
|
# This is achieved by specifying cpack_post.cmake as a post-install script run
|
|
# by cpack. cpack_post.ps1 takes care of actual code signing.
|
|
post_script_file_path = script_folder_path / "cpack_post.cmake"
|
|
|
|
app_id = "Blender"
|
|
final_package_name = get_package_name(builder)
|
|
# MSI needs the app id for the Windows menu folder name
|
|
# It will also fail if anything else.
|
|
cpack_package_name = app_id if pack_format == "msi" else final_package_name
|
|
|
|
cmake_cmd = [
|
|
"cmake",
|
|
f"-DCPACK_PACKAGE_NAME:STRING={cpack_package_name}",
|
|
f"-DCPACK_OVERRIDE_PACKAGENAME:STRING={cpack_package_name}",
|
|
# Only works with ZIP, ignored by MSI
|
|
# f'-DARCHIVE_FILE:STRING={package_name}',
|
|
# f'-DCPACK_PACKAGE_FILE_NAME:STRING={cpack_package_name}',
|
|
f"-DCMAKE_INSTALL_PREFIX:PATH={builder.install_dir}",
|
|
f"-DPOSTINSTALL_SCRIPT:PATH={post_script_file_path}",
|
|
".",
|
|
]
|
|
builder.call(cmake_cmd)
|
|
|
|
worker.utils.info("Packaging Blender files")
|
|
cpack_cmd = [
|
|
"cpack",
|
|
"-G",
|
|
cpack_type,
|
|
# '--verbose',
|
|
"--trace-expand",
|
|
"-C",
|
|
builder.build_configuration,
|
|
"-B",
|
|
str(builder.package_dir), # CPACK_PACKAGE_DIRECTORY
|
|
"-P",
|
|
cpack_package_name,
|
|
]
|
|
builder.call(cpack_cmd)
|
|
|
|
final_package_file_name = f"{final_package_name}.{package_extension}"
|
|
final_package_file_path = builder.package_dir / final_package_file_name
|
|
|
|
# HACK: Rename files correctly, packages are appended `-windows64` with no option to rename
|
|
bogus_cpack_file_path = (
|
|
builder.package_dir / f"{cpack_package_name}-windows64.{package_extension}"
|
|
)
|
|
|
|
if pack_format == "zip":
|
|
if bogus_cpack_file_path.exists():
|
|
worker.utils.info(f"Removing bogus file [{bogus_cpack_file_path}]")
|
|
worker.utils.remove_file(bogus_cpack_file_path)
|
|
|
|
source_cpack_file_path = (
|
|
builder.package_dir
|
|
/ "_CPack_Packages"
|
|
/ "Windows"
|
|
/ "ZIP"
|
|
/ f"{final_package_file_name}"
|
|
)
|
|
worker.utils.info(
|
|
f"Moving [{source_cpack_file_path}] to [{final_package_file_path}]"
|
|
)
|
|
os.rename(source_cpack_file_path, final_package_file_path)
|
|
else:
|
|
os.rename(bogus_cpack_file_path, final_package_file_path)
|
|
version_info = worker.blender.version.VersionInfo(builder)
|
|
description = f"Blender {version_info.version}"
|
|
worker.blender.sign.sign_windows_files(
|
|
builder.service_env_id, [final_package_file_path], description=description
|
|
)
|
|
|
|
generate_file_hash(final_package_file_path)
|
|
|
|
|
|
def pack_linux(builder: worker.blender.CodeBuilder) -> None:
|
|
blender_executable = builder.install_dir / "blender"
|
|
|
|
version_info = worker.blender.version.VersionInfo(builder)
|
|
|
|
# Strip all unused symbols from the binaries
|
|
worker.utils.info("Stripping binaries")
|
|
builder.call(["strip", "--strip-all", blender_executable])
|
|
|
|
worker.utils.info("Stripping python")
|
|
|
|
# This should work for 3.0, but for now it is in 3.00
|
|
py_target = builder.install_dir / version_info.short_version
|
|
if not os.path.exists(py_target):
|
|
# Support older format and current issue with 3.00
|
|
py_target = builder.install_dir / (
|
|
"%d.%02d" % (version_info.major, version_info.minor)
|
|
)
|
|
|
|
worker.utils.call(
|
|
["find", py_target, "-iname", "*.so", "-exec", "strip", "-s", "{}", ";"]
|
|
)
|
|
|
|
package_name = get_package_name(builder)
|
|
package_file_name = f"{package_name}.tar.xz"
|
|
package_file_path = builder.package_dir / package_file_name
|
|
|
|
worker.utils.info(f"Creating [{package_file_path}] archive")
|
|
|
|
os.makedirs(builder.package_dir, exist_ok=True)
|
|
|
|
create_tar_xz(builder.install_dir, package_file_path, package_name)
|
|
|
|
generate_file_hash(package_file_path)
|
|
|
|
|
|
def pack_python_module(builder: worker.blender.CodeBuilder) -> None:
|
|
cleanup_files(builder.package_dir, ".whl")
|
|
cleanup_files(builder.package_dir, ".zip")
|
|
|
|
package_name = get_package_name(builder) + ".zip"
|
|
package_filepath = builder.package_dir / package_name
|
|
pack_script = builder.blender_dir / "build_files" / "utils" / "make_bpy_wheel.py"
|
|
|
|
# Make wheel
|
|
worker.utils.info("Packaging Python Wheel")
|
|
cmd = [sys.executable, pack_script, builder.install_dir]
|
|
cmd += ["--build-dir", builder.build_dir]
|
|
cmd += ["--output-dir", builder.package_dir]
|
|
builder.call(cmd)
|
|
|
|
# Pack wheel in zip, until pipeline and www can deal with .whl files.
|
|
import zipfile
|
|
|
|
with zipfile.ZipFile(package_filepath, "w") as zipf:
|
|
for whl_name in os.listdir(builder.package_dir):
|
|
if whl_name.endswith(".whl"):
|
|
whl_filepath = builder.package_dir / whl_name
|
|
zipf.write(whl_filepath, arcname=whl_name)
|
|
|
|
cleanup_files(builder.package_dir, ".whl")
|
|
|
|
generate_file_hash(package_filepath)
|
|
|
|
|
|
def pack(builder: worker.blender.CodeBuilder) -> None:
|
|
builder.setup_build_environment()
|
|
|
|
# Create clean package directory
|
|
worker.utils.remove_dir(builder.package_dir)
|
|
os.makedirs(builder.package_dir, exist_ok=True)
|
|
|
|
# Make sure install directory always exists
|
|
os.makedirs(builder.install_dir, exist_ok=True)
|
|
|
|
if builder.python_module:
|
|
pack_python_module(builder)
|
|
elif builder.platform == "darwin":
|
|
pack_mac(builder)
|
|
elif builder.platform == "windows":
|
|
pack_win(builder, "zip")
|
|
if builder.track_id not in ["vdev", "vexp"]:
|
|
pack_win(builder, "msi")
|
|
elif builder.platform == "linux":
|
|
pack_linux(builder)
|