Get back to original
This commit is contained in:
Bart van der Braak
parent
77ae214d24
commit
5cc9d7b0e9
68 changed files with 83 additions and 42 deletions
383
buildbot/config/worker/blender/pack.py
Normal file
383
buildbot/config/worker/blender/pack.py
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||
# SPDX-FileCopyrightText: 2011-2024 Blender Authors
|
||||
# <pep8 compliant>
|
||||
|
||||
# Runs on buildbot worker, creating a release package using the build
|
||||
# system and zipping it into buildbot_upload.zip. This is then uploaded
|
||||
# to the master in the next buildbot step.
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import pathlib
|
||||
import tarfile
|
||||
|
||||
import worker.blender
|
||||
import worker.utils
|
||||
|
||||
import worker.blender.sign
|
||||
import worker.blender.bundle_dmg
|
||||
import worker.blender.version
|
||||
|
||||
|
||||
# SemVer based file naming
|
||||
def get_package_name(builder: worker.blender.CodeBuilder) -> str:
|
||||
version_info = worker.blender.version.VersionInfo(builder)
|
||||
|
||||
# For release branch we will trim redundant info
|
||||
branch_id = (
|
||||
builder.branch_id.replace("/", "-")
|
||||
.replace(".", "")
|
||||
.replace("blender-", "")
|
||||
.replace("-release", "")
|
||||
)
|
||||
package_name = "bpy" if builder.python_module else "blender"
|
||||
package_name += f"-{version_info.version}"
|
||||
package_name += f"-{version_info.risk_id}"
|
||||
package_name += f"+{branch_id}"
|
||||
if builder.patch_id:
|
||||
if builder.patch_id.startswith("D"):
|
||||
package_name += f"-{builder.patch_id}"
|
||||
else:
|
||||
package_name += f"-PR{builder.patch_id}"
|
||||
|
||||
package_name += f".{version_info.hash}"
|
||||
package_name += f"-{builder.platform}"
|
||||
package_name += f".{builder.architecture}"
|
||||
package_name += f"-{builder.build_configuration}"
|
||||
|
||||
return package_name
|
||||
|
||||
|
||||
# Generate .sha256 file next to packge
|
||||
def generate_file_hash(package_file_path: pathlib.Path) -> None:
|
||||
hash_algorithm = hashlib.sha256()
|
||||
|
||||
mem_array = bytearray(128 * 1024)
|
||||
mem_view = memoryview(mem_array)
|
||||
with open(package_file_path, "rb", buffering=0) as f:
|
||||
while 1:
|
||||
# https://github.com/python/typeshed/issues/2166
|
||||
n = f.readinto(mem_view) # type: ignore
|
||||
if not n:
|
||||
break
|
||||
hash_algorithm.update(mem_view[:n])
|
||||
|
||||
hash_file_path = (package_file_path.parent) / (package_file_path.name + ".sha256")
|
||||
hash_text = hash_algorithm.hexdigest()
|
||||
hash_file_path.write_text(hash_text)
|
||||
|
||||
worker.utils.info(f"Generated hash [{hash_file_path}]")
|
||||
print(hash_text)
|
||||
|
||||
|
||||
# tar cf archive.tar test.c --owner=0 --group=0
|
||||
def create_tar_xz(src: pathlib.Path, dest: pathlib.Path, package_name: str) -> None:
|
||||
# One extra to remove leading os.sep when cleaning root for package_root
|
||||
ln = len(str(src)) + 1
|
||||
flist = list()
|
||||
|
||||
# Create list of tuples containing file and archive name
|
||||
for root, dirs, files in os.walk(src):
|
||||
package_root = os.path.join(package_name, root[ln:])
|
||||
flist.extend(
|
||||
[
|
||||
(os.path.join(root, file), os.path.join(package_root, file))
|
||||
for file in files
|
||||
]
|
||||
)
|
||||
|
||||
# Set UID/GID of archived files to 0, otherwise they'd be owned by whatever
|
||||
# user compiled the package. If root then unpacks it to /usr/local/ you get
|
||||
# a security issue.
|
||||
def _fakeroot(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo:
|
||||
tarinfo.gid = 0
|
||||
tarinfo.gname = "root"
|
||||
tarinfo.uid = 0
|
||||
tarinfo.uname = "root"
|
||||
return tarinfo
|
||||
|
||||
# Silence false positive mypy error.
|
||||
package = tarfile.open(dest, "w:xz", preset=6) # type: ignore[call-arg]
|
||||
for entry in flist:
|
||||
worker.utils.info(f"Adding [{entry[0]}] to archive [{entry[1]}]")
|
||||
package.add(entry[0], entry[1], recursive=False, filter=_fakeroot)
|
||||
package.close()
|
||||
|
||||
|
||||
def cleanup_files(dirpath: pathlib.Path, extension: str) -> None:
|
||||
if dirpath.exists():
|
||||
for filename in os.listdir(dirpath):
|
||||
filepath = pathlib.Path(os.path.join(dirpath, filename))
|
||||
if filepath.is_file() and filename.endswith(extension):
|
||||
worker.utils.remove_file(filepath)
|
||||
|
||||
|
||||
def pack_mac(builder: worker.blender.CodeBuilder) -> None:
|
||||
worker.blender.version.VersionInfo(builder)
|
||||
|
||||
os.chdir(builder.build_dir)
|
||||
cleanup_files(builder.package_dir, ".dmg")
|
||||
|
||||
package_name = get_package_name(builder)
|
||||
package_file_name = package_name + ".dmg"
|
||||
package_file_path = builder.package_dir / package_file_name
|
||||
|
||||
applescript_file_path = (
|
||||
pathlib.Path(__file__).parent.resolve() / "blender.applescript"
|
||||
)
|
||||
background_image_file_path = (
|
||||
builder.blender_dir / "release" / "darwin" / "background.tif"
|
||||
)
|
||||
|
||||
worker.blender.bundle_dmg.bundle(
|
||||
builder.install_dir,
|
||||
package_file_path,
|
||||
applescript_file_path,
|
||||
background_image_file_path,
|
||||
)
|
||||
|
||||
# Sign
|
||||
worker.blender.sign.sign_darwin_files(
|
||||
builder, [package_file_path], "entitlements.plist"
|
||||
)
|
||||
|
||||
# Notarize
|
||||
worker_config = builder.get_worker_config()
|
||||
team_id = worker_config.sign_code_darwin_team_id
|
||||
apple_id = worker_config.sign_code_darwin_apple_id
|
||||
keychain_profile = worker_config.sign_code_darwin_keychain_profile
|
||||
timeout = "30m"
|
||||
|
||||
if builder.service_env_id == "LOCAL" and not apple_id:
|
||||
worker.utils.info("Skipping notarization without Apple ID in local build")
|
||||
return
|
||||
|
||||
# Upload file and wait for completion.
|
||||
notarize_cmd = [
|
||||
"xcrun",
|
||||
"notarytool",
|
||||
"submit",
|
||||
package_file_path,
|
||||
"--apple-id",
|
||||
worker.utils.HiddenArgument(apple_id),
|
||||
"--keychain-profile",
|
||||
worker.utils.HiddenArgument(keychain_profile),
|
||||
"--team-id",
|
||||
worker.utils.HiddenArgument(team_id),
|
||||
"--timeout",
|
||||
timeout,
|
||||
"--wait",
|
||||
"--output-format",
|
||||
"json",
|
||||
]
|
||||
|
||||
request = worker.utils.check_output(notarize_cmd)
|
||||
|
||||
request_data = json.loads(request)
|
||||
request_id = request_data["id"]
|
||||
request_status = request_data["status"]
|
||||
|
||||
# Show logs
|
||||
worker.utils.call(
|
||||
[
|
||||
"xcrun",
|
||||
"notarytool",
|
||||
"log",
|
||||
"--keychain-profile",
|
||||
keychain_profile,
|
||||
request_id,
|
||||
],
|
||||
retry_count=5,
|
||||
retry_wait_time=10.0,
|
||||
)
|
||||
|
||||
# Failed?
|
||||
if request_status != "Accepted":
|
||||
raise Exception("Notarization failed, aborting")
|
||||
|
||||
# Staple it
|
||||
worker.utils.call(["xcrun", "stapler", "staple", package_file_path])
|
||||
|
||||
generate_file_hash(package_file_path)
|
||||
|
||||
|
||||
def pack_win(builder: worker.blender.CodeBuilder, pack_format: str) -> None:
|
||||
os.chdir(builder.build_dir)
|
||||
|
||||
if pack_format == "msi":
|
||||
cpack_type = "WIX"
|
||||
else:
|
||||
cpack_type = "ZIP"
|
||||
|
||||
package_extension = pack_format
|
||||
cleanup_files(builder.package_dir, f".{package_extension}")
|
||||
|
||||
script_folder_path = pathlib.Path(os.path.realpath(__file__)).parent
|
||||
|
||||
# Will take care of codesigning and correct the folder name in zip
|
||||
#
|
||||
# Code signing is done as part of INSTALL target, which makes it possible to sign
|
||||
# files which are aimed into a bundle and coming from a non-signed source (such as
|
||||
# libraries SVN).
|
||||
#
|
||||
# This is achieved by specifying cpack_post.cmake as a post-install script run
|
||||
# by cpack. cpack_post.ps1 takes care of actual code signing.
|
||||
post_script_file_path = script_folder_path / "cpack_post.cmake"
|
||||
|
||||
app_id = "Blender"
|
||||
final_package_name = get_package_name(builder)
|
||||
# MSI needs the app id for the Windows menu folder name
|
||||
# It will also fail if anything else.
|
||||
cpack_package_name = app_id if pack_format == "msi" else final_package_name
|
||||
|
||||
cmake_cmd = [
|
||||
"cmake",
|
||||
f"-DCPACK_PACKAGE_NAME:STRING={cpack_package_name}",
|
||||
f"-DCPACK_OVERRIDE_PACKAGENAME:STRING={cpack_package_name}",
|
||||
# Only works with ZIP, ignored by MSI
|
||||
# f'-DARCHIVE_FILE:STRING={package_name}',
|
||||
# f'-DCPACK_PACKAGE_FILE_NAME:STRING={cpack_package_name}',
|
||||
f"-DCMAKE_INSTALL_PREFIX:PATH={builder.install_dir}",
|
||||
f"-DPOSTINSTALL_SCRIPT:PATH={post_script_file_path}",
|
||||
".",
|
||||
]
|
||||
builder.call(cmake_cmd)
|
||||
|
||||
worker.utils.info("Packaging Blender files")
|
||||
cpack_cmd = [
|
||||
"cpack",
|
||||
"-G",
|
||||
cpack_type,
|
||||
# '--verbose',
|
||||
"--trace-expand",
|
||||
"-C",
|
||||
builder.build_configuration,
|
||||
"-B",
|
||||
str(builder.package_dir), # CPACK_PACKAGE_DIRECTORY
|
||||
"-P",
|
||||
cpack_package_name,
|
||||
]
|
||||
builder.call(cpack_cmd)
|
||||
|
||||
final_package_file_name = f"{final_package_name}.{package_extension}"
|
||||
final_package_file_path = builder.package_dir / final_package_file_name
|
||||
|
||||
# HACK: Rename files correctly, packages are appended `-windows64` with no option to rename
|
||||
bogus_cpack_file_path = (
|
||||
builder.package_dir / f"{cpack_package_name}-windows64.{package_extension}"
|
||||
)
|
||||
|
||||
if pack_format == "zip":
|
||||
if bogus_cpack_file_path.exists():
|
||||
worker.utils.info(f"Removing bogus file [{bogus_cpack_file_path}]")
|
||||
worker.utils.remove_file(bogus_cpack_file_path)
|
||||
|
||||
source_cpack_file_path = (
|
||||
builder.package_dir
|
||||
/ "_CPack_Packages"
|
||||
/ "Windows"
|
||||
/ "ZIP"
|
||||
/ f"{final_package_file_name}"
|
||||
)
|
||||
worker.utils.info(
|
||||
f"Moving [{source_cpack_file_path}] to [{final_package_file_path}]"
|
||||
)
|
||||
os.rename(source_cpack_file_path, final_package_file_path)
|
||||
else:
|
||||
os.rename(bogus_cpack_file_path, final_package_file_path)
|
||||
version_info = worker.blender.version.VersionInfo(builder)
|
||||
description = f"Blender {version_info.version}"
|
||||
worker.blender.sign.sign_windows_files(
|
||||
builder.service_env_id, [final_package_file_path], description=description
|
||||
)
|
||||
|
||||
generate_file_hash(final_package_file_path)
|
||||
|
||||
|
||||
def pack_linux(builder: worker.blender.CodeBuilder) -> None:
|
||||
blender_executable = builder.install_dir / "blender"
|
||||
|
||||
version_info = worker.blender.version.VersionInfo(builder)
|
||||
|
||||
# Strip all unused symbols from the binaries
|
||||
worker.utils.info("Stripping binaries")
|
||||
builder.call(["strip", "--strip-all", blender_executable])
|
||||
|
||||
worker.utils.info("Stripping python")
|
||||
|
||||
# This should work for 3.0, but for now it is in 3.00
|
||||
py_target = builder.install_dir / version_info.short_version
|
||||
if not os.path.exists(py_target):
|
||||
# Support older format and current issue with 3.00
|
||||
py_target = builder.install_dir / (
|
||||
"%d.%02d" % (version_info.major, version_info.minor)
|
||||
)
|
||||
|
||||
worker.utils.call(
|
||||
["find", py_target, "-iname", "*.so", "-exec", "strip", "-s", "{}", ";"]
|
||||
)
|
||||
|
||||
package_name = get_package_name(builder)
|
||||
package_file_name = f"{package_name}.tar.xz"
|
||||
package_file_path = builder.package_dir / package_file_name
|
||||
|
||||
worker.utils.info(f"Creating [{package_file_path}] archive")
|
||||
|
||||
os.makedirs(builder.package_dir, exist_ok=True)
|
||||
|
||||
create_tar_xz(builder.install_dir, package_file_path, package_name)
|
||||
|
||||
generate_file_hash(package_file_path)
|
||||
|
||||
|
||||
def pack_python_module(builder: worker.blender.CodeBuilder) -> None:
|
||||
cleanup_files(builder.package_dir, ".whl")
|
||||
cleanup_files(builder.package_dir, ".zip")
|
||||
|
||||
package_name = get_package_name(builder) + ".zip"
|
||||
package_filepath = builder.package_dir / package_name
|
||||
pack_script = builder.blender_dir / "build_files" / "utils" / "make_bpy_wheel.py"
|
||||
|
||||
# Make wheel
|
||||
worker.utils.info("Packaging Python Wheel")
|
||||
cmd = [sys.executable, pack_script, builder.install_dir]
|
||||
cmd += ["--build-dir", builder.build_dir]
|
||||
cmd += ["--output-dir", builder.package_dir]
|
||||
builder.call(cmd)
|
||||
|
||||
# Pack wheel in zip, until pipeline and www can deal with .whl files.
|
||||
import zipfile
|
||||
|
||||
with zipfile.ZipFile(package_filepath, "w") as zipf:
|
||||
for whl_name in os.listdir(builder.package_dir):
|
||||
if whl_name.endswith(".whl"):
|
||||
whl_filepath = builder.package_dir / whl_name
|
||||
zipf.write(whl_filepath, arcname=whl_name)
|
||||
|
||||
cleanup_files(builder.package_dir, ".whl")
|
||||
|
||||
generate_file_hash(package_filepath)
|
||||
|
||||
|
||||
def pack(builder: worker.blender.CodeBuilder) -> None:
|
||||
builder.setup_build_environment()
|
||||
|
||||
# Create clean package directory
|
||||
worker.utils.remove_dir(builder.package_dir)
|
||||
os.makedirs(builder.package_dir, exist_ok=True)
|
||||
|
||||
# Make sure install directory always exists
|
||||
os.makedirs(builder.install_dir, exist_ok=True)
|
||||
|
||||
if builder.python_module:
|
||||
pack_python_module(builder)
|
||||
elif builder.platform == "darwin":
|
||||
pack_mac(builder)
|
||||
elif builder.platform == "windows":
|
||||
pack_win(builder, "zip")
|
||||
if builder.track_id not in ["vdev", "vexp"]:
|
||||
pack_win(builder, "msi")
|
||||
elif builder.platform == "linux":
|
||||
pack_linux(builder)
|
||||
Loading…
Add table
Add a link
Reference in a new issue