bartvdbraak/builder.braak.pro
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Formatting and linting fixes

Browse source
This commit is contained in:
Bart van der Braak 2024-11-19 21:59:53 +01:00
parent 0a1454d250
commit d6bce1b39d
32 changed files with 675 additions and 268 deletions
Download patch file Download diff file Expand all files Collapse all files

65
config/worker/deploy/artifacts.py
View file

@ -19,10 +19,6 @@ checksums = ["md5", "sha256"]
def pull(builder: worker.deploy.CodeDeployBuilder) -> None:
retry_count = 0
retry_delay_in_seconds = 30
timeout_in_seconds = 60
pipeline_category = "daily"
if builder.track_id == "vexp":
pipeline_category = "experimental"
@ -75,7 +71,11 @@ def pull(builder: worker.deploy.CodeDeployBuilder) -> None:
# Prefer more stable builds, to avoid issue when multiple are present.
risk_id_order = ["stable", "candidate", "rc", "beta", "alpha", "edge"]
risk = build["risk_id"]
risk = risk_id_order.index(risk) if risk in risk_id_order else len(risk_id_order)
risk = (
risk_id_order.index(risk)
if risk in risk_id_order
else len(risk_id_order)
)
other_risk = unique_builds[key]["risk_id"]
other_risk = (
risk_id_order.index(other_risk)
@ -92,7 +92,9 @@ def pull(builder: worker.deploy.CodeDeployBuilder) -> None:
builds = list(unique_builds.values())
if len(builds) == 0:
raise Exception(f"No builds found for version [{version_info.version}] in [{search_url}]")
raise Exception(
f"No builds found for version [{version_info.version}] in [{search_url}]"
)
# Download builds.
worker.utils.remove_dir(builder.download_dir)
@ -113,7 +115,9 @@ def pull(builder: worker.deploy.CodeDeployBuilder) -> None:
# Moving to build_package folder
worker.utils.info(f"Move to [{builder.package_dir}]")
worker.utils.move(download_file_path, builder.package_dir / download_file_path.name)
worker.utils.move(
download_file_path, builder.package_dir / download_file_path.name
)
worker.utils.remove_dir(builder.download_dir)
@ -164,7 +168,9 @@ def repackage(builder: worker.deploy.CodeDeployBuilder) -> None:
if file_extension == "zip":
worker.utils.info(f"Renaming internal folder to [{new_folder_name}]")
worker.utils.call(["7z", "rn", dest_file_path, current_folder_name, new_folder_name])
worker.utils.call(
["7z", "rn", dest_file_path, current_folder_name, new_folder_name]
)
elif file_extension == "tar.xz":
worker.utils.info(f"Extracting [{source_file_path}] to [{dest_file_path}]")
worker.utils.call(["tar", "-xf", source_file_path, "--directory", "."])
@ -198,11 +204,15 @@ def repackage(builder: worker.deploy.CodeDeployBuilder) -> None:
for checksum in checksums:
checksum_text = ""
for filepath in checksum_file_paths:
checksum_line = worker.utils.check_output([f"{checksum}sum", filepath.name]).strip()
checksum_line = worker.utils.check_output(
[f"{checksum}sum", filepath.name]
).strip()
checksum_text += checksum_line + "\n"
print(checksum_text)
checksum_filepath = deployable_path / f"blender-{version_info.version}.{checksum}"
checksum_filepath = (
deployable_path / f"blender-{version_info.version}.{checksum}"
)
checksum_filepath.write_text(checksum_text)
@ -218,34 +228,53 @@ def deploy(builder: worker.deploy.CodeDeployBuilder) -> None:
if builder.service_env_id != "PROD":
# Already assumed to exist on production
worker.utils.call_ssh(connect_id, ["mkdir", "-p", remote_dest_path], dry_run=dry_run)
worker.utils.call_ssh(
connect_id, ["mkdir", "-p", remote_dest_path], dry_run=dry_run
)
for source_path in builder.package_source_dir.iterdir():
dest_path = f"{connect_id}:{remote_dest_path}/"
worker.utils.info(f"Deploying source package [{source_path}]")
worker.utils.rsync(
source_path, dest_path, change_modes=change_modes, show_names=True, dry_run=dry_run
source_path,
dest_path,
change_modes=change_modes,
show_names=True,
dry_run=dry_run,
)
worker.utils.call_ssh(connect_id, ["ls", "-al", f"{remote_dest_path}/"], dry_run=dry_run)
worker.utils.call_ssh(
connect_id, ["ls", "-al", f"{remote_dest_path}/"], dry_run=dry_run
)
# Copy binaries
version_info = worker.blender.version.VersionInfo(builder)
major_minor_version = version_info.short_version
remote_dest_path = (
pathlib.Path(worker_config.download_release_folder) / f"Blender{major_minor_version}"
pathlib.Path(worker_config.download_release_folder)
/ f"Blender{major_minor_version}"
)
deployable_path = builder.package_dir / "deployable"
change_modes = ["F0444"]
worker.utils.call_ssh(connect_id, ["mkdir", "-p", remote_dest_path], dry_run=dry_run)
worker.utils.call_ssh(connect_id, ["ls", "-al", f"{remote_dest_path}/"], dry_run=dry_run)
worker.utils.call_ssh(
connect_id, ["mkdir", "-p", remote_dest_path], dry_run=dry_run
)
worker.utils.call_ssh(
connect_id, ["ls", "-al", f"{remote_dest_path}/"], dry_run=dry_run
)
for source_path in deployable_path.iterdir():
dest_path = f"{connect_id}:{remote_dest_path}/"
worker.utils.info(f"Deploying binary package [{source_path}]")
worker.utils.rsync(
source_path, dest_path, change_modes=change_modes, show_names=True, dry_run=dry_run
source_path,
dest_path,
change_modes=change_modes,
show_names=True,
dry_run=dry_run,
)
worker.utils.call_ssh(connect_id, ["ls", "-al", f"{remote_dest_path}/"], dry_run=dry_run)
worker.utils.call_ssh(
connect_id, ["ls", "-al", f"{remote_dest_path}/"], dry_run=dry_run
)

14
config/worker/deploy/monitor.py
View file

@ -37,7 +37,9 @@ def monitor(builder: worker.deploy.CodeDeployBuilder) -> None:
stop_on_required_site_found = False
branches_config = builder.get_branches_config()
expected_platforms = branches_config.code_official_platform_architectures[builder.track_id]
expected_platforms = branches_config.code_official_platform_architectures[
builder.track_id
]
expected_file_count = len(worker.deploy.artifacts.checksums)
for expected_platform in expected_platforms:
@ -61,7 +63,9 @@ def monitor(builder: worker.deploy.CodeDeployBuilder) -> None:
print(f"Checking [{search_url}] for version [{version_info.version}]")
# Header to avoid getting permission denied.
request = urllib.request.Request(search_url, headers={"User-Agent": "Mozilla"})
request = urllib.request.Request(
search_url, headers={"User-Agent": "Mozilla"}
)
try:
response = urllib.request.urlopen(request, timeout=5.0)
@ -71,7 +75,7 @@ def monitor(builder: worker.deploy.CodeDeployBuilder) -> None:
text = ""
matches = set(re.findall(file_pattern, text))
found_file_count = len(matches)
len(matches)
for match in matches:
print(f"File [{match}]")
@ -93,7 +97,9 @@ def monitor(builder: worker.deploy.CodeDeployBuilder) -> None:
print("")
print("=" * 80)
print(f"Sites [{found_site_count} of {len(monitored_base_urls)}] have all files")
print(
f"Sites [{found_site_count} of {len(monitored_base_urls)}] have all files"
)
print("=" * 80)
if found_site_count == len(monitored_base_urls):

8
config/worker/deploy/pypi.py
View file

@ -54,7 +54,9 @@ def pull(builder: worker.deploy.CodeDeployBuilder) -> None:
# Check expected platforms
branches_config = builder.get_branches_config()
expected_platforms = branches_config.code_official_platform_architectures[builder.track_id]
expected_platforms = branches_config.code_official_platform_architectures[
builder.track_id
]
if len(expected_platforms) != len(matching_builds):
platform_names = "\n".join(expected_platforms)
raise Exception("Unexpected number of builds, expected:\n" + platform_names)
@ -81,7 +83,9 @@ def deliver(builder: worker.deploy.CodeDeployBuilder) -> None:
# Check expected platforms
branches_config = builder.get_branches_config()
expected_platforms = branches_config.code_official_platform_architectures[builder.track_id]
expected_platforms = branches_config.code_official_platform_architectures[
builder.track_id
]
wheel_names = "\n".join([wheel.name for wheel in wheels])
wheel_paths = [str(wheel) for wheel in wheels]
print(wheel_names)

30
config/worker/deploy/snap.py
View file

@ -51,14 +51,20 @@ def package(builder: worker.deploy.CodeStoreBuilder) -> None:
snap_source_root_path = freedesktop_path / "snap"
blender_icon_file_name = "blender.svg"
snapcraft_template_file_path = snap_source_root_path / "blender-snapcraft-template.yaml"
snapcraft_template_file_path = (
snap_source_root_path / "blender-snapcraft-template.yaml"
)
worker.utils.info(f"Using snap config file [{snapcraft_template_file_path}]")
snapcraft_text = snapcraft_template_file_path.read_text()
snapcraft_text = snapcraft_text.replace("@VERSION@", version_info.version)
snapcraft_text = snapcraft_text.replace("@GRADE@", grade)
snapcraft_text = snapcraft_text.replace("@ICON_PATH@", f"./{blender_icon_file_name}")
snapcraft_text = snapcraft_text.replace("@PACKAGE_PATH@", f"./{linux_package_file_path.name}")
snapcraft_text = snapcraft_text.replace(
"@ICON_PATH@", f"./{blender_icon_file_name}"
)
snapcraft_text = snapcraft_text.replace(
"@PACKAGE_PATH@", f"./{linux_package_file_path.name}"
)
snapcraft_file_path = builder.store_snap_dir / "snapcraft.yaml"
worker.utils.info(f"Saving snapcraft config file [{snapcraft_file_path}]")
@ -74,7 +80,9 @@ def package(builder: worker.deploy.CodeStoreBuilder) -> None:
os.chdir(builder.store_snap_dir)
# Copy all required files into working folder
source_file_path = freedesktop_path / "icons" / "scalable" / "apps" / blender_icon_file_name
source_file_path = (
freedesktop_path / "icons" / "scalable" / "apps" / blender_icon_file_name
)
dest_file_path = builder.store_snap_dir / "blender.svg"
worker.utils.info(f"Copy file [{source_file_path}] -> [{dest_file_path}]")
worker.utils.copy_file(source_file_path, dest_file_path)
@ -87,7 +95,8 @@ def package(builder: worker.deploy.CodeStoreBuilder) -> None:
worker.utils.call(["snapcraft", "clean", "--use-lxd"], dry_run=dry_run)
worker.utils.call(["snapcraft", "--use-lxd"], dry_run=dry_run)
worker.utils.call(
["review-tools.snap-review", snap_package_file_path, "--allow-classic"], dry_run=dry_run
["review-tools.snap-review", snap_package_file_path, "--allow-classic"],
dry_run=dry_run,
)
if dry_run:
@ -110,11 +119,14 @@ def deliver(builder: worker.deploy.CodeStoreBuilder) -> None:
branches_config = builder.get_branches_config()
is_lts = builder.track_id in branches_config.all_lts_tracks
is_latest = (
branches_config.track_major_minor_versions[builder.track_id] == version_info.short_version
branches_config.track_major_minor_versions[builder.track_id]
== version_info.short_version
)
# Never push to stable
snap_risk_id = version_info.risk_id.replace("stable", "candidate").replace("alpha", "edge")
snap_risk_id = version_info.risk_id.replace("stable", "candidate").replace(
"alpha", "edge"
)
if snap_risk_id == "stable":
raise Exception("Delivery to [stable] channel not allowed")
@ -139,7 +151,9 @@ def deliver(builder: worker.deploy.CodeStoreBuilder) -> None:
worker_config = builder.get_worker_config()
env = os.environ.copy()
env["SNAPCRAFT_STORE_CREDENTIALS"] = worker_config.snap_credentials(builder.service_env_id)
env["SNAPCRAFT_STORE_CREDENTIALS"] = worker_config.snap_credentials(
builder.service_env_id
)
# If this fails, then the permissions were not set correcty with acls
worker.utils.call(["snapcraft", "status", "blender"], dry_run=dry_run, env=env)

4
config/worker/deploy/source.py
View file

@ -9,7 +9,9 @@ import worker.deploy
import worker.utils
def _package(builder: worker.deploy.CodeDeployBuilder, needs_complete: bool = False) -> None:
def _package(
builder: worker.deploy.CodeDeployBuilder, needs_complete: bool = False
) -> None:
os.chdir(builder.code_path)
if needs_complete:
worker.utils.call(["make", "source_archive_complete"])

25
config/worker/deploy/steam.py
View file

@ -13,7 +13,9 @@ import worker.utils
def extract_file(
builder: worker.deploy.CodeStoreBuilder, source_file_path: pathlib.Path, platform: str
builder: worker.deploy.CodeStoreBuilder,
source_file_path: pathlib.Path,
platform: str,
) -> None:
worker.utils.info(f"Extracting artifact [{source_file_path}] for Steam")
if not source_file_path.exists():
@ -33,7 +35,9 @@ def extract_file(
# Move any folder there as ./content
for source_content_path in dest_extract_path.iterdir():
if source_content_path.is_dir():
worker.utils.info(f"Move [{source_content_path.name}] -> [{dest_content_path}]")
worker.utils.info(
f"Move [{source_content_path.name}] -> [{dest_content_path}]"
)
worker.utils.move(source_content_path, dest_content_path)
break
@ -55,8 +59,12 @@ def extract_file(
worker.utils.remove_file(image_file_path)
worker.utils.info(f"Move Blender app from [{source_content_path}] -> [{dest_content_path}]")
worker.utils.move(source_content_path / "Blender.app", dest_content_path / "Blender.app")
worker.utils.info(
f"Move Blender app from [{source_content_path}] -> [{dest_content_path}]"
)
worker.utils.move(
source_content_path / "Blender.app", dest_content_path / "Blender.app"
)
worker.utils.remove_dir(source_content_path)
elif platform == "windows":
worker.utils.info(f"Extracting zip file [{source_file_path}]")
@ -66,7 +74,9 @@ def extract_file(
# Move any folder there as ./content
for source_content_path in dest_extract_path.iterdir():
if source_content_path.is_dir():
worker.utils.info(f"Move [{source_content_path.name}] -> [{dest_content_path}]")
worker.utils.info(
f"Move [{source_content_path.name}] -> [{dest_content_path}]"
)
worker.utils.move(source_content_path, dest_content_path)
break
else:
@ -97,9 +107,10 @@ def build(builder: worker.deploy.CodeStoreBuilder, is_preview: bool) -> None:
version_info = worker.blender.version.VersionInfo(builder)
branches_config = builder.get_branches_config()
is_lts = builder.track_id in branches_config.all_lts_tracks
is_latest = branches_config.track_major_minor_versions["vdev"] == version_info.short_version
is_latest = (
branches_config.track_major_minor_versions["vdev"] == version_info.short_version
)
track_path = builder.track_path
log_path = builder.track_path / "log"
worker.utils.remove_dir(log_path)
os.makedirs(log_path, exist_ok=True)

12
config/worker/deploy/windows.py
View file

@ -52,12 +52,16 @@ def _package_architecture(
input_file_path = builder.package_dir / build["file_name"]
break
if not input_file_path:
raise Exception(f"Windows package not found in [{builder.package_dir}] manifest")
raise Exception(
f"Windows package not found in [{builder.package_dir}] manifest"
)
# Copy all required files into working folder
source_path = builder.code_path / "release" / "windows" / "msix"
dest_path = builder.store_windows_dir
worker.utils.info(f"Copying [{source_path}] -> [{dest_path}] for windows store packaging")
worker.utils.info(
f"Copying [{source_path}] -> [{dest_path}] for windows store packaging"
)
for source_file in source_path.iterdir():
if source_file.name == "README.md":
@ -104,7 +108,9 @@ def package(builder: worker.deploy.CodeStoreBuilder) -> None:
raise Exception("Can only run this on Windows, aborting")
branches_config = builder.get_branches_config()
expected_platforms = branches_config.code_official_platform_architectures[builder.track_id]
expected_platforms = branches_config.code_official_platform_architectures[
builder.track_id
]
for expected_platform in expected_platforms:
if expected_platform.startswith("windows"):