# SPDX-License-Identifier: GPL-2.0-or-later # SPDX-FileCopyrightText: 2011-2024 Blender Authors # import json import os import pathlib import urllib.request from typing import Any, Dict import worker.blender import worker.blender.version import worker.deploy import worker.utils checksums = ["md5", "sha256"] def pull(builder: worker.deploy.CodeDeployBuilder) -> None: pipeline_category = "daily" if builder.track_id == "vexp": pipeline_category = "experimental" log_path = builder.track_path / "log" worker.utils.remove_dir(log_path) os.makedirs(log_path, exist_ok=True) worker.utils.info("Cleaning package directory") worker.utils.remove_dir(builder.package_dir) os.makedirs(builder.package_dir, exist_ok=True) # Fetch builds information. env_base_url = { "LOCAL": "https://builder.blender.org", "UATEST": "https://builder.uatest.blender.org", "PROD": "https://builder.blender.org", } base_url = env_base_url[builder.service_env_id] search_url = f"{base_url}/download/{pipeline_category}?format=json&v=1" worker.utils.info(f"Fetching build JSON from [{search_url}]") builds_response = urllib.request.urlopen(search_url) # TODO -timeout_sec timeout_in_seconds -retry_interval_sec retry_delay_in_seconds -maximum_retry_count retry_count builds_json = json.load(builds_response) # Get builds matching our version. worker.utils.info("Processing build JSON") version_info = worker.blender.version.VersionInfo(builder) unique_builds: Dict[Any, Dict[Any, Any]] = {} for build in builds_json: if build["version"] != version_info.version: continue if build["file_extension"] in checksums: continue # Correct incomplete file extension in JSON. if build["file_name"].endswith(".tar.xz"): build["file_extension"] = "tar.xz" elif build["file_name"].endswith(".tar.gz"): build["file_extension"] = "tar.gz" elif build["file_name"].endswith(".tar.bz2"): build["file_extension"] = "tar.bz2" key = (build["platform"], build["architecture"], build["file_extension"]) if key in unique_builds: # Prefer more stable builds, to avoid issue when multiple are present. risk_id_order = ["stable", "candidate", "rc", "beta", "alpha", "edge"] risk = build["risk_id"] risk = ( risk_id_order.index(risk) if risk in risk_id_order else len(risk_id_order) ) other_risk = unique_builds[key]["risk_id"] other_risk = ( risk_id_order.index(other_risk) if other_risk in risk_id_order else len(risk_id_order) ) if other_risk <= risk: continue else: print(" ".join(key)) unique_builds[key] = build builds = list(unique_builds.values()) if len(builds) == 0: raise Exception( f"No builds found for version [{version_info.version}] in [{search_url}]" ) # Download builds. worker.utils.remove_dir(builder.download_dir) os.makedirs(builder.download_dir, exist_ok=True) for build in builds: file_uri = build["url"] file_name = build["file_name"] worker.utils.info(f"Pull [{file_name}]") download_file_path = builder.download_dir / file_name worker.utils.info(f"Download [{file_uri}]") urllib.request.urlretrieve(file_uri, download_file_path) # TODO: retry and resume # -resume -timeout_sec timeout_in_seconds -retry_interval_sec retry_delay_in_seconds -maximum_retry_count retry_count # Moving to build_package folder worker.utils.info(f"Move to [{builder.package_dir}]") worker.utils.move( download_file_path, builder.package_dir / download_file_path.name ) worker.utils.remove_dir(builder.download_dir) # Write manifest of downloaded packages. package_manifest = builder.package_dir / "manifest.json" package_manifest.write_text(json.dumps(builds, indent=2)) def repackage(builder: worker.deploy.CodeDeployBuilder) -> None: version_info = worker.blender.version.VersionInfo(builder) deployable_path = builder.package_dir / "deployable" worker.utils.remove_dir(deployable_path) os.makedirs(deployable_path, exist_ok=True) os.chdir(deployable_path) package_manifest = builder.package_dir / "manifest.json" builds = json.loads(package_manifest.read_text()) checksum_file_paths = [] # Rename the files and the internal folders for zip and tar.xz files for build in builds: file_name = build["file_name"] file_path = builder.package_dir / file_name worker.utils.info(f"Repackaging {file_name}") if builder.service_env_id == "PROD" and build["risk_id"] != "stable": raise Exception( "Can only repackage and deploy stable versions, found risk id '{build['risk_id']}'" ) version = build["version"] platform = build["platform"].replace("darwin", "macos") architecture = build["architecture"].replace("86_", "").replace("amd", "x") file_extension = build["file_extension"] current_folder_name = file_path.name[: -len("." + file_extension)] new_folder_name = f"blender-{version}-{platform}-{architecture}" new_file_name = f"{new_folder_name}.{file_extension}" source_file_path = file_path dest_file_path = deployable_path / new_file_name worker.utils.info(f"Renaming file [{source_file_path}] to [{dest_file_path}]") worker.utils.copy_file(source_file_path, dest_file_path) if file_extension == "zip": worker.utils.info(f"Renaming internal folder to [{new_folder_name}]") worker.utils.call( ["7z", "rn", dest_file_path, current_folder_name, new_folder_name] ) elif file_extension == "tar.xz": worker.utils.info(f"Extracting [{source_file_path}] to [{dest_file_path}]") worker.utils.call(["tar", "-xf", source_file_path, "--directory", "."]) worker.utils.remove_file(dest_file_path) worker.utils.move( deployable_path / current_folder_name, deployable_path / new_folder_name ) worker.utils.info(f"Compressing [{new_folder_name}] to [{dest_file_path}]") cmd = [ "tar", "-cv", "--owner=0", "--group=0", "--use-compress-program", "xz -6", "-f", dest_file_path, new_folder_name, ] worker.utils.call(cmd) worker.utils.remove_dir(deployable_path / new_folder_name) checksum_file_paths.append(dest_file_path) # Create checksums worker.utils.info("Creating checksums") os.chdir(deployable_path) for checksum in checksums: checksum_text = "" for filepath in checksum_file_paths: checksum_line = worker.utils.check_output( [f"{checksum}sum", filepath.name] ).strip() checksum_text += checksum_line + "\n" print(checksum_text) checksum_filepath = ( deployable_path / f"blender-{version_info.version}.{checksum}" ) checksum_filepath.write_text(checksum_text) def deploy(builder: worker.deploy.CodeDeployBuilder) -> None: # No testable on UATEST currently. dry_run = builder.service_env_id not in ("LOCAL", "PROD") worker_config = builder.get_worker_config() connect_id = f"{worker_config.download_user}@{worker_config.download_machine}" # Copy source remote_dest_path = pathlib.Path(worker_config.download_source_folder) change_modes = ["F0444"] if builder.service_env_id != "PROD": # Already assumed to exist on production worker.utils.call_ssh( connect_id, ["mkdir", "-p", remote_dest_path], dry_run=dry_run ) for source_path in builder.package_source_dir.iterdir(): dest_path = f"{connect_id}:{remote_dest_path}/" worker.utils.info(f"Deploying source package [{source_path}]") worker.utils.rsync( source_path, dest_path, change_modes=change_modes, show_names=True, dry_run=dry_run, ) worker.utils.call_ssh( connect_id, ["ls", "-al", f"{remote_dest_path}/"], dry_run=dry_run ) # Copy binaries version_info = worker.blender.version.VersionInfo(builder) major_minor_version = version_info.short_version remote_dest_path = ( pathlib.Path(worker_config.download_release_folder) / f"Blender{major_minor_version}" ) deployable_path = builder.package_dir / "deployable" change_modes = ["F0444"] worker.utils.call_ssh( connect_id, ["mkdir", "-p", remote_dest_path], dry_run=dry_run ) worker.utils.call_ssh( connect_id, ["ls", "-al", f"{remote_dest_path}/"], dry_run=dry_run ) for source_path in deployable_path.iterdir(): dest_path = f"{connect_id}:{remote_dest_path}/" worker.utils.info(f"Deploying binary package [{source_path}]") worker.utils.rsync( source_path, dest_path, change_modes=change_modes, show_names=True, dry_run=dry_run, ) worker.utils.call_ssh( connect_id, ["ls", "-al", f"{remote_dest_path}/"], dry_run=dry_run )