diff --git a/.forgejo/workflows/check.yml b/.forgejo/workflows/check.yml index 9d564eb..3cd9af4 100644 --- a/.forgejo/workflows/check.yml +++ b/.forgejo/workflows/check.yml @@ -1,4 +1,3 @@ -name: Run checks on: pull_request: branches: diff --git a/config/conf/__init__.py b/config/conf/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/conf/auth.py b/config/conf/auth.py new file mode 100644 index 0000000..93efe66 --- /dev/null +++ b/config/conf/auth.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import importlib + +import buildbot.plugins + + +def _get_auth_config(devops_env_id: str): + if devops_env_id == "LOCAL": + import conf.local.auth + + importlib.reload(conf.local.auth) + return conf.local.auth + else: + import conf.production.auth + + importlib.reload(conf.production.auth) + return conf.production.auth + + +def fetch_authentication(devops_env_id: str): + auth_config = _get_auth_config(devops_env_id) + return auth_config.get_authentication(devops_env_id) + + +def fetch_authorization(devops_env_id: str): + auth_config = _get_auth_config(devops_env_id) + + admin_usernames = auth_config.admin_usernames + deploy_dev_usernames = auth_config.deploy_dev_usernames + trusted_dev_usernames = auth_config.trusted_dev_usernames + + dev_usernames = list(set(deploy_dev_usernames + trusted_dev_usernames + admin_usernames)) + deploy_usernames = list(set(deploy_dev_usernames + admin_usernames)) + + file_based_group_username_role_matchers = [ + buildbot.plugins.util.RolesFromUsername(roles=["admin"], usernames=admin_usernames), + buildbot.plugins.util.RolesFromUsername(roles=["deploy"], usernames=deploy_usernames), + buildbot.plugins.util.RolesFromUsername(roles=["dev"], usernames=dev_usernames), + ] + + my_authz = buildbot.plugins.util.Authz( + stringsMatcher=buildbot.plugins.util.fnmatchStrMatcher, + allowRules=[ + # Admins can do anything, + # + # defaultDeny=False: if user does not have the admin role, we continue + # parsing rules + # buildbot.plugins.util.AnyEndpointMatcher(role='admin', defaultDeny=False), + # buildbot.plugins.util.AnyEndpointMatcher(role='dev', defaultDeny=False), + # buildbot.plugins.util.AnyEndpointMatcher(role='coordinator', defaultDeny=False), + # buildbot.plugins.util.AnyEndpointMatcher(role='anonymous', defaultDeny=False), + buildbot.plugins.util.StopBuildEndpointMatcher(role="dev", defaultDeny=True), + buildbot.plugins.util.RebuildBuildEndpointMatcher(role="dev", defaultDeny=True), + buildbot.plugins.util.EnableSchedulerEndpointMatcher(role="admin", defaultDeny=True), + # buildbot.plugins.util.AnyEndpointMatcher(role='any', defaultDeny=False), + # Force roles + buildbot.plugins.util.ForceBuildEndpointMatcher( + builder="*-code-experimental-*", role="dev", defaultDeny=True + ), + buildbot.plugins.util.ForceBuildEndpointMatcher( + builder="*-code-patch-*", role="dev", defaultDeny=True + ), + buildbot.plugins.util.ForceBuildEndpointMatcher( + builder="*-code-daily-*", role="dev", defaultDeny=True + ), + buildbot.plugins.util.ForceBuildEndpointMatcher( + builder="*-store-*", role="deploy", defaultDeny=True + ), + buildbot.plugins.util.ForceBuildEndpointMatcher( + builder="*-deploy-*", role="deploy", defaultDeny=True + ), + buildbot.plugins.util.ForceBuildEndpointMatcher( + builder="*-doc-*", role="dev", defaultDeny=True + ), + # Rebuild roles + buildbot.plugins.util.RebuildBuildEndpointMatcher( + builder="*-code-experimental-*", role="dev", defaultDeny=True + ), + buildbot.plugins.util.RebuildBuildEndpointMatcher( + builder="*-code-patch-*", role="dev", defaultDeny=True + ), + buildbot.plugins.util.RebuildBuildEndpointMatcher( + builder="*-code-daily-*", role="dev", defaultDeny=True + ), + buildbot.plugins.util.RebuildBuildEndpointMatcher( + builder="*-store-*", role="deploy", defaultDeny=True + ), + buildbot.plugins.util.RebuildBuildEndpointMatcher( + builder="*-deploy-*", role="deploy", defaultDeny=True + ), + buildbot.plugins.util.RebuildBuildEndpointMatcher( + builder="*-doc-*", role="dev", defaultDeny=True + ), + # This also affects starting jobs via force scheduler + buildbot.plugins.util.AnyControlEndpointMatcher(role="admin", defaultDeny=True), + # A default deny for any endpoint if not admin + # If this is missing at the end, any UNMATCHED group will get 'allow'... + buildbot.plugins.util.AnyControlEndpointMatcher(role="admin", defaultDeny=True), + ], + roleMatchers=file_based_group_username_role_matchers, + ) + + return my_authz diff --git a/config/conf/branches.py b/config/conf/branches.py new file mode 100644 index 0000000..fff848c --- /dev/null +++ b/config/conf/branches.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import copy + +from collections import OrderedDict + +# Blender repository branches used for daily builds and API doc generation. +code_tracked_branch_ids = { + "vdev": "main", + "vexp": "", + "v360": "blender-v3.6-release", + "v420": "blender-v4.2-release", + "v430": "blender-v4.3-release", +} + +# Processor architectures to build for each track. +code_official_platform_architectures = { + "vdev": ["darwin-x86_64", "darwin-arm64", "linux-x86_64", "windows-amd64"], + "vexp": ["darwin-x86_64", "darwin-arm64", "linux-x86_64", "windows-amd64"], + "v360": ["darwin-x86_64", "darwin-arm64", "linux-x86_64", "windows-amd64"], + "v420": ["darwin-x86_64", "darwin-arm64", "linux-x86_64", "windows-amd64"], + "v430": ["darwin-x86_64", "darwin-arm64", "linux-x86_64", "windows-amd64"], +} + +# Windows ARM64 not used by default yet. +code_all_platform_architectures = copy.deepcopy(code_official_platform_architectures) +code_all_platform_architectures["vdev"].append("windows-arm64") +code_all_platform_architectures["vexp"].append("windows-arm64") +code_all_platform_architectures["v430"].append("windows-arm64") + +track_major_minor_versions = { + "vdev": "4.4", + "vexp": "4.4", + "v360": "3.6", + "v330": "3.3", + "v420": "4.2", + "v430": "4.3", +} + +# Blender code and manual git branches. +track_code_branches = { + "vdev": "main", + "vexp": "main", + "v360": "blender-v3.6-release", + "v420": "blender-v4.2-release", + "v430": "blender-v4.3-release", +} + +# Tracks that correspond to an LTS version released on the Windows Store. +# Only add entries here AFTER the regular release is out, since it will +# otherwise generate the wrong package for the regular release. +windows_store_lts_tracks = ["v360", "v420"] + +# Tracks that correspond to active and upcoming LTS releases. Used for +# the Snap track name, and for Steam to determine if there is a daily LTS +# track to upload to. +all_lts_tracks = ["v360", "v420"] + +# Tracks for automated delivery of daily builds to stores. +code_store_track_ids = [ + "vdev", + "v360", + "v420", + "v430", +] + +# Tracks to deploy releases (regular and LTS) to download.blender.org. +code_deploy_track_ids = { + "v360": None, + "v420": None, + "v430": None, +} + +# Stable track for manual and API docs. +# Update on release. +doc_stable_major_minor_version = "4.3" + +# Versions and labels for the user manual version switching menu. +# Update when creating new release branch, and on release. +doc_manual_version_labels = OrderedDict( + [ + ("2.79", "2.79"), + ("2.80", "2.80"), + ("2.81", "2.81"), + ("2.82", "2.82"), + ("2.83", "2.83 (LTS)"), + ("2.90", "2.90"), + ("2.91", "2.91"), + ("2.92", "2.92"), + ("2.93", "2.93 (LTS)"), + ("3.0", "3.0"), + ("3.1", "3.1"), + ("3.2", "3.2"), + ("3.3", "3.3 (LTS)"), + ("3.4", "3.4"), + ("3.5", "3.5"), + ("3.6", "3.6 (LTS)"), + ("4.0", "4.0"), + ("4.1", "4.1"), + ("4.2", "4.2 (LTS)"), + ("4.3", "4.3"), + ("4.4", "4.4 (develop)"), + ] +) diff --git a/config/conf/local/__init__.py b/config/conf/local/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/conf/local/auth.py b/config/conf/local/auth.py new file mode 100644 index 0000000..b677be9 --- /dev/null +++ b/config/conf/local/auth.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import buildbot.plugins + +# Buildbot admin with access to everything. +admin_usernames = [ + "admin", +] + +# Release engineers with access to store and deploy builders. +deploy_dev_usernames = [ + "admin", +] + +# Trusted developers with access to trigger daily, doc and patch builds. +trusted_dev_usernames = [ + "admin", +] + + +def get_authentication(devops_env_id: str): + class LocalEnvAuth(buildbot.plugins.util.CustomAuth): + def check_credentials(self, user, password): + return user.decode() == "admin" and password.decode() == "admin" + + return LocalEnvAuth() diff --git a/config/conf/local/machines.py b/config/conf/local/machines.py new file mode 100644 index 0000000..9087e67 --- /dev/null +++ b/config/conf/local/machines.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +_worker_names = { + "code-lint": ["localhost"], + "linux-x86_64-code": ["localhost"], + "linux-x86_64-code-gpu": ["localhost"], + "linux-x86_64-doc-api": ["localhost"], + "linux-x86_64-doc-studio-tools": ["localhost"], + "linux-x86_64-general": ["localhost"], + "linux-x86_64-store-snap": ["localhost"], + "linux-x86_64-store-steam": ["localhost"], + "darwin-arm64-code": ["localhost"], + "darwin-arm64-code-gpu": ["localhost"], + "darwin-x86_64-code": ["localhost"], + "darwin-x86_64-code-gpu": ["localhost"], + "windows-amd64-code": ["localhost"], + "windows-amd64-code-gpu": [], + "windows-amd64-store-windows": ["localhost"], + "windows-arm64-code": ["localhost"], + "windows-arm64-code-gpu": [], +} + + +def get_worker_password(worker_name: str) -> str: + return "localhost" + + +def get_worker_names(devops_env_id: str): + return _worker_names diff --git a/config/conf/local/worker.py b/config/conf/local/worker.py new file mode 100644 index 0000000..e178165 --- /dev/null +++ b/config/conf/local/worker.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import os +import pathlib + +from typing import Optional, Tuple + +# Where tracks data is stored. +tracks_root_path = pathlib.Path.home() / "git" + +# Software cache +software_cache_path = tracks_root_path / "downloads" / "software" / "workers" + +# Docs delivery. +docs_user = os.getlogin() +docs_machine = "127.0.0.1" +docs_folder = tracks_root_path / "delivery" / "docs" +docs_port = 22 + +# Studio docs delivery. +studio_user = os.getlogin() +studio_machine = "127.0.0.1" +studio_folder = tracks_root_path / "delivery" / "studio" / "blender-studio-tools" +studio_port = 22 + +# Download delivery. +download_user = os.getlogin() +download_machine = "127.0.0.1" +download_source_folder = tracks_root_path / "delivery" / "download" / "source" +download_release_folder = tracks_root_path / "delivery" / "download" / "release" +download_port = 22 + +# Buildbot download delivery +buildbot_download_folder = tracks_root_path / "delivery" / "buildbot" + +# Code signing +sign_code_windows_certificate = None # "Blender Self Code Sign SPC" +sign_code_windows_time_servers = ["http://ts.ssl.com"] +sign_code_windows_server_url = "http://fake-windows-sign-server" + +sign_code_darwin_certificate = None +sign_code_darwin_team_id = None +sign_code_darwin_apple_id = None +sign_code_darwin_keychain_profile = None + + +def darwin_keychain_password(service_env_id: str) -> str: + return "fake_keychain_password" + + +# Steam +steam_app_id = None +steam_platform_depot_ids = { + "windows": None, + "linux": None, + "darwin": None, +} + + +def steam_credentials(service_env_id: str) -> Tuple[str, str]: + return "fake_steam_username", "fake_steam_password" + + +# Snap +def snap_credentials(service_env_id: str) -> str: + return "fake_snap_credentials" + + +# Windows Store +windows_store_self_sign = False + + +def windows_store_certificate(service_env_id: str) -> str: + # return sign_code_windows_certificate + return "fake_windows_store_publisher" + + +# PyPI +def pypi_token(service_env_id: str) -> str: + return "fake_pypi_token" + + +# Gitea +def gitea_api_token(service_env_id: str) -> Optional[str]: + return None diff --git a/config/conf/machines.py b/config/conf/machines.py new file mode 100644 index 0000000..55b1aa5 --- /dev/null +++ b/config/conf/machines.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import importlib + + +def _get_config(devops_env_id: str): + if devops_env_id == "LOCAL": + import conf.local.machines + + importlib.reload(conf.local.machines) + return conf.local.machines + else: + import conf.production.machines + + importlib.reload(conf.production.machines) + return conf.production.machines + + +def fetch_platform_worker_names(devops_env_id: str): + machines_config = _get_config(devops_env_id) + return machines_config.get_worker_names(devops_env_id) + + +def get_worker_password(devops_env_id: str, worker_name: str) -> str: + machines_config = _get_config(devops_env_id) + return machines_config.get_worker_password(worker_name) + + +def fetch_local_worker_names(): + worker_names = [] + worker_numbers = range(1, 5, 1) + for worker_number in worker_numbers: + worker_id = str(worker_number).zfill(2) + worker_name = f"local-coordinator-{worker_id}" + worker_names += [worker_name] + + return worker_names diff --git a/config/conf/worker.py b/config/conf/worker.py new file mode 100644 index 0000000..963becf --- /dev/null +++ b/config/conf/worker.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import importlib + +from typing import Any + + +def get_config(devops_env_id: str) -> Any: + if devops_env_id == "LOCAL": + import conf.local.worker + + importlib.reload(conf.local.worker) + return conf.local.worker + else: + import conf.production.worker + + importlib.reload(conf.production.worker) + return conf.production.worker + + +# Maybe useful in the future. +# +# import pathlib +# import importlib.util +# +# def _load_module_config(path: pathlib.Path) -> Any: +# filepath = pathlib.Path(__file__).parent / path +# spec = importlib.util.spec_from_file_location("config_module", filepath) +# if not spec: +# raise BaseException("Failed to load config module spec") +# config_module = importlib.util.module_from_spec(spec) +# if not spec.loader: +# raise BaseException("Failed to load config module spec loader") +# spec.loader.exec_module(config_module) +# return config_module diff --git a/config/gitea/LICENSE b/config/gitea/LICENSE new file mode 100644 index 0000000..be288ed --- /dev/null +++ b/config/gitea/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 LAB132 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/config/gitea/README.md b/config/gitea/README.md new file mode 100644 index 0000000..7d7eb6a --- /dev/null +++ b/config/gitea/README.md @@ -0,0 +1,4 @@ +### Buildbot Gitea Integration + +Based on: +https://github.com/lab132/buildbot-gitea diff --git a/config/gitea/__init__.py b/config/gitea/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/gitea/blender.py b/config/gitea/blender.py new file mode 100644 index 0000000..8f53811 --- /dev/null +++ b/config/gitea/blender.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2018 LAB132 +# SPDX-FileCopyrightText: 2013-2024 Blender Authors +# + +# Based on the gitlab reporter from buildbot + +from twisted.python import log + +import buildbot.plugins + +import importlib +import requests + +import gitea.reporter + +importlib.reload(gitea.reporter) + +# Create status reporter service. +gitea_url = "https://projects.blender.org" +gitea_api_token = None +gitea_status_service = None + + +def setup_service(devops_env_id: str): + import conf.worker + + importlib.reload(conf.worker) + worker_config = conf.worker.get_config(devops_env_id) + gitea_api_token = worker_config.gitea_api_token(devops_env_id) + + if gitea_api_token: + log.msg("Found Gitea API token, enabling status push") + return gitea.reporter.GiteaStatusService11(gitea_url, gitea_api_token, verbose=False) + else: + log.msg("No Gitea API token found, status push disabled") + return None + + +# Get revision for coordinator. +@buildbot.plugins.util.renderer +def get_patch_revision(props): + if "revision" in props and props["revision"]: + return {} + if "pull_revision" in props and props["pull_revision"]: + return {"revision": props["pull_revision"]} + pull_id = props["patch_id"] + url = f"{gitea_url}/api/v1/repos/blender/blender/pulls/{pull_id}" + response = requests.get(url, headers={"accept": "application/json"}) + sha = response.json().get("head", {"sha": ""}).get("sha") + return {"revision": sha} + + +@buildbot.plugins.util.renderer +def get_branch_revision(props): + if "revision" in props and props["revision"]: + return {} + branch = props["override_branch_id"] + url = f"{gitea_url}/api/v1/repos/blender/blender/git/commits/{branch}" + response = requests.get(url, headers={"accept": "application/json"}) + sha = response.json().get("sha", "") + return {"revision": sha} diff --git a/config/gitea/reporter.py b/config/gitea/reporter.py new file mode 100644 index 0000000..1e1f610 --- /dev/null +++ b/config/gitea/reporter.py @@ -0,0 +1,279 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2018 LAB132 +# SPDX-FileCopyrightText: 2013-2024 Blender Authors +# + +# Based on the gitlab reporter from buildbot + +from __future__ import absolute_import +from __future__ import print_function + +from twisted.internet import defer +from twisted.python import log + +from buildbot.process.properties import Interpolate +from buildbot.process.properties import Properties +from buildbot.process.results import CANCELLED +from buildbot.process.results import EXCEPTION +from buildbot.process.results import FAILURE +from buildbot.process.results import RETRY +from buildbot.process.results import SKIPPED +from buildbot.process.results import SUCCESS +from buildbot.process.results import WARNINGS +from buildbot.reporters import http +from buildbot.util import httpclientservice +from buildbot.reporters.generators.build import BuildStartEndStatusGenerator +from buildbot.reporters.message import MessageFormatterRenderable + + +import re + + +# This name has a number in it to trick buildbot into reloading it on without +# restart. Needs to be incremented every time this file is changed. Is there +# a better solution? +class GiteaStatusService11(http.ReporterBase): + name = "GiteaStatusService11" + ssh_url_match = re.compile( + r"(ssh://)?[\w+\-\_]+@[\w\.\-\_]+:?(\d*/)?(?P[\w_\-\.]+)/(?P[\w_\-\.]+?)(\.git)?$" + ) + + def checkConfig( + self, + baseURL, + token, + context=None, + context_pr=None, + verbose=False, + debug=None, + verify=None, + generators=None, + warningAsSuccess=False, + **kwargs, + ): + if generators is None: + generators = self._create_default_generators() + + super().checkConfig(generators=generators, **kwargs) + httpclientservice.HTTPClientService.checkAvailable(self.__class__.__name__) + + @defer.inlineCallbacks + def reconfigService( + self, + baseURL, + token, + context=None, + context_pr=None, + verbose=False, + debug=None, + verify=None, + generators=None, + warningAsSuccess=False, + **kwargs, + ): + token = yield self.renderSecrets(token) + self.debug = debug + self.verify = verify + self.verbose = verbose + if generators is None: + generators = self._create_default_generators() + + yield super().reconfigService(generators=generators, **kwargs) + + self.context = context or Interpolate("buildbot/%(prop:buildername)s") + self.context_pr = context_pr or Interpolate("buildbot/pull_request/%(prop:buildername)s") + if baseURL.endswith("/"): + baseURL = baseURL[:-1] + self.baseURL = baseURL + self._http = yield httpclientservice.HTTPClientService.getService( + self.master, + baseURL, + headers={"Authorization": "token {}".format(token)}, + debug=self.debug, + verify=self.verify, + ) + self.verbose = verbose + self.project_ids = {} + self.warningAsSuccess = warningAsSuccess + + def _create_default_generators(self): + start_formatter = MessageFormatterRenderable("Build started.") + end_formatter = MessageFormatterRenderable("Build done.") + + return [ + BuildStartEndStatusGenerator( + start_formatter=start_formatter, end_formatter=end_formatter + ) + ] + + def createStatus( + self, project_owner, repo_name, sha, state, target_url=None, description=None, context=None + ): + """ + :param project_owner: username of the owning user or organization + :param repo_name: name of the repository + :param sha: Full sha to create the status for. + :param state: one of the following 'pending', 'success', 'failed' + or 'cancelled'. + :param target_url: Target url to associate with this status. + :param description: Short description of the status. + :param context: Context of the result + :return: A deferred with the result from GitLab. + + """ + payload = {"state": state} + + if description is not None: + payload["description"] = description + + if target_url is not None: + payload["target_url"] = target_url + + if context is not None: + payload["context"] = context + + url = "/api/v1/repos/{owner}/{repository}/statuses/{sha}".format( + owner=project_owner, repository=repo_name, sha=sha + ) + log.msg(f"Sending status to {url}: {payload}") + + return self._http.post(url, json=payload) + + @defer.inlineCallbacks + def sendMessage(self, reports): + yield self._send_impl(reports) + + @defer.inlineCallbacks + def _send_status( + self, build, repository_owner, repository_name, sha, state, context, description + ): + try: + target_url = build["url"] + res = yield self.createStatus( + project_owner=repository_owner, + repo_name=repository_name, + sha=sha, + state=state, + target_url=target_url, + context=context, + description=description, + ) + if res.code not in (200, 201, 204): + message = yield res.json() + message = message.get("message", "unspecified error") + log.msg( + 'Could not send status "{state}" for ' + "{repo} at {sha}: {code} : {message}".format( + state=state, repo=repository_name, sha=sha, code=res.code, message=message + ) + ) + elif self.verbose: + log.msg( + 'Status "{state}" sent for ' + "{repo} at {sha}.".format(state=state, repo=repository_name, sha=sha) + ) + except Exception as e: + log.err( + e, + 'Failed to send status "{state}" for ' + "{repo} at {sha}".format(state=state, repo=repository_name, sha=sha), + ) + + @defer.inlineCallbacks + def _send_impl(self, reports): + for report in reports: + try: + builds = report["builds"] + except KeyError: + continue + + for build in builds: + builder_name = build["builder"]["name"] + + props = Properties.fromDict(build["properties"]) + props.master = self.master + + description = report.get("body", None) + + if build["complete"]: + state = { + SUCCESS: "success", + WARNINGS: "success" if self.warningAsSuccess else "warning", + FAILURE: "failure", + SKIPPED: "success", + EXCEPTION: "error", + RETRY: "pending", + CANCELLED: "error", + }.get(build["results"], "failure") + else: + state = "pending" + + if "pr_id" in props: + context = yield props.render(self.context_pr) + else: + context = yield props.render(self.context) + + sourcestamps = build["buildset"]["sourcestamps"] + + # BLENDER: some hardcoded logic for now. + if ( + "-code-daily-" in builder_name + or "-code-patch-" in builder_name + or "-code-experimental-" in builder_name + ): + repository_owner = "blender" + repository_name = "blender" + elif "-doc-manual-" in builder_name: + repository_owner = "blender" + repository_name = "blender-manual" + elif "-doc-developer" in builder_name: + repository_owner = "blender" + repository_name = "blender-developer-docs" + else: + continue + + # Source change from Git poller. + for sourcestamp in sourcestamps: + sha = sourcestamp["revision"] + if sha not in {None, "", "HEAD"}: + self._send_status( + build, + repository_owner, + repository_name, + sha, + state, + context, + description, + ) + continue + + # Revision specified by get-revision step. + if "revision" in props: + sha = props["revision"] + if sha not in {None, "", "HEAD"}: + self._send_status( + build, + repository_owner, + repository_name, + sha, + state, + context, + description, + ) + + # Revision from blender-bot, so we can send a status before + # the get-revision step runs. + if "pull_revision" in props: + sha = props["pull_revision"] + if sha not in {None, "", "HEAD"}: + self._send_status( + build, + repository_owner, + repository_name, + sha, + state, + context, + description, + ) + + continue diff --git a/config/pipeline/__init__.py b/config/pipeline/__init__.py new file mode 100644 index 0000000..5ee14bc --- /dev/null +++ b/config/pipeline/__init__.py @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import importlib + +from buildbot.plugins import changes as plugins_changes + +import conf.branches + +import pipeline.common +import pipeline.code +import pipeline.code_benchmark +import pipeline.code_deploy +import pipeline.code_bpy_deploy +import pipeline.code_store +import pipeline.doc_api +import pipeline.doc_manual +import pipeline.doc_developer +import pipeline.doc_studio + +importlib.reload(pipeline.common) +importlib.reload(conf.branches) + + +def populate(devops_env_id): + pipelines_modules = [ + pipeline.code, + pipeline.code_benchmark, + pipeline.code_deploy, + pipeline.code_bpy_deploy, + pipeline.code_store, + pipeline.doc_api, + pipeline.doc_manual, + pipeline.doc_developer, + pipeline.doc_studio, + ] + + builders = [] + schedulers = [] + + for pipelines_module in pipelines_modules: + importlib.reload(pipelines_module) + b, s = pipelines_module.populate(devops_env_id) + builders += b + schedulers += s + + return builders, schedulers + + +def change_sources(): + branch_ids = list(conf.branches.code_tracked_branch_ids.values()) + + pollers = [] + poll_interval_in_seconds = 2 * 60 + + pollers += [ + plugins_changes.GitPoller( + repourl="https://projects.blender.org/blender/blender.git", + pollAtLaunch=True, + pollInterval=poll_interval_in_seconds, + workdir="blender-gitpoller-workdir", + project="blender.git", + branches=branch_ids, + ) + ] + + pollers += [ + plugins_changes.GitPoller( + repourl="https://projects.blender.org/blender/blender-manual.git", + pollAtLaunch=True, + pollInterval=poll_interval_in_seconds, + workdir="blender-manual-gitpoller-workdir", + project="blender-manual.git", + branches=branch_ids, + ) + ] + + pollers += [ + plugins_changes.GitPoller( + repourl="https://projects.blender.org/blender/blender-developer-docs.git", + pollAtLaunch=True, + pollInterval=poll_interval_in_seconds, + workdir="blender-developer-docs-gitpoller-workdir", + project="blender-developer-docs.git", + branches=["main"], + ) + ] + + pollers += [ + plugins_changes.GitPoller( + repourl="https://projects.blender.org/studio/blender-studio-tools.git", + pollAtLaunch=True, + pollInterval=poll_interval_in_seconds, + workdir="blender-studio-tools-gitpoller-workdir", + project="blender-studio-tools.git", + branches=["main"], + ) + ] + + return pollers diff --git a/config/pipeline/code.py b/config/pipeline/code.py new file mode 100644 index 0000000..69d0ef2 --- /dev/null +++ b/config/pipeline/code.py @@ -0,0 +1,748 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +from functools import partial +import pathlib +import random + +import buildbot.plugins + +from buildbot.plugins import steps as plugins_steps +from buildbot.plugins import schedulers as plugins_schedulers + +import conf.branches +import conf.worker + +import pipeline.common +import gitea.reporter + +# Timeouts. +default_step_timeout_in_seconds = 10 * 60 +# TODO: Compile step needs more because of the link on Windows +compile_code_step_timeout_in_seconds = 10 * 60 +compile_gpu_step_timeout_in_seconds = 1.5 * 60 * 60 + +tree_stable_timer_in_seconds = 15 * 60 + +package_step_timeout_in_seconds = 20 * 60 + +# Build steps. +code_pipeline_general_step_names = [ + "configure-machine", + "update-code", + "compile-code", + "compile-gpu", + "compile-install", + "test-code", + "sign-code-binaries", + "package-code-binaries", + "deliver-code-binaries", + "deliver-test-results", + "clean", +] + +code_pipeline_daily_step_names = code_pipeline_general_step_names + +code_pipeline_patch_step_names = [ + "configure-machine", + "update-code", + "compile-code", + "compile-gpu", + "compile-install", + "test-code", + "sign-code-binaries", + "package-code-binaries", + "deliver-code-binaries", + "deliver-test-results", + "clean", +] + +code_pipeline_experimental_step_names = code_pipeline_general_step_names + +pipeline_types_step_names = { + "daily": code_pipeline_daily_step_names, + "patch": code_pipeline_patch_step_names, + "experimental": code_pipeline_experimental_step_names, +} + +code_pipeline_lint_step_names = [ + "configure-machine", + "update-code", + "lint-code", +] + +# Steps for testing. +code_pipeline_test_step_names = [ + "test-code", +] + +# Steps for package delivery. +code_delivery_step_names = [ + "sign-code-binaries", + "package-code-binaries", + "deliver-code-binaries", +] + +# Steps skipped for Python module. +code_python_module_skip_test_names = ["sign-code-binaries"] + + +# Tracks. +code_tracked_branch_ids = conf.branches.code_tracked_branch_ids +code_track_ids = list(code_tracked_branch_ids.keys()) +code_all_platform_architectures = conf.branches.code_all_platform_architectures +code_official_platform_architectures = conf.branches.code_official_platform_architectures + +code_track_pipeline_types = {} +track_properties = {} +for track, branch in code_tracked_branch_ids.items(): + if track == "vdev": + code_track_pipeline_types[track] = ["daily"] + elif track == "vexp": + code_track_pipeline_types[track] = ["experimental", "patch"] + else: + code_track_pipeline_types[track] = ["daily"] + + # Track properties. + track_properties[track] = [ + buildbot.plugins.util.ChoiceStringParameter( + name="platform_architectures", + label="Platforms:", + required=True, + choices=code_all_platform_architectures[track], + multiple=True, + strict=True, + default=code_official_platform_architectures[track], + ), + ] + +# Scheduler properties. +scheduler_properties_common = [ + buildbot.plugins.util.BooleanParameter( + name="python_module", + label="Python module -> build bpy module instead of Blender", + required=True, + strict=True, + default=False, + ), + buildbot.plugins.util.BooleanParameter( + name="needs_full_clean", + label="Full clean -> removes build workspace on machine", + required=True, + strict=True, + default=False, + ), + buildbot.plugins.util.BooleanParameter( + name="needs_package_delivery", + label="Package delivery -> push files to configured services", + required=True, + strict=True, + default=False, + ), + buildbot.plugins.util.BooleanParameter( + name="needs_gpu_binaries", + label="GPU binaries -> build Cycles GPU kernels", + required=True, + strict=True, + default=False, + ), + buildbot.plugins.util.BooleanParameter( + name="needs_gpu_tests", + label="GPU tests -> run EEVEE, Viewport and Cycles GPU tests", + required=True, + strict=True, + default=False, + ), +] + +# code-daily +scheduler_properties_daily = scheduler_properties_common + +# code-experimental properties. +scheduler_properties_experimental = [ + buildbot.plugins.util.StringParameter( + name="override_branch_id", + label="Branch:", + required=True, + size=80, + regex=r"^[a-zA-Z0-9][A-Za-z0-9\._-]*$", + default="", + ), + buildbot.plugins.util.ChoiceStringParameter( + name="build_configuration", + label="Configuration:", + required=True, + choices=["release", "sanitizer", "debug"], + multiple=False, + strict=True, + default="release", + ), + buildbot.plugins.util.BooleanParameter( + name="needs_skip_tests", + label="Skip tests -> bypass running all tests", + required=True, + strict=True, + default=False, + ), +] +scheduler_properties_experimental += scheduler_properties_common + + +# code-patch properties. +scheduler_properties_patch = [ + buildbot.plugins.util.StringParameter( + name="patch_id", label="Patch Id:", required=True, size=80, default="" + ), + buildbot.plugins.util.ChoiceStringParameter( + name="build_configuration", + label="Configuration:", + required=True, + choices=["release", "sanitizer", "debug"], + multiple=False, + strict=True, + default="release", + ), + buildbot.plugins.util.BooleanParameter( + name="needs_skip_tests", + label="Skip tests -> bypass running all tests", + required=True, + strict=True, + default=False, + ), + buildbot.plugins.util.StringParameter( + name="pull_revision", label="Pull Revision:", required=False, hide=True, size=80, default="" + ), +] + +scheduler_properties_patch += scheduler_properties_common + +scheduler_properties = { + "code-daily": scheduler_properties_daily, + "code-experimental": scheduler_properties_experimental, + "code-patch": scheduler_properties_patch, +} + + +@buildbot.plugins.util.renderer +def create_code_worker_command_args(props, devops_env_id, track_id, pipeline_type, step_name): + commit_id = pipeline.common.fetch_property(props, key="revision", default="HEAD") + patch_id = pipeline.common.fetch_property(props, key="patch_id", default="") + override_branch_id = pipeline.common.fetch_property(props, key="override_branch_id", default="") + python_module = pipeline.common.fetch_property(props, key="python_module", default=False) + needs_gpu_tests = pipeline.common.fetch_property(props, key="needs_gpu_tests", default=False) + needs_gpu_binaries = pipeline.common.fetch_property( + props, key="needs_gpu_binaries", default=False + ) + build_configuration = pipeline.common.fetch_property( + props, key="build_configuration", default="release" + ) + needs_full_clean = pipeline.common.fetch_property( + props, key="needs_full_clean", default="false" + ) + needs_full_clean = needs_full_clean in ["true", True] + needs_package_delivery = pipeline.common.fetch_property( + props, key="needs_package_delivery", default="false" + ) + needs_package_delivery = needs_package_delivery in ["true", True] + + # Auto enable asserts when not using package delivery. Only support in 4.1+. + if track_id not in ("v360"): + if build_configuration == "release" and not needs_package_delivery: + build_configuration = "asserts" + + platform_id, architecture = pipeline.common.fetch_platform_architecture(props) + + args = [] + + if architecture: + args += ["--architecture", architecture] + + if pipeline_type == "patch": + # Powershell doesn't like # in string argument so strip it. + args += ["--patch-id", patch_id.lstrip("#")] + elif pipeline_type == "experimental": + args += ["--branch-id", override_branch_id] + + args += ["--commit-id", commit_id] + args += ["--build-configuration", build_configuration] + + if python_module: + args += ["--python-module"] + if needs_full_clean: + args += ["--needs-full-clean"] + if step_name in ["compile-gpu", "compile-install", "test-code"]: + if needs_package_delivery or needs_gpu_binaries: + args += ["--needs-gpu-binaries"] + if needs_gpu_tests: + args += ["--needs-gpu-tests"] + + args += [step_name] + + return pipeline.common.create_worker_command("code.py", devops_env_id, track_id, args) + + +def needs_do_code_pipeline_step(step): + build = step.build + # Use this to test master steps only, otherwise we be waiting for 30 minutes + needs_master_steps_only = False + + if needs_master_steps_only: + is_master_step = step.name in pipeline.common.code_pipeline_master_step_names + return is_master_step + + worker = step.worker + worker_name = step.getWorkerName() + worker_system = worker.worker_system + + is_package_delivery_step = (step.name in code_delivery_step_names) or ( + step.name in pipeline.common.code_pipeline_master_step_names + ) + needs_package_delivery = step.getProperty("needs_package_delivery") + needs_gpu_binaries = step.getProperty("needs_gpu_binaries") + needs_skip_tests = step.getProperty("needs_skip_tests") + + python_module = step.getProperty("python_module") + + needs_do_it = True + + if step.name in code_pipeline_test_step_names: + needs_do_it = not needs_skip_tests + elif step.name == "compile-gpu": + needs_do_it = needs_package_delivery or needs_gpu_binaries + elif is_package_delivery_step: + needs_do_it = needs_package_delivery + + if python_module and (step.name in code_python_module_skip_test_names): + needs_do_it = False + + return needs_do_it + + +# Custom file upload that shows links to download files. +class LinkMultipleFileUpload(plugins_steps.MultipleFileUpload): + def uploadDone(self, result, source, masterdest): + if not self.url: + return + + name = pathlib.Path(source).name + if name.endswith(".zip"): + self.addURL(name, self.url + "/" + name) + else: + self.addURL(name, self.url + "/" + name + "/report.html") + + def allUploadsDone(self, result, sources, masterdest): + return + + +def create_deliver_code_binaries_step(worker_config, track_id, pipeline_type): + file_size_in_mb = 500 * 1024 * 1024 + worker_source_path = pathlib.Path(f"../../../../git/blender-{track_id}/build_package") + master_dest_path = pathlib.Path( + f"{worker_config.buildbot_download_folder}/{pipeline_type}" + ).expanduser() + + return plugins_steps.MultipleFileUpload( + name="deliver-code-binaries", + maxsize=file_size_in_mb, + workdir=f"{worker_source_path}", + glob=True, + workersrcs=["*.*"], + masterdest=f"{master_dest_path}", + mode=0o644, + url=None, + description="running", + descriptionDone="completed", + doStepIf=needs_do_code_pipeline_step, + ) + + +def create_deliver_test_results_step(worker_config, track_id, pipeline_type): + file_size_in_mb = 500 * 1024 * 1024 + worker_source_path = pathlib.Path(f"../../../../git/blender-{track_id}/build_package") + master_dest_path = pathlib.Path( + f"{worker_config.buildbot_download_folder}/{pipeline_type}" + ).expanduser() + + tests_worker_source_path = worker_source_path / "tests" + tests_master_dest_path = master_dest_path / "tests" + tests_worker_srcs = ["tests-*.zip"] + + branch_id = code_tracked_branch_ids[track_id] + if branch_id: + branch_id = branch_id.replace("blender-", "").replace("-release", "") + tests_worker_srcs.append(branch_id + "-*") + + return LinkMultipleFileUpload( + name="deliver-test-results", + maxsize=file_size_in_mb, + workdir=f"{tests_worker_source_path}", + glob=True, + workersrcs=tests_worker_srcs, + masterdest=f"{tests_master_dest_path}", + mode=0o644, + url=f"../download/{pipeline_type}/tests", + description="running", + descriptionDone="completed", + alwaysRun=True, + ) + + +def next_worker_code(worker_names_gpu, builder, workers, request): + # Use a GPU worker if needed and supported for this platform. + # NVIDIA worker is currently reserved for GPU builds only. + compatible_workers = [] + if request.properties.getProperty("needs_gpu_tests", False) and worker_names_gpu: + for worker in workers: + if worker.worker.workername in worker_names_gpu: + compatible_workers.append(worker) + else: + for worker in workers: + if "nvidia" not in worker.worker.workername: + compatible_workers.append(worker) + + if not compatible_workers: + return None + + return random.choice(compatible_workers) + + +class PlatformTrigger(plugins_steps.Trigger): + def getSchedulersAndProperties(self): + schedulers = [] + + platform_architectures = self.set_properties["platform_architectures"] + + for scheduler in self.schedulerNames: + found = False + if "lint" in scheduler: + found = True + for platform_architecture in platform_architectures: + if platform_architecture in scheduler: + found = True + + if found: + schedulers.append( + { + "sched_name": scheduler, + "props_to_set": self.set_properties, + "unimportant": False, + } + ) + + return schedulers + + +def populate(devops_env_id): + builders = [] + schedulers = [] + + platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id) + local_worker_names = conf.machines.fetch_local_worker_names() + + worker_config = conf.worker.get_config(devops_env_id) + + needs_incremental_schedulers = devops_env_id in ["PROD"] + needs_nightly_schedulers = devops_env_id in ["PROD"] + + print("*** Creating [code] pipeline") + for track_id in code_track_ids: + pipeline_types = code_track_pipeline_types[track_id] + for pipeline_type in pipeline_types: + # Create steps. + step_names = pipeline_types_step_names[pipeline_type] + pipeline_build_factory = buildbot.plugins.util.BuildFactory() + + print(f"Creating [{track_id}] [code] [{pipeline_type}] pipeline steps") + for step_name in step_names: + if step_name == "deliver-code-binaries": + step = create_deliver_code_binaries_step(worker_config, track_id, pipeline_type) + elif step_name == "deliver-test-results": + step = create_deliver_test_results_step(worker_config, track_id, pipeline_type) + else: + needs_halt_on_failure = True + if step_name in code_pipeline_test_step_names: + needs_halt_on_failure = track_id != "vexp" + + step_timeout_in_seconds = default_step_timeout_in_seconds + if step_name == "compile-code": + step_timeout_in_seconds = compile_code_step_timeout_in_seconds + elif step_name == "compile-gpu": + step_timeout_in_seconds = compile_gpu_step_timeout_in_seconds + + step_command = create_code_worker_command_args.withArgs( + devops_env_id, track_id, pipeline_type, step_name + ) + + step = buildbot.plugins.steps.ShellCommand( + name=step_name, + logEnviron=True, + haltOnFailure=needs_halt_on_failure, + timeout=step_timeout_in_seconds, + description="running", + descriptionDone="completed", + doStepIf=needs_do_code_pipeline_step, + command=step_command, + ) + + pipeline_build_factory.addStep(step) + + for master_step_name in pipeline.common.code_pipeline_master_step_names: + master_step_command = pipeline.common.create_master_command_args.withArgs( + devops_env_id, track_id, pipeline_type, master_step_name, single_platform=True + ) + + # Master to archive and purge builds + pipeline_build_factory.addStep( + plugins_steps.MasterShellCommand( + name=master_step_name, + logEnviron=False, + command=master_step_command, + description="running", + descriptionDone="completed", + doStepIf=needs_do_code_pipeline_step, + ) + ) + + # Create lint pipeline + pipeline_lint_factory = buildbot.plugins.util.BuildFactory() + for step_name in code_pipeline_lint_step_names: + step_command = create_code_worker_command_args.withArgs( + devops_env_id, track_id, pipeline_type, step_name + ) + + pipeline_lint_factory.addStep( + buildbot.plugins.steps.ShellCommand( + name=step_name, + logEnviron=True, + haltOnFailure=True, + timeout=default_step_timeout_in_seconds, + description="running", + descriptionDone="completed", + command=step_command, + ) + ) + + triggerable_scheduler_names = [] + trigger_factory = buildbot.plugins.util.BuildFactory() + + # Create builders. + for platform_architecture in code_all_platform_architectures[track_id]: + print(f"Creating [{track_id}] [{pipeline_type}] [{platform_architecture}] builders") + + worker_group_id = f"{platform_architecture}-code" + worker_group_id_gpu = f"{platform_architecture}-code-gpu" + + pipeline_worker_names = platform_worker_names[worker_group_id] + pipeline_worker_names_gpu = platform_worker_names[worker_group_id_gpu] + if pipeline_worker_names: + # Only create the builders if the worker exists + pipeline_builder_name = ( + f"{track_id}-code-{pipeline_type}-{platform_architecture}" + ) + pipeline_builder_tags = pipeline_builder_name.split("-") + + # Assigning different workers for different tracks, specifically Linux builders. + suitable_pipeline_worker_names = pipeline_worker_names + if platform_architecture == "linux-x86_64" and devops_env_id != "LOCAL": + selector = "rocky" + suitable_pipeline_worker_names = [ + worker for worker in pipeline_worker_names if selector in worker + ] + + builders += [ + buildbot.plugins.util.BuilderConfig( + name=pipeline_builder_name, + workernames=suitable_pipeline_worker_names, + nextWorker=partial(next_worker_code, pipeline_worker_names_gpu), + tags=pipeline_builder_tags, + factory=pipeline_build_factory, + ) + ] + + pipeline_scheduler_name = ( + f"{track_id}-code-{pipeline_type}-{platform_architecture}-triggerable" + ) + triggerable_scheduler_names += [pipeline_scheduler_name] + + schedulers += [ + plugins_schedulers.Triggerable( + name=pipeline_scheduler_name, builderNames=[pipeline_builder_name] + ) + ] + + # Create lint builder + if track_id not in conf.branches.all_lts_tracks: + print(f"Creating [{track_id}] [{pipeline_type}] [lint] builders") + + pipeline_worker_names = platform_worker_names["code-lint"] + if pipeline_worker_names: + # Only create the builders if the worker exists + pipeline_builder_name = f"{track_id}-code-{pipeline_type}-lint" + pipeline_builder_tags = pipeline_builder_name.split("-") + + builders += [ + buildbot.plugins.util.BuilderConfig( + name=pipeline_builder_name, + workernames=pipeline_worker_names, + tags=pipeline_builder_tags, + factory=pipeline_lint_factory, + ) + ] + + pipeline_scheduler_name = f"{track_id}-code-{pipeline_type}-lint-triggerable" + triggerable_scheduler_names += [pipeline_scheduler_name] + + schedulers += [ + plugins_schedulers.Triggerable( + name=pipeline_scheduler_name, builderNames=[pipeline_builder_name] + ) + ] + + # Create coordinator. + if triggerable_scheduler_names: + trigger_properties = { + "python_module": buildbot.plugins.util.Property("python_module"), + "needs_full_clean": buildbot.plugins.util.Property("needs_full_clean"), + "needs_package_delivery": buildbot.plugins.util.Property( + "needs_package_delivery" + ), + "needs_gpu_binaries": buildbot.plugins.util.Property("needs_gpu_binaries"), + "needs_gpu_tests": buildbot.plugins.util.Property("needs_gpu_tests"), + "needs_skip_tests": buildbot.plugins.util.Property("needs_skip_tests"), + "platform_architectures": buildbot.plugins.util.Property( + "platform_architectures" + ), + } + if pipeline_type == "patch": + trigger_properties["patch_id"] = buildbot.plugins.util.Property("patch_id") + trigger_properties["revision"] = buildbot.plugins.util.Property("revision") + trigger_properties["build_configuration"] = buildbot.plugins.util.Property( + "build_configuration" + ) + trigger_factory.addStep( + plugins_steps.SetProperties( + name="get-revision", properties=gitea.blender.get_patch_revision + ) + ) + elif pipeline_type == "experimental": + trigger_properties["override_branch_id"] = buildbot.plugins.util.Property( + "override_branch_id" + ) + trigger_properties["revision"] = buildbot.plugins.util.Property("revision") + trigger_properties["build_configuration"] = buildbot.plugins.util.Property( + "build_configuration" + ) + trigger_factory.addStep( + plugins_steps.SetProperties( + name="get-revision", properties=gitea.blender.get_branch_revision + ) + ) + + trigger_factory.addStep( + PlatformTrigger( + schedulerNames=triggerable_scheduler_names, + waitForFinish=True, + updateSourceStamp=False, + set_properties=trigger_properties, + description="running", + descriptionDone="completed", + ) + ) + + coordinator_builder_name = f"{track_id}-code-{pipeline_type}-coordinator" + builder_tags = coordinator_builder_name.split("-") + + builders += [ + buildbot.plugins.util.BuilderConfig( + name=coordinator_builder_name, + workernames=local_worker_names, + tags=builder_tags, + factory=trigger_factory, + ) + ] + + coordinator_scheduler_name = f"{track_id}-code-{pipeline_type}-coordinator-force" + schedulers += [ + plugins_schedulers.ForceScheduler( + name=coordinator_scheduler_name, + buttonName=f"Trigger {pipeline_type} build", + builderNames=[coordinator_builder_name], + codebases=[ + buildbot.plugins.util.CodebaseParameter( + codebase="blender.git", + project="blender.git", + branch=code_tracked_branch_ids[track_id], + hide=True, + ) + ], + properties=track_properties[track_id] + + scheduler_properties[f"code-{pipeline_type}"], + ) + ] + + # Daily scheduler. + if pipeline_type == "daily": + print(f"Adding [{pipeline_type}] schedulers") + if needs_incremental_schedulers and (track_id in code_track_ids): + incremental_scheduler_name = ( + f"{track_id}-code-{pipeline_type}-coordinator-incremental" + ) + incremental_scheduler_properties = { + "revision": "HEAD", + "python_module": False, + "needs_skip_tests": False, + "needs_package_delivery": False, + "needs_gpu_binaries": False, + "build_configuration": "release", + "platform_architectures": code_official_platform_architectures[ + track_id + ], + } + + change_filter = buildbot.plugins.util.ChangeFilter( + project=["blender.git"], branch=code_tracked_branch_ids[track_id] + ) + schedulers += [ + plugins_schedulers.SingleBranchScheduler( + name=incremental_scheduler_name, + builderNames=[coordinator_builder_name], + change_filter=change_filter, + properties=incremental_scheduler_properties, + treeStableTimer=tree_stable_timer_in_seconds, + ) + ] + + if needs_nightly_schedulers and (track_id in code_track_ids): + nightly_scheduler_name = ( + f"{track_id}-code-{pipeline_type}-coordinator-nightly" + ) + nightly_properties = { + "revision": "HEAD", + "python_module": False, + "needs_skip_tests": False, + "needs_package_delivery": True, + "needs_gpu_binaries": True, + "build_configuration": "release", + "platform_architectures": code_all_platform_architectures[track_id], + } + nightly_codebases = { + "blender.git": { + "repository": "", + "branch": code_tracked_branch_ids[track_id], + "revision": None, + } + } + schedulers += [ + plugins_schedulers.Nightly( + name=nightly_scheduler_name, + builderNames=[coordinator_builder_name], + codebases=nightly_codebases, + properties=nightly_properties, + onlyIfChanged=False, + hour=1, + minute=30, + ) + ] + + return builders, schedulers diff --git a/config/pipeline/code_benchmark.py b/config/pipeline/code_benchmark.py new file mode 100644 index 0000000..ab695d0 --- /dev/null +++ b/config/pipeline/code_benchmark.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import pathlib +from functools import partial + +import buildbot.plugins +from buildbot.plugins import steps as plugins_steps + +import conf.branches +import conf.worker +import pipeline.common + + +# Custom file upload that shows links to download files. +class LinkMultipleFileUpload(plugins_steps.MultipleFileUpload): + def uploadDone(self, result, source, masterdest): + if not self.url: + return + + name = pathlib.Path(source).name + self.addURL(name, self.url + "/" + name + "/report.html") + + def allUploadsDone(self, result, sources, masterdest): + return + + +def create_deliver_step(devops_env_id): + worker_config = conf.worker.get_config(devops_env_id) + + file_size_in_mb = 500 * 1024 * 1024 + worker_source_path = pathlib.Path("../../../../git/blender-vdev/build_package") + master_dest_path = worker_config.buildbot_download_folder / "daily" / "benchmarks" + + return LinkMultipleFileUpload( + name="deliver", + maxsize=file_size_in_mb, + workdir=f"{worker_source_path}", + glob=True, + workersrcs=["main-*"], + masterdest=f"{master_dest_path}", + mode=0o644, + url="../download/daily/benchmarks", + description="running", + descriptionDone="completed", + alwaysRun=True, + ) + + +def populate(devops_env_id): + properties = [ + buildbot.plugins.util.StringParameter( + name="commit_id", + label="Commit:", + required=True, + size=80, + default="HEAD", + ), + buildbot.plugins.util.BooleanParameter( + name="needs_gpu_binaries", + label="GPU binaries -> build Cycles GPU kernels", + required=True, + strict=True, + default=True, + hide=True, + ), + ] + + return pipeline.common.create_pipeline( + devops_env_id, + "code-benchmark", + "code_benchmark.py", + [ + "configure-machine", + "update-code", + "compile-code", + "compile-gpu", + "compile-install", + "benchmark", + partial(create_deliver_step, devops_env_id), + "clean", + ], + {"vdev": "main"}, + properties, + "blender.git", + ["linux-x86_64-code-gpu", "darwin-arm64-code-gpu"], + # Compile GPU step needs a long timeout. + default_step_timeout_in_seconds=90 * 60, + variations=["linux", "darwin"], + nightly_properties={"commit_id": "HEAD", "needs_gpu_binaries": True}, + hour=7, + minute=30, + ) diff --git a/config/pipeline/code_bpy_deploy.py b/config/pipeline/code_bpy_deploy.py new file mode 100644 index 0000000..b5e5f48 --- /dev/null +++ b/config/pipeline/code_bpy_deploy.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +# Builders for deploying Python module releases to PyPI. + + +import conf.branches +import pipeline.common + + +def populate(devops_env_id): + properties = [] + + return pipeline.common.create_pipeline( + devops_env_id, + "code-bpy-deploy", + "code_bpy_deploy.py", + [ + "configure-machine", + "update-code", + "pull", + "deliver-pypi", + "clean", + ], + conf.branches.code_deploy_track_ids, + properties, + "blender.git", + ["linux-x86_64-general"], + ) diff --git a/config/pipeline/code_deploy.py b/config/pipeline/code_deploy.py new file mode 100644 index 0000000..cb89aba --- /dev/null +++ b/config/pipeline/code_deploy.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +# Builders for deploying Blender releases. + +import buildbot.plugins + +import conf.branches +import pipeline.common + + +def populate(devops_env_id): + properties = [ + buildbot.plugins.util.BooleanParameter( + name="needs_full_clean", + label="Full clean -> removes build workspace on machine", + required=True, + strict=True, + default=False, + ), + ] + + return pipeline.common.create_pipeline( + devops_env_id, + "code-artifacts-deploy", + "code_deploy.py", + [ + "configure-machine", + "update-code", + "package-source", + "pull-artifacts", + "repackage-artifacts", + "deploy-artifacts", + "monitor-artifacts", + "clean", + ], + conf.branches.code_deploy_track_ids, + properties, + "blender.git", + ["linux-x86_64-general"], + default_step_timeout_in_seconds=30 * 60, + ) diff --git a/config/pipeline/code_store.py b/config/pipeline/code_store.py new file mode 100644 index 0000000..35aaf09 --- /dev/null +++ b/config/pipeline/code_store.py @@ -0,0 +1,235 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +# Builders for releasing Blender to stores. + +import pathlib + +import buildbot.plugins + +from buildbot.plugins import steps as plugins_steps +from buildbot.plugins import schedulers as plugins_schedulers + +import conf.branches +import conf.worker +import pipeline.common + +# Timeouts. +default_step_timeout_in_seconds = 60 * 60 + +# Tracks. +track_ids = conf.branches.code_store_track_ids +tracked_branch_ids = {} +for track_id in track_ids: + tracked_branch_ids[track_id] = conf.branches.code_tracked_branch_ids[track_id] + +# Properties. +scheduler_properties = [ + buildbot.plugins.util.ChoiceStringParameter( + name="store_id", + label="Store:", + required=True, + choices=["snap", "steam", "windows"], + multiple=True, + strict=True, + default=["snap", "steam", "windows"], + ), +] + + +def create_deliver_binaries_windows_step(worker_config, track_id, pipeline_type): + # Create step for uploading msix to download.blender.org. + file_size_in_mb = 500 * 1024 * 1024 + worker_source_path = pathlib.Path(f"../../../../git/blender-{track_id}/build_package") + master_dest_path = pathlib.Path( + f"{worker_config.buildbot_download_folder}/{pipeline_type}" + ).expanduser() + + return plugins_steps.MultipleFileUpload( + name="deliver-binaries", + maxsize=file_size_in_mb, + workdir=f"{worker_source_path}", + glob=True, + workersrcs=["*.msix*"], + masterdest=f"{master_dest_path}", + mode=0o644, + url=None, + description="running", + descriptionDone="completed", + ) + + +def populate(devops_env_id): + builders = [] + schedulers = [] + + platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id) + local_worker_names = conf.machines.fetch_local_worker_names() + + worker_config = conf.worker.get_config(devops_env_id) + + needs_nightly_schedulers = devops_env_id == "PROD" + + pipeline_type = "daily" + + store_ids = ["steam", "snap", "windows"] + + print("*** Creating [code] [store] pipeline") + for track_id in track_ids: + triggerable_scheduler_names = [] + trigger_factory = buildbot.plugins.util.BuildFactory() + + for store_id in store_ids: + # Create build steps. + pipeline_build_factory = buildbot.plugins.util.BuildFactory() + step_names = [ + "configure-machine", + "update-code", + "pull-artifacts", + "package", + ] + + if store_id == "windows": + step_names += ["deliver-binaries"] + else: + step_names += ["deliver"] + + step_names += ["clean"] + + print(f"Creating [{track_id}] [code] [store] [{store_id}] pipeline steps") + for step_name in step_names: + if step_name == "deliver-binaries": + step = create_deliver_binaries_windows_step( + worker_config, track_id, pipeline_type + ) + else: + args = ["--store-id", store_id, step_name] + step_command = pipeline.common.create_worker_command( + "code_store.py", devops_env_id, track_id, args + ) + + step = plugins_steps.ShellCommand( + name=step_name, + logEnviron=True, + haltOnFailure=True, + timeout=default_step_timeout_in_seconds, + description="running", + descriptionDone="completed", + command=step_command, + ) + + pipeline_build_factory.addStep(step) + + for master_step_name in pipeline.common.code_pipeline_master_step_names: + master_step_command = pipeline.common.create_master_command_args.withArgs( + devops_env_id, track_id, pipeline_type, master_step_name, single_platform=False + ) + + # Master to archive and purge builds + pipeline_build_factory.addStep( + plugins_steps.MasterShellCommand( + name=master_step_name, + logEnviron=False, + command=master_step_command, + description="running", + descriptionDone="completed", + ) + ) + + # Create builders. + worker_group_id = ( + f"windows-amd64-store-{store_id}" + if store_id == "windows" + else f"linux-x86_64-store-{store_id}" + ) + pipeline_worker_names = platform_worker_names[worker_group_id] + if pipeline_worker_names: + pipeline_builder_name = f"{track_id}-code-store-{store_id}" + + builder_tags = pipeline_builder_name.split("-") + + builders += [ + buildbot.plugins.util.BuilderConfig( + name=pipeline_builder_name, + workernames=pipeline_worker_names, + tags=builder_tags, + factory=pipeline_build_factory, + ) + ] + + scheduler_name = f"{track_id}-code-store-{store_id}-triggerable" + triggerable_scheduler_names += [scheduler_name] + + schedulers += [ + plugins_schedulers.Triggerable( + name=scheduler_name, builderNames=[pipeline_builder_name] + ) + ] + + # Create coordinator. + if triggerable_scheduler_names: + trigger_properties = {} + trigger_factory.addStep( + plugins_steps.Trigger( + schedulerNames=triggerable_scheduler_names, + waitForFinish=True, + updateSourceStamp=False, + set_properties=trigger_properties, + description="running", + descriptionDone="completed", + ) + ) + + coordinator_builder_name = f"{track_id}-code-store-coordinator" + builder_tags = coordinator_builder_name.split("-") + + builders += [ + buildbot.plugins.util.BuilderConfig( + name=coordinator_builder_name, + workernames=local_worker_names, + tags=builder_tags, + factory=trigger_factory, + ) + ] + + coordinator_scheduler_name = f"{track_id}-code-store-coordinator-force" + schedulers += [ + plugins_schedulers.ForceScheduler( + name=coordinator_scheduler_name, + buttonName="Trigger store build", + builderNames=[coordinator_builder_name], + codebases=[ + buildbot.plugins.util.CodebaseParameter( + codebase="", revision=None, hide=True + ) + ], + properties=scheduler_properties, + ) + ] + + if needs_nightly_schedulers and (track_id in track_ids): + nightly_scheduler_name = f"{track_id}-code-store-coordinator-nightly" + nightly_properties = { + "revision": "HEAD", + } + nightly_codebases = { + "blender.git": { + "repository": "", + "branch": tracked_branch_ids[track_id], + "revision": None, + } + } + schedulers += [ + plugins_schedulers.Nightly( + name=nightly_scheduler_name, + builderNames=[coordinator_builder_name], + codebases=nightly_codebases, + properties=nightly_properties, + onlyIfChanged=False, + hour=5, + minute=30, + ) + ] + + return builders, schedulers diff --git a/config/pipeline/common.py b/config/pipeline/common.py new file mode 100644 index 0000000..dfd7124 --- /dev/null +++ b/config/pipeline/common.py @@ -0,0 +1,335 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import buildbot.plugins + +from buildbot.plugins import steps as plugins_steps +from buildbot.plugins import schedulers as plugins_schedulers + +import conf.machines + +devops_git_root_path = "~/git" + +# Steps that run on the buildbot master. +code_pipeline_master_step_names = [ + "deduplicate-binaries", + "purge-binaries", +] + + +def fetch_property(props, key, default=None): + value = default + if key in props: + value = props[key] + + return value + + +def fetch_platform_architecture(props): + platform_architectures = fetch_property(props, key="platform_architectures") + + # Find the platform arch for this builder + buildername = fetch_property(props, key="buildername") + builder_platform_architecture = "-".join(buildername.split("-")[-2:]) + + found_platform_architecture = None + if platform_architectures: + for platform_architecture in platform_architectures: + if platform_architecture in builder_platform_architecture: + found_platform_architecture = platform_architecture + break + + if found_platform_architecture: + return found_platform_architecture.split("-") + else: + return None, None + + +def always_do_step(step): + return True + + +def needs_do_doc_pipeline_step(step): + if "package" in step.name or "deliver" in step.name: + return step.getProperty("needs_package_delivery") + else: + return True + + +def create_worker_command(script, devops_env_id, track_id, args): + # This relative path assume were are in: + # ~/.devops/services/buildbot-worker//build + # There appears to be no way to expand a tilde here? + # + # This is assumed to run within the buildbot worker pipenv, + # so the python command should match the python version and + # available packages. + cmd = [ + "python", + f"../../../../../git/blender-devops/buildbot/worker/{script}", + "--track-id", + track_id, + "--service-env-id", + devops_env_id, + ] + + return cmd + list(args) + + +@buildbot.plugins.util.renderer +def create_master_command_args( + props, devops_env_id, track_id, pipeline_type, step_name, single_platform +): + build_configuration = fetch_property(props, key="build_configuration", default="release") + python_module = fetch_property(props, key="python_module", default=False) + python_module_string = "true" if python_module else "false" + + args = [ + "--pipeline-type", + pipeline_type, + "--build-configuration", + build_configuration, + ] + + if single_platform: + # Archive binaries for a single architecture only? + platform_id, architecture = fetch_platform_architecture(props) + args += ["--platform-id", platform_id, "--architecture", architecture] + + if python_module: + args += ["--python-module"] + + args += [step_name] + + # This relative path assume were are in: + # ~/.devops/services/buildbot-master + # There appears to be no way to expand a tilde here? + # + # This is assumed to run within the buildbot master pipenv, + # so the python command should match the python version and + # available packages. + cmd = [ + "python", + "../../../git/blender-devops/buildbot/worker/archive.py", + "--track-id", + track_id, + "--service-env-id", + devops_env_id, + ] + + return cmd + list(args) + + +@buildbot.plugins.util.renderer +def create_pipeline_worker_command( + props, + devops_env_id, + track_id, + script, + step_name, + variation_property, + variation, + builder_properties, +): + args = [step_name] + + if variation_property: + args += ["--" + variation_property.replace("_", "-"), variation] + + for builder_prop in builder_properties: + if builder_prop.name in props: + prop_value = props[builder_prop.name] + else: + prop_value = builder_prop.default + + argument_name = "--" + builder_prop.name.replace("_", "-") + if isinstance(builder_prop, buildbot.plugins.util.BooleanParameter): + if prop_value in ["true", True]: + args += [argument_name] + else: + args += [argument_name, prop_value] + + if "revision" in props and props["revision"]: + args += ["--commit-id", props["revision"]] + + return create_worker_command(script, devops_env_id, track_id, args) + + +def create_pipeline( + devops_env_id, + artifact_id, + script, + steps, + tracked_branch_ids, + properties, + codebase, + worker_group_ids, + variation_property=None, + variations=[""], + incremental_properties=None, + nightly_properties=None, + do_step_if=always_do_step, + default_step_timeout_in_seconds=600, + tree_stable_timer_in_seconds=180, + hour=5, + minute=0, +): + builders = [] + schedulers = [] + + platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id) + local_worker_names = conf.machines.fetch_local_worker_names() + + needs_incremental_schedulers = incremental_properties is not None and devops_env_id in ["PROD"] + needs_nightly_schedulers = nightly_properties is not None and devops_env_id in ["PROD"] + track_ids = tracked_branch_ids.keys() + + print(f"*** Creating [{artifact_id}] pipeline") + for track_id in track_ids: + triggerable_scheduler_names = [] + trigger_factory = buildbot.plugins.util.BuildFactory() + + for worker_group_id, variation in zip(worker_group_ids, variations): + if variation: + pipeline_builder_name = f"{track_id}-{artifact_id}-{variation}" + else: + pipeline_builder_name = f"{track_id}-{artifact_id}" + + pipeline_build_factory = buildbot.plugins.util.BuildFactory() + + print(f"Creating [{pipeline_builder_name}] pipeline steps") + for step in steps: + if callable(step): + pipeline_build_factory.addStep(step()) + continue + + step_command = create_pipeline_worker_command.withArgs( + devops_env_id, + track_id, + script, + step, + variation_property, + variation, + properties, + ) + + pipeline_build_factory.addStep( + plugins_steps.ShellCommand( + name=step, + logEnviron=True, + haltOnFailure=True, + timeout=default_step_timeout_in_seconds, + description="running", + descriptionDone="completed", + command=step_command, + doStepIf=do_step_if, + ) + ) + + # Create builder. + pipeline_worker_names = platform_worker_names[worker_group_id] + if pipeline_worker_names: + builder_tags = pipeline_builder_name.split("-") + + builders += [ + buildbot.plugins.util.BuilderConfig( + name=pipeline_builder_name, + workernames=pipeline_worker_names, + tags=builder_tags, + factory=pipeline_build_factory, + ) + ] + + scheduler_name = f"{pipeline_builder_name}-triggerable" + triggerable_scheduler_names += [scheduler_name] + + schedulers += [ + plugins_schedulers.Triggerable( + name=scheduler_name, builderNames=[pipeline_builder_name] + ) + ] + + # Only create scheduler if we have something to to trigger + if triggerable_scheduler_names: + trigger_properties = {} + for property in properties: + if property != variation_property: + trigger_properties[property.name] = buildbot.plugins.util.Property( + property.name + ) + + trigger_factory.addStep( + plugins_steps.Trigger( + schedulerNames=triggerable_scheduler_names, + waitForFinish=True, + updateSourceStamp=False, + set_properties=trigger_properties, + description="running", + descriptionDone="completed", + ) + ) + + coordinator_builder_name = f"{track_id}-{artifact_id}-coordinator" + builder_tags = coordinator_builder_name.split("-") + builders += [ + buildbot.plugins.util.BuilderConfig( + name=coordinator_builder_name, + workernames=local_worker_names, + tags=builder_tags, + factory=trigger_factory, + ) + ] + + coordinator_scheduler_name = f"{track_id}-{artifact_id}-coordinator-force" + schedulers += [ + plugins_schedulers.ForceScheduler( + name=coordinator_scheduler_name, + buttonName="Trigger build", + builderNames=[coordinator_builder_name], + codebases=[ + buildbot.plugins.util.CodebaseParameter( + codebase="", revision=None, hide=True + ) + ], + properties=properties, + ) + ] + + if needs_incremental_schedulers and (track_id in track_ids): + incremental_scheduler_name = f"{track_id}-{artifact_id}-coordinator-incremental" + change_filter = buildbot.plugins.util.ChangeFilter( + project=[codebase], branch=tracked_branch_ids[track_id] + ) + schedulers += [ + plugins_schedulers.SingleBranchScheduler( + name=incremental_scheduler_name, + builderNames=[coordinator_builder_name], + change_filter=change_filter, + properties=incremental_properties, + treeStableTimer=tree_stable_timer_in_seconds, + ) + ] + + if needs_nightly_schedulers and (track_id in track_ids): + nightly_codebases = { + codebase: { + "repository": "", + "branch": tracked_branch_ids[track_id], + "revision": None, + } + } + nightly_scheduler_name = f"{track_id}-{artifact_id}-coordinator-nightly" + schedulers += [ + plugins_schedulers.Nightly( + name=nightly_scheduler_name, + builderNames=[coordinator_builder_name], + codebases=nightly_codebases, + properties=nightly_properties, + onlyIfChanged=False, + hour=hour, + minute=minute, + ) + ] + + return builders, schedulers diff --git a/config/pipeline/doc_api.py b/config/pipeline/doc_api.py new file mode 100644 index 0000000..09c1239 --- /dev/null +++ b/config/pipeline/doc_api.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import buildbot.plugins + +import conf.branches +import pipeline.common + + +def populate(devops_env_id): + properties = [ + buildbot.plugins.util.BooleanParameter( + name="needs_full_clean", + label="Full clean -> removes build workspace on machine", + required=True, + strict=True, + default=False, + ), + buildbot.plugins.util.BooleanParameter( + name="needs_package_delivery", + label="Package delivery -> push build to configured services", + required=True, + strict=True, + default=False, + ), + ] + + return pipeline.common.create_pipeline( + devops_env_id, + "doc-api", + "doc_api.py", + [ + "configure-machine", + "update-code", + "compile-code", + "compile-install", + "compile", + "package", + "deliver", + "clean", + ], + conf.branches.code_tracked_branch_ids, + properties, + "blender.git", + ["linux-x86_64-general"], + variations=["html"], + incremental_properties={"needs_package_delivery": False}, + nightly_properties={"needs_package_delivery": True}, + tree_stable_timer_in_seconds=15 * 60, + do_step_if=pipeline.common.needs_do_doc_pipeline_step, + hour=1, + minute=30, + ) diff --git a/config/pipeline/doc_developer.py b/config/pipeline/doc_developer.py new file mode 100644 index 0000000..2333f98 --- /dev/null +++ b/config/pipeline/doc_developer.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import buildbot.plugins + +import pipeline.common + + +def populate(devops_env_id): + properties = [ + buildbot.plugins.util.BooleanParameter( + name="needs_package_delivery", + label="Package delivery -> push build to configured services", + required=True, + strict=True, + default=True, + ), + ] + + return pipeline.common.create_pipeline( + devops_env_id, + "doc-developer", + "doc_developer.py", + ["update", "compile", "deliver"], + {"vdev": "main"}, + properties, + "blender-developer-docs.git", + ["linux-x86_64-general"], + incremental_properties={"needs_package_delivery": True}, + do_step_if=pipeline.common.needs_do_doc_pipeline_step, + ) diff --git a/config/pipeline/doc_manual.py b/config/pipeline/doc_manual.py new file mode 100644 index 0000000..4cdd619 --- /dev/null +++ b/config/pipeline/doc_manual.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import buildbot.plugins + +import conf.branches +import pipeline.common + + +def populate(devops_env_id): + properties = [ + buildbot.plugins.util.BooleanParameter( + name="needs_package_delivery", + label="Package delivery -> push build to configured services", + required=True, + strict=True, + default=True, + ), + buildbot.plugins.util.BooleanParameter( + name="needs_all_locales", + label="All locales -> process all configure locales", + required=True, + strict=True, + default=False, + ), + ] + + return pipeline.common.create_pipeline( + devops_env_id, + "doc-manual", + "doc_manual.py", + ["configure-machine", "update", "compile", "package", "deliver", "clean"], + conf.branches.code_tracked_branch_ids, + properties, + "blender-manual.git", + ["linux-x86_64-general", "linux-x86_64-general"], + variation_property="doc_format", + variations=["html", "epub"], + incremental_properties={"needs_package_delivery": True, "needs_all_locales": False}, + nightly_properties={"needs_package_delivery": True, "needs_all_locales": True}, + tree_stable_timer_in_seconds=15 * 60, + do_step_if=pipeline.common.needs_do_doc_pipeline_step, + ) diff --git a/config/pipeline/doc_studio.py b/config/pipeline/doc_studio.py new file mode 100644 index 0000000..279b8f6 --- /dev/null +++ b/config/pipeline/doc_studio.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import buildbot.plugins + +import pipeline.common + + +def populate(devops_env_id): + properties = [ + buildbot.plugins.util.BooleanParameter( + name="needs_package_delivery", + label="Package delivery -> push build to configured services", + required=True, + strict=True, + default=True, + ), + ] + + return pipeline.common.create_pipeline( + devops_env_id, + "doc-studio-tools", + "doc_studio.py", + ["update", "compile", "deliver"], + {"vdev": "main"}, + properties, + "blender-studio-tools.git", + ["linux-x86_64-doc-studio-tools"], + incremental_properties={"needs_package_delivery": True}, + do_step_if=pipeline.common.needs_do_doc_pipeline_step, + ) diff --git a/config/worker/__init__.py b/config/worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/worker/archive.py b/config/worker/archive.py new file mode 100755 index 0000000..6cfcaed --- /dev/null +++ b/config/worker/archive.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import argparse +import datetime +import os +import pathlib +import random +import re +import sys +import time + +from collections import OrderedDict +from typing import Any, Dict, List, Optional, Sequence, Union + +sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) + +import worker.utils + +package_file_pattern = re.compile( + r"^(?P(blender|bpy))\-" + + r"(?P[0-9]+\.[0-9]+\.[0-9]+)\-" + + r"(?P[a-z]+)\+" + + r"(?P[A-Za-z0-9_\-]+)\." + + r"(?P[a-fA-f0-9]+)\-" + + r"(?P[A-Za-z0-9_]+)\." + + r"(?P[A-Za-z0-9_]+)\-" + + r"(?P(release|asserts|sanitizer|debug))\." + + r"(?P[A-Za-z0-9\.]+)" +) + +pipeline_types = ["daily", "experimental", "patch"] +platforms = ["linux", "windows", "darwin"] +architectures = ["x86_64", "amd64", "arm64"] +build_configurations = ["release", "asserts", "sanitizer", "debug"] + + +class ArchiveBuilder(worker.utils.Builder): + def __init__(self, args: argparse.Namespace): + super().__init__(args, "blender", "blender") + self.pipeline_type = args.pipeline_type + self.platform_id = args.platform_id + self.architecture = args.architecture + self.build_configuration = args.build_configuration + self.python_module = args.python_module + self.dry_run = args.dry_run + self.retention_in_days = args.retention_in_days + + +def file_age_in_days(file_path: pathlib.Path) -> float: + try: + file_path_mtime = os.path.getmtime(file_path) + except: + return 0.0 + + age_in_seconds = time.time() - file_path_mtime + return age_in_seconds / (3600.0 * 24.0) + + +def parse_build_info(file_path: pathlib.Path) -> Optional[Dict]: + file_name = file_path.name + matches = re.match(package_file_pattern, file_path.name) + if not matches: + return None + build_info: Dict[str, Union[str, float, pathlib.Path]] = dict(matches.groupdict()) + build_info["file_age_in_days"] = file_age_in_days(file_path) + build_info["file_path"] = file_path + return build_info + + +def archive_build(file_path: pathlib.Path, dry_run: bool) -> None: + # Archive build file itself and checksum + checksum_file_path = file_path.parent / (file_path.name + ".sha256") + + for source_file_path in [file_path, checksum_file_path]: + if not source_file_path.exists(): + continue + + archive_path = source_file_path.parent / "archive" + os.makedirs(archive_path, exist_ok=True) + dest_file_path = archive_path / source_file_path.name + + worker.utils.remove_file(dest_file_path, dry_run=dry_run) + worker.utils.move(source_file_path, dest_file_path, dry_run=dry_run) + + +def fetch_current_builds( + builder: ArchiveBuilder, + pipeline_type: str, + short_version: Optional[str] = None, + all_platforms: bool = False, +) -> Dict[Any, List[Any]]: + app_id = "bpy" if builder.python_module else "blender" + + worker_config = builder.get_worker_config() + download_path = worker_config.buildbot_download_folder + pipeline_build_path = download_path / pipeline_type + + print(f"Fetching current builds in [{pipeline_build_path}]") + build_groups: Dict[Any, List[Any]] = {} + for file_path in pipeline_build_path.glob("*.*"): + if not file_path.is_file(): + continue + if file_path.name.endswith(".sha256"): + continue + + build_info = parse_build_info(file_path) + if not build_info: + continue + if short_version and not build_info["version_id"].startswith(short_version + "."): + continue + + if not all_platforms: + if builder.architecture and build_info["architecture"] != builder.architecture: + continue + if builder.platform_id and build_info["platform_id"] != builder.platform_id: + continue + if ( + builder.build_configuration + and build_info["build_configuration"] != builder.build_configuration + ): + continue + + if pipeline_type == "daily": + key = ( + "daily", + build_info["file_extension"], + build_info["architecture"], + build_info["platform_id"], + ) + else: + key = ( + build_info["branch_id"], + build_info["file_extension"], + build_info["architecture"], + build_info["platform_id"], + ) + + if key in build_groups: + build_groups[key].append(build_info) + else: + build_groups[key] = [build_info] + + return build_groups + + +def archive_build_group( + builds: Sequence[Dict], retention_in_days: int, dry_run: bool = True +) -> None: + builds = sorted(builds, key=lambda build: build["file_age_in_days"]) + + for i, build in enumerate(builds): + build_age = build["file_age_in_days"] + build_name = build["file_path"].name + + # Only keep the most recent build if there are multiple + if i > 0 or build_age > retention_in_days: + print(f"Archiving [{build_name}] (age: {build_age:.3f} days)") + archive_build(build["file_path"], dry_run=dry_run) + else: + print(f"Keeping [{build_name}] (age: {build_age:.3f} days)") + + +def deduplicate(builder: ArchiveBuilder) -> None: + retention_in_days = builder.retention_in_days + dry_run = builder.dry_run + + # Get major.minor version to match. + short_version = "" + if builder.pipeline_type == "daily": + branches_config = builder.get_branches_config() + short_version = branches_config.track_major_minor_versions[builder.track_id] + + if not short_version: + raise BaseException(f"Missing version in [{builder.pipeline_type}] builds, aborting") + + build_groups = fetch_current_builds(builder, builder.pipeline_type, short_version=short_version) + + print( + f"Deduplicating [{builder.pipeline_type}] builds for [{short_version}] [{builder.build_configuration}] [{builder.platform_id}] [{builder.architecture}]" + ) + for key, build_group in build_groups.items(): + print("") + print("--- Group: " + str(key)) + archive_build_group(build_group, retention_in_days, dry_run=dry_run) + + +def fetch_purge_builds( + builder: ArchiveBuilder, pipeline_type: str, folder: str +) -> Sequence[pathlib.Path]: + worker_config = builder.get_worker_config() + download_path = worker_config.buildbot_download_folder + archive_path = download_path / pipeline_type / folder + os.makedirs(archive_path, exist_ok=True) + + print(f"Fetching archived builds in [{archive_path}]") + builds = [] + for file_path in archive_path.glob("*.*"): + if not file_path.is_file(): + continue + if file_path.name.endswith(".sha256"): + continue + + builds.append(file_path) + + return builds + + +def purge(builder: ArchiveBuilder) -> None: + builds_retention_in_days = builder.retention_in_days + tests_retention_in_days = 10 + dry_run = builder.dry_run + + for pipeline_type in pipeline_types: + if pipeline_type != "daily": + print("=" * 120) + print(f"Deduplicating [{pipeline_type}] builds") + build_groups = fetch_current_builds(builder, pipeline_type, all_platforms=True) + for key, build_group in build_groups.items(): + print("") + print("--- Group: " + str(key)) + archive_build_group(build_group, builds_retention_in_days, dry_run=dry_run) + + print("=" * 120) + print(f"Purging [{pipeline_type}] builds older than [{builds_retention_in_days}] days") + for file_path in fetch_purge_builds(builder, pipeline_type, "archive"): + if file_age_in_days(file_path) < builds_retention_in_days: + continue + + age = file_age_in_days(file_path) + checksum_file_path = file_path.parent / (file_path.name + ".sha256") + + print(f"Deleting [{file_path.name}] (age: {age:.3f} days)") + worker.utils.remove_file(file_path, dry_run=dry_run) + worker.utils.remove_file(checksum_file_path, dry_run=dry_run) + + print("=" * 120) + print(f"Purging [{pipeline_type}] tests older than [{tests_retention_in_days}] days") + for file_path in fetch_purge_builds(builder, pipeline_type, "tests"): + if file_age_in_days(file_path) < tests_retention_in_days: + continue + + age = file_age_in_days(file_path) + checksum_file_path = file_path.parent / (file_path.name + ".sha256") + + print(f"Deleting [{file_path.name}] (age: {age:.3f} days)") + worker.utils.remove_file(file_path, dry_run=dry_run) + worker.utils.remove_file(checksum_file_path, dry_run=dry_run) + + +def generate_test_data(builder: ArchiveBuilder) -> None: + worker_config = builder.get_worker_config() + download_path = worker_config.buildbot_download_folder + + branches_config = builder.get_branches_config() + short_version = branches_config.track_major_minor_versions[builder.track_id] + version = short_version + ".0" + + app_id = "bpy" if builder.python_module else "blender" + commit_hashes = ["1ddf858", "03a2a53"] + risk_ids = ["stable", "alpha"] + file_extensions = ["zip", "msi"] + + if builder.pipeline_type == "daily": + versions = [short_version + ".0", short_version + ".1"] + branches = ["main", "v50"] + build_configurations = ["release"] + elif builder.pipeline_type == "patch": + versions = ["5.0.0", "7.0.0"] + branches = ["PR123", "PR456", "PR789"] + build_configurations = ["release", "debug"] + else: + versions = ["4.0.0", "6.0.0"] + branches = ["realtime-compositor", "cycles-x"] + build_configurations = ["release", "debug"] + + pipeline_path = download_path / builder.pipeline_type + os.makedirs(pipeline_path, exist_ok=True) + + for i in range(0, 25): + filename = ( + app_id + + "-" + + random.choice(versions) + + "-" + + random.choice(risk_ids) + + "+" + + random.choice(branches) + + "." + + random.choice(commit_hashes) + + "-" + + builder.platform_id + + "." + + builder.architecture + + "-" + + random.choice(build_configurations) + + "." + + random.choice(file_extensions) + ) + + file_path = pipeline_path / filename + file_path.write_text("Test") + + checksum_file_path = file_path.parent / (file_path.name + ".sha256") + checksum_file_path.write_text("Test") + + delta = datetime.timedelta(days=365 * random.random()) + filetime = time.mktime((datetime.datetime.today() - delta).timetuple()) + os.utime(file_path, (filetime, filetime)) + os.utime(checksum_file_path, (filetime, filetime)) + + +if __name__ == "__main__": + steps: worker.utils.BuilderSteps = OrderedDict() + steps["deduplicate-binaries"] = deduplicate + steps["purge-binaries"] = purge + + parser = worker.utils.create_argument_parser(steps=steps) + parser.add_argument( + "--pipeline-type", default="daily", type=str, choices=pipeline_types, required=False + ) + parser.add_argument("--platform-id", default="", type=str, choices=platforms, required=False) + parser.add_argument( + "--architecture", default="", type=str, choices=architectures, required=False + ) + parser.add_argument( + "--build-configuration", + default="release", + type=str, + choices=build_configurations, + required=False, + ) + parser.add_argument("--retention-in-days", default=100, type=int, required=False) + parser.add_argument("--python-module", action="store_true", required=False) + parser.add_argument("--dry-run", action="store_true", required=False) + parser.add_argument("--generate-test-data", action="store_true", required=False) + + args = parser.parse_args() + builder = ArchiveBuilder(args) + + if args.generate_test_data: + generate_test_data(builder) + + builder.run(args.step, steps) diff --git a/config/worker/blender/__init__.py b/config/worker/blender/__init__.py new file mode 100644 index 0000000..8b31b65 --- /dev/null +++ b/config/worker/blender/__init__.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import argparse +import os +import pathlib +import re +import subprocess + +from collections import OrderedDict +from typing import Callable, Any + +import worker.utils + + +class CodeBuilder(worker.utils.Builder): + def __init__(self, args: argparse.Namespace): + super().__init__(args, "blender", "blender") + self.needs_full_clean = args.needs_full_clean + self.needs_gpu_binaries = args.needs_gpu_binaries + self.needs_gpu_tests = args.needs_gpu_tests + self.needs_ninja = True + self.python_module = args.python_module + self.build_configuration = args.build_configuration + + track_path: pathlib.Path = self.track_path + + if self.platform in {"darwin", "windows"}: + if len(args.architecture): + self.architecture = args.architecture + + if self.platform == "darwin": + self.build_dir = track_path / f"build_{self.architecture}_{self.build_configuration}" + else: + self.build_dir = track_path / f"build_{self.build_configuration}" + + self.blender_dir = track_path / "blender.git" + self.install_dir = track_path / f"install_{self.build_configuration}" + self.package_dir = track_path / "build_package" + self.build_doc_path = track_path / "build_doc_api" + + def clean(self): + worker.utils.remove_dir(self.install_dir) + worker.utils.remove_dir(self.package_dir) + worker.utils.remove_dir(self.build_doc_path) + + # Call command with in compiler environment. + def call(self, cmd: worker.utils.CmdSequence, env: worker.utils.CmdEnvironment = None) -> int: + cmd_prefix: worker.utils.CmdList = [] + + if self.platform == "darwin": + # On macOS, override Xcode version if requested. + pipeline_config = self.pipeline_config() + xcode = pipeline_config.get("xcode", None) + xcode_version = xcode.get("version", None) if xcode else None + + if xcode_version: + developer_dir = f"/Applications/Xcode-{xcode_version}.app/Contents/Developer" + else: + developer_dir = "/Applications/Xcode.app/Contents/Developer" + + if self.service_env_id == "LOCAL" and not pathlib.Path(developer_dir).exists(): + worker.utils.warning( + f"Skip using non-existent {developer_dir} in LOCAL service environment" + ) + else: + cmd_prefix = ["xcrun"] + env = dict(env) if env else os.environ.copy() + env["DEVELOPER_DIR"] = developer_dir + + elif worker.utils.is_tool("scl"): + pipeline_config = self.pipeline_config() + gcc_version = pipeline_config["gcc"]["version"] + gcc_major_version = gcc_version.split(".")[0] + + # On Rocky + if os.path.exists("/etc/rocky-release"): + # Stub to override configured GCC version, remove when blender build config is fixed + gcc_major_version = "11" + cmd_prefix = ["scl", "enable", f"gcc-toolset-{gcc_major_version}", "--"] + + return worker.utils.call(cmd_prefix + list(cmd), env=env) + + def pipeline_config(self) -> dict: + config_file_path = self.code_path / "build_files" / "config" / "pipeline_config.json" + if not config_file_path.exists(): + config_file_path = config_file_path.with_suffix(".yaml") + if not config_file_path.exists(): + raise Exception(f"Config file [{config_file_path}] not found, aborting") + + with open(config_file_path, "r") as read_file: + if config_file_path.suffix == ".json": + import json + + pipeline_config = json.load(read_file) + else: + import yaml + + pipeline_config = yaml.load(read_file, Loader=yaml.SafeLoader) + + return pipeline_config["buildbot"] + + def blender_command_path(self) -> pathlib.Path: + if self.platform == "darwin": + return self.install_dir / "Blender.app" / "Contents" / "macOS" / "Blender" + elif self.platform == "windows": + return self.install_dir / "blender.exe" + else: + return self.install_dir / "blender" + + def setup_build_environment(self) -> None: + if self.platform != "windows": + return + + # CMake goes first to avoid using chocolaty cpack command. + worker.utils.info("Setting CMake path") + os.environ["PATH"] = "C:\\Program Files\\CMake\\bin" + os.pathsep + os.environ["PATH"] + + worker.utils.info("Setting VC Tools env variables") + windows_build_version = "10.0.19041.0" + os.environ["PATH"] = ( + f"C:\\Program Files (x86)\\Windows Kits\\10\\bin\\{windows_build_version}\\x64" + + os.pathsep + + os.environ["PATH"] + ) + os.environ["PATH"] = ( + "C:\\Program Files (x86)\\WiX Toolset v3.11\\bin" + os.pathsep + os.environ["PATH"] + ) + + if self.architecture == "arm64": + vs_build_tool_path = pathlib.Path( + "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\VC\\Auxiliary\\Build\\vcvarsarm64.bat" + ) + vs_tool_install_dir_suffix = "\\bin\\Hostarm64\\arm64" + else: + vs_build_tool_path = pathlib.Path( + "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\VC\\Auxiliary\\Build\\vcvars64.bat" + ) + vs_tool_install_dir_suffix = "\\bin\\Hostx64\\x64" + + vcvars_output = subprocess.check_output([vs_build_tool_path, "&&", "set"], shell=True) + vcvars_text = vcvars_output.decode("utf-8", "ignore") + + for line in vcvars_text.splitlines(): + match = re.match(r"(.*?)=(.*)", line) + if match: + key = match.group(1) + value = match.group(2) + + if key not in os.environ: + if key not in ["PROMPT", "Path"]: + worker.utils.info(f"Adding key {key}={value}") + os.environ[key] = value + + os.environ["PATH"] = ( + os.environ["VCToolsInstallDir"] + + vs_tool_install_dir_suffix + + os.pathsep + + os.environ["PATH"] + ) + + +def create_argument_parser(steps: worker.utils.BuilderSteps) -> argparse.ArgumentParser: + parser = worker.utils.create_argument_parser(steps=steps) + parser.add_argument("--needs-full-clean", action="store_true", required=False) + parser.add_argument("--needs-gpu-binaries", action="store_true", required=False) + parser.add_argument("--needs-gpu-tests", action="store_true", required=False) + parser.add_argument("--python-module", action="store_true", required=False) + parser.add_argument( + "--build-configuration", + default="release", + type=str, + choices=["release", "asserts", "sanitizer", "debug"], + required=False, + ) + parser.add_argument( + "--architecture", + default="", + type=str, + choices=["arm64", "x86_64", "amd64"], + required=False, + ) + return parser diff --git a/config/worker/blender/benchmark.py b/config/worker/blender/benchmark.py new file mode 100644 index 0000000..5280e9b --- /dev/null +++ b/config/worker/blender/benchmark.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import json +import os +import pathlib +import urllib.request +import sys + +import conf.worker + +import worker.blender +import worker.utils + + + +def create_upload( + builder: worker.blender.CodeBuilder, benchmark_path: pathlib.Path, revision: str +) -> None: + # Create package directory. + branch = builder.branch_id.replace("blender-", "").replace("-release", "") + name = f"{branch}-{builder.platform}-{builder.architecture}" + package_dir = builder.package_dir / name + + worker.utils.remove_dir(package_dir) + os.makedirs(package_dir, exist_ok=True) + + # Fetch existing summary + worker_config = conf.worker.get_config(builder.service_env_id) + base_urls = { + "LOCAL": str(worker_config.buildbot_download_folder), + "UATEST": "https://builder.uatest.blender.org/download", + "PROD": "https://builder.blender.org/download", + } + base_url = base_urls[builder.service_env_id] + + summary_json_url = f"{base_url}/daily/benchmarks/{name}/summary.json" + summary_json_path = package_dir / "summary.json" + try: + if builder.service_env_id == "LOCAL": + worker.utils.copy_file(pathlib.Path(summary_json_url), summary_json_path) + else: + urllib.request.urlretrieve(summary_json_url, summary_json_path) + except Exception as e: + error_msg = str(e) + worker.utils.warning(f"Could not retrieve benchmark summary.json: {error_msg}") + + # Create json files in package directory. + results_json_path = benchmark_path / "results.json" + revision_json_path = package_dir / f"{revision}.json" + + worker.utils.copy_file(results_json_path, revision_json_path) + + summary_json = [] + if summary_json_path.exists(): + summary_json = json.loads(summary_json_path.read_text()) + summary_json += json.loads(results_json_path.read_text()) + summary_json_path.write_text(json.dumps(summary_json, indent=2)) + + # Create html file in package directory. + report_html_path = package_dir / "report.html" + cmd = [ + sys.executable, + builder.code_path / "tests" / "performance" / "benchmark.py", + "graph", + summary_json_path, + "-o", + report_html_path, + ] + worker.utils.call(cmd) + + +def benchmark(builder: worker.blender.CodeBuilder) -> None: + # Parameters + os.chdir(builder.code_path) + revision = worker.utils.check_output(["git", "rev-parse", "HEAD"]) + revision = revision[:12] + blender_command = builder.blender_command_path() + gpu_device = "METAL" if builder.platform == "darwin" else "OPTIX" + background = False if builder.platform == "darwin" else True + + worker.utils.info(f"Benchmark revision {revision}, GPU device {gpu_device}") + + # Create clean benchmark folder + benchmark_path = builder.track_path / "benchmark" / "default" + worker.utils.remove_dir(benchmark_path) + os.makedirs(benchmark_path, exist_ok=True) + + # Initialize configuration + config_py_path = benchmark_path / "config.py" + config_py_text = f""" +devices = ["CPU", "{gpu_device}_0"] +background = {background} +builds = {{"{revision}": "{blender_command}"}} +benchmark_type = "time_series" +""" + config_py_path.write_text(config_py_text) + + # Checkout benchmark files + tests_benchmarks_path = builder.code_path / "tests" / "benchmarks" + if not tests_benchmarks_path.exists(): + benchmarks_url = "https://projects.blender.org/blender/blender-benchmarks.git" + worker.utils.call(["git", "clone", benchmarks_url, tests_benchmarks_path]) + + # Run benchmark + cmd = [ + sys.executable, + builder.code_path / "tests" / "performance" / "benchmark.py", + "list", + ] + worker.utils.call(cmd) + + cmd = [ + sys.executable, + builder.code_path / "tests" / "performance" / "benchmark.py", + "run", + "default", + ] + exit_code = worker.utils.call(cmd, exit_on_error=False) + + # Write results to be uploaded + create_upload(builder, benchmark_path, revision) + + sys.exit(exit_code) diff --git a/config/worker/blender/blender.applescript b/config/worker/blender/blender.applescript new file mode 100644 index 0000000..29b0c2c --- /dev/null +++ b/config/worker/blender/blender.applescript @@ -0,0 +1,25 @@ +tell application "Finder" + tell disk "Blender" + log "applescript: opening [Blender]. This will seem to hang with a pop up dialog on applescript permissions for the first run. You have 10 minutes, get on machine now and push that button !!!" + with timeout of 600 seconds + open + log "applescript: yay it opened !" + log "applescript: setting current view" + set current view of container window to icon view + set toolbar visible of container window to false + set statusbar visible of container window to false + set the bounds of container window to {100, 100, 640, 472} + set theViewOptions to icon view options of container window + set arrangement of theViewOptions to not arranged + set icon size of theViewOptions to 128 + set background picture of theViewOptions to file ".background:background.tif" + set position of item " " of container window to {400, 190} + set position of item "blender.app" of container window to {135, 190} + log "applescript: updating applications" + update without registering applications + delay 5 + log "applescript: closing" + close + end timeout + end tell +end tell diff --git a/config/worker/blender/bundle_dmg.py b/config/worker/blender/bundle_dmg.py new file mode 100644 index 0000000..cee3a33 --- /dev/null +++ b/config/worker/blender/bundle_dmg.py @@ -0,0 +1,473 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import os +import re +import time +import subprocess +import platform +import pathlib +import tempfile +import typing + +import worker.utils + +# Extra size which is added on top of actual files size when estimating size +# of destination DNG. +_extra_dmg_size_in_bytes = 800 * 1024 * 1024 + +################################################################################ +# Common utilities + + +def get_directory_size(root_directory: pathlib.Path) -> int: + """ + Get size of directory on disk + """ + + total_size = 0 + for file in root_directory.glob("**/*"): + total_size += file.lstat().st_size + return total_size + + +################################################################################ +# DMG bundling specific logic + + +def collect_app_bundles(source_dir: pathlib.Path) -> typing.List[pathlib.Path]: + """ + Collect all app bundles which are to be put into DMG + + If the source directory points to FOO.app it will be the only app bundle + packed. + + Otherwise all .app bundles from given directory are placed to a single + DMG. + """ + + if source_dir.name.endswith(".app"): + return [source_dir] + + app_bundles = [] + for filename in source_dir.glob("*"): + if not filename.is_dir(): + continue + if not filename.name.endswith(".app"): + continue + + app_bundles.append(filename) + + return app_bundles + + +def collect_and_log_app_bundles(source_dir: pathlib.Path) -> typing.List[pathlib.Path]: + app_bundles = collect_app_bundles(source_dir) + + if not app_bundles: + worker.utils.info("No app bundles found for packing") + return [] + + worker.utils.info(f"Found {len(app_bundles)} to pack:") + for app_bundle in app_bundles: + worker.utils.info(f"- {app_bundle}") + + return app_bundles + + +def estimate_dmg_size(app_bundles: typing.List[pathlib.Path]) -> int: + """ + Estimate size of DMG to hold requested app bundles + + The size is based on actual size of all files in all bundles plus some + space to compensate for different size-on-disk plus some space to hold + codesign signatures. + + Is better to be on a high side since the empty space is compressed, but + lack of space might cause silent failures later on. + """ + + app_bundles_size = 0 + for app_bundle in app_bundles: + app_bundles_size += get_directory_size(app_bundle) + + return app_bundles_size + _extra_dmg_size_in_bytes + + +def copy_app_bundles(app_bundles: typing.List[pathlib.Path], dir_path: pathlib.Path) -> None: + """ + Copy all bundles to a given directory + + This directory is what the DMG will be created from. + """ + for app_bundle in app_bundles: + destination_dir_path = dir_path / app_bundle.name + + worker.utils.info(f"Copying app bundle [{app_bundle}] to [{dir_path}]") + + worker.utils.copy_dir(app_bundle, destination_dir_path) + + # Only chmod if we can;t get cmake install to do it - james + # for r, d, f in os.walk(destination_dir_path): + # worker.utils.info(f'chmoding [{r}] -> 0o755') + # os.chmod(r, 0o755) + + +def get_main_app_bundle(app_bundles: typing.List[pathlib.Path]) -> pathlib.Path: + """ + Get application bundle main for the installation + """ + return app_bundles[0] + + +def create_dmg_image( + app_bundles: typing.List[pathlib.Path], dmg_file_path: pathlib.Path, volume_name: str +) -> None: + """ + Create DMG disk image and put app bundles in it + + No DMG configuration or codesigning is happening here. + """ + if dmg_file_path.exists(): + worker.utils.info(f"Removing existing writable DMG {dmg_file_path}...") + worker.utils.remove_file(dmg_file_path) + + temp_content_path = tempfile.TemporaryDirectory(prefix="blender-dmg-content-") + worker.utils.info(f"Preparing directory with app bundles for the DMG [{temp_content_path}]") + with temp_content_path as content_dir_str: + # Copy all bundles to a clean directory. + content_dir_path = pathlib.Path(content_dir_str) + # worker.utils.info(f'content_dir_path={content_dir_path}') + copy_app_bundles(app_bundles, content_dir_path) + + # Estimate size of the DMG. + dmg_size = estimate_dmg_size(app_bundles) + worker.utils.info(f"Estimated DMG size: [{dmg_size:,}] bytes.") + + # Create the DMG. + worker.utils.info(f"Creating writable DMG [{dmg_file_path}]") + command = ( + "hdiutil", + "create", + "-size", + str(dmg_size), + "-fs", + "HFS+", + "-srcfolder", + content_dir_path, + "-volname", + volume_name, + "-format", + "UDRW", + "-mode", + "755", + dmg_file_path, + ) + + worker.utils.call(command) + + +def get_writable_dmg_file_path(dmg_file_path: pathlib.Path) -> pathlib.Path: + """ + Get file path for writable DMG image + """ + parent = dmg_file_path.parent + return parent / (dmg_file_path.stem + "-temp.dmg") + + +def mount_readwrite_dmg(dmg_file_path: pathlib.Path) -> None: + """ + Mount writable DMG + + Mounting point would be /Volumes/ + """ + + worker.utils.info(f"Mounting read-write DMG ${dmg_file_path}") + cmd: worker.utils.CmdSequence = [ + "hdiutil", + "attach", + "-readwrite", + "-noverify", + "-noautoopen", + dmg_file_path, + ] + worker.utils.call(cmd) + + +def get_mount_directory_for_volume_name(volume_name: str) -> pathlib.Path: + """ + Get directory under which the volume will be mounted + """ + + return pathlib.Path("/Volumes") / volume_name + + +def eject_volume(volume_name: str) -> None: + """ + Eject given volume, if mounted + """ + mount_directory = get_mount_directory_for_volume_name(volume_name) + if not mount_directory.exists(): + return + mount_directory_str = str(mount_directory) + + worker.utils.info(f"Ejecting volume [{volume_name}]") + + # First try through Finder, as sometimes diskutil fails for unknown reasons. + command = [ + "osascript", + "-e", + f"""tell application "Finder" to eject (every disk whose name is "{volume_name}")""", + ] + worker.utils.call(command) + if not mount_directory.exists(): + return + + # Figure out which device to eject. + mount_output = subprocess.check_output(["mount"]).decode() + device = "" + for line in mount_output.splitlines(): + if f"on {mount_directory_str} (" not in line: + continue + tokens = line.split(" ", 3) + if len(tokens) < 3: + continue + if tokens[1] != "on": + continue + if device: + raise Exception(f"Multiple devices found for mounting point [{mount_directory}]") + device = tokens[0] + + if not device: + raise Exception(f"No device found for mounting point [{mount_directory}]") + + worker.utils.info(f"[{mount_directory}] is mounted as device [{device}], ejecting...") + command = ["diskutil", "eject", device] + worker.utils.call(command) + + +def copy_background_if_needed( + background_image_file_path: pathlib.Path, mount_directory: pathlib.Path +) -> None: + """ + Copy background to the DMG + + If the background image is not specified it will not be copied. + """ + + if not background_image_file_path: + worker.utils.info("No background image provided.") + return + + destination_dir = mount_directory / ".background" + destination_dir.mkdir(exist_ok=True) + + destination_file_path = destination_dir / background_image_file_path.name + + worker.utils.info( + f"Copying background image [{background_image_file_path}] to [{destination_file_path}]" + ) + worker.utils.copy_file(background_image_file_path, destination_file_path) + + +def create_applications_link(mount_directory: pathlib.Path) -> None: + """ + Create link to /Applications in the given location + """ + worker.utils.info(f"Creating link to /Applications -> {mount_directory}") + target_path = mount_directory / " " + cmd: worker.utils.CmdSequence = ["ln", "-s", "/Applications", target_path] + worker.utils.call(cmd) + + +def run_applescript_file_path( + applescript_file_path: pathlib.Path, + volume_name: str, + app_bundles: typing.List[pathlib.Path], + background_image_file_path: pathlib.Path, +) -> None: + """ + Run given applescript to adjust look and feel of the DMG + """ + main_app_bundle = get_main_app_bundle(app_bundles) + + architecture = platform.machine().lower() + # needs_run_applescript = (architecture != "x86_64") + needs_run_applescript = True + + if not needs_run_applescript: + worker.utils.info(f"Having issues with apple script on [{architecture}], skipping !") + return + + temp_script_file_path = tempfile.NamedTemporaryFile(mode="w", suffix=".applescript") + with temp_script_file_path as temp_applescript_file: + worker.utils.info( + f"Adjusting applescript [{temp_script_file_path.name}] for volume name [{volume_name}]" + ) + # Adjust script to the specific volume name. + with open(applescript_file_path, mode="r") as input_file: + worker.utils.info("Start script update") + for line in input_file.readlines(): + stripped_line = line.strip() + if stripped_line.startswith("tell disk"): + line = re.sub('tell disk ".*"', f'tell disk "{volume_name}"', line) + elif stripped_line.startswith("set background picture"): + if not background_image_file_path: + continue + else: + background_image_short = f".background:{background_image_file_path.name}" + line = re.sub('to file ".*"', f'to file "{background_image_short}"', line) + line = line.replace("blender.app", main_app_bundle.name) + stripped_line = line.rstrip("\r\n") + worker.utils.info(f"line={stripped_line}") + temp_applescript_file.write(line) + + temp_applescript_file.flush() + worker.utils.info("End script update") + + # This does not help issues when running applescript + worker.utils.info("Updating permissions") + os.chmod(temp_script_file_path.name, 0o755) + + # Setting flags to this applescript will fail execution, not permitted + # command = ['chflags', "uchg", temp_script_file_path.name] + # worker.utils.call(command) + + command = ["osascript", "-s", "o", temp_script_file_path.name] + worker.utils.call(command) + + # NOTE: This is copied from bundle.sh. The exact reason for sleep is + # still remained a mystery. + worker.utils.info("Waiting for applescript...") + time.sleep(5) + + +def compress_dmg(writable_dmg_file_path: pathlib.Path, final_dmg_file_path: pathlib.Path) -> None: + """ + Compress temporary read-write DMG + """ + cmd: worker.utils.CmdSequence = [ + "hdiutil", + "convert", + writable_dmg_file_path, + "-format", + "UDZO", + "-o", + final_dmg_file_path, + ] + + if final_dmg_file_path.exists(): + worker.utils.info(f"Removing old compressed DMG [{final_dmg_file_path}]") + worker.utils.remove_file(final_dmg_file_path) + + worker.utils.info("Compressing disk image...") + worker.utils.call(cmd) + + +def create_final_dmg( + app_bundles: typing.List[pathlib.Path], + dmg_file_path: pathlib.Path, + background_image_file_path: pathlib.Path, + volume_name: str, + applescript_file_path: pathlib.Path, +) -> None: + """ + Create DMG with all app bundles + + Will take care configuring background + """ + + worker.utils.info("Running all routines to create final DMG") + + writable_dmg_file_path = get_writable_dmg_file_path(dmg_file_path) + worker.utils.info(f"Mouting volume [{volume_name}]") + mount_directory = get_mount_directory_for_volume_name(volume_name) + worker.utils.info(f"Mount at [{mount_directory}]") + + # Make sure volume is not mounted. + # If it is mounted it will prevent removing old DMG files and could make + # it so app bundles are copied to the wrong place. + eject_volume(volume_name) + + worker.utils.info(f"Creating image [{writable_dmg_file_path}] to [{volume_name}]") + create_dmg_image(app_bundles, writable_dmg_file_path, volume_name) + + worker.utils.info(f"Mount r/w mode [{writable_dmg_file_path}]") + mount_readwrite_dmg(writable_dmg_file_path) + + copy_background_if_needed(background_image_file_path, mount_directory) + create_applications_link(mount_directory) + + run_applescript_file_path( + applescript_file_path, volume_name, app_bundles, background_image_file_path + ) + + eject_volume(volume_name) + + compress_dmg(writable_dmg_file_path, dmg_file_path) + worker.utils.remove_file(writable_dmg_file_path) + + +def ensure_dmg_extension(filepath: pathlib.Path) -> pathlib.Path: + """ + Make sure given file have .dmg extension + """ + + if filepath.suffix != ".dmg": + return filepath.with_suffix(f"{filepath.suffix}.dmg") + return filepath + + +def get_dmg_file_path( + requested_file_path: pathlib.Path, app_bundles: typing.List[pathlib.Path] +) -> pathlib.Path: + """ + Get full file path for the final DMG image + + Will use the provided one when possible, otherwise will deduct it from + app bundles. + + If the name is deducted, the DMG is stored in the current directory. + """ + + if requested_file_path: + return ensure_dmg_extension(requested_file_path.absolute()) + + # TODO(sergey): This is not necessarily the main one. + main_bundle = app_bundles[0] + # Strip .app from the name + return pathlib.Path(main_bundle.name[:-4] + ".dmg").absolute() + + +def get_volume_name_from_dmg_file_path(dmg_file_path: pathlib.Path) -> str: + """ + Deduct volume name from the DMG path + + Will use first part of the DMG file name prior to dash. + """ + + tokens = dmg_file_path.stem.split("-") + words = tokens[0].split() + + return " ".join(word.capitalize() for word in words) + + +def bundle( + source_dir: pathlib.Path, + dmg_file_path: pathlib.Path, + applescript_file_path: pathlib.Path, + background_image_file_path: pathlib.Path, +) -> None: + app_bundles = collect_and_log_app_bundles(source_dir) + for app_bundle in app_bundles: + worker.utils.info(f"App bundle path [{app_bundle}]") + + dmg_file_path = get_dmg_file_path(dmg_file_path, app_bundles) + volume_name = get_volume_name_from_dmg_file_path(dmg_file_path) + + worker.utils.info(f"Will produce DMG [{dmg_file_path.name}]") + + create_final_dmg( + app_bundles, dmg_file_path, background_image_file_path, volume_name, applescript_file_path + ) diff --git a/config/worker/blender/compile.py b/config/worker/blender/compile.py new file mode 100644 index 0000000..07ff990 --- /dev/null +++ b/config/worker/blender/compile.py @@ -0,0 +1,534 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import multiprocessing +import os +import platform +import pathlib + +from typing import Dict +from pathlib import Path + +import worker.blender +import worker.utils + + +def fetch_ideal_cpu_count(estimate_core_memory_in_mb: int) -> int: + """Fetch cpu ideal for the building process based on machine info""" + worker.utils.info(f"estimate_core_memory_in_mb={estimate_core_memory_in_mb}") + + total_cpu_count = multiprocessing.cpu_count() + worker.utils.info(f"total_cpu_count={total_cpu_count}") + + ideal_cpu_count = total_cpu_count + spare_cpu_count = 2 + + if platform.system().lower() != "darwin": + worker.utils.info(f"In current path {os.getcwd()}") + import psutil + + virtual_memory = psutil.virtual_memory() + worker.utils.info(f"virtual_memory={virtual_memory}") + + total_memory_in_bytes = virtual_memory.total + worker.utils.info(f"total_memory_in_bytes={total_memory_in_bytes}") + + available_memory_in_bytes = virtual_memory.available + worker.utils.info(f"available_memory_in_bytes={available_memory_in_bytes}") + + usable_memory_in_bytes = available_memory_in_bytes + worker.utils.info(f"usable_memory_in_bytes={usable_memory_in_bytes}") + + estimate_memory_per_code_in_bytes = estimate_core_memory_in_mb * 1024 * 1024 + worker.utils.info(f"estimate_memory_per_code_in_bytes={estimate_memory_per_code_in_bytes}") + + capable_cpu_count = int(total_memory_in_bytes / estimate_memory_per_code_in_bytes) + worker.utils.info(f"capable_cpu_count={capable_cpu_count}") + + min_cpu_count = min(total_cpu_count, capable_cpu_count) + worker.utils.info(f"min_cpu_count={min_cpu_count}") + + ideal_cpu_count = min_cpu_count if min_cpu_count <= 8 else min_cpu_count - spare_cpu_count + worker.utils.info(f"ideal_cpu_count={ideal_cpu_count}") + return ideal_cpu_count + + +def get_cmake_build_type(builder: worker.blender.CodeBuilder) -> str: + if builder.build_configuration == "debug": + return "Debug" + elif builder.build_configuration == "sanitizer": + # No reliable ASAN on Windows currently. + if builder.platform != "windows": + return "RelWithDebInfo" + else: + return "Release" + else: + return "Release" + + +def get_cmake_options(builder: worker.blender.CodeBuilder) -> worker.utils.CmdSequence: + needs_gtest_compile = not builder.python_module + + with_gtests_state = "ON" if needs_gtest_compile else "OFF" + with_gpu_binaries_state = "ON" if builder.needs_gpu_binaries else "OFF" + with_gpu_tests = False + + buildbotConfig = builder.pipeline_config() + + # This is meant for stable build compilation + config_file_path = "build_files/cmake/config/blender_release.cmake" + + platform_config_file_path = None + if builder.platform == "darwin": + platform_config_file_path = "build_files/buildbot/config/blender_macos.cmake" + elif builder.platform == "linux": + platform_config_file_path = "build_files/buildbot/config/blender_linux.cmake" + elif builder.platform == "windows": + platform_config_file_path = "build_files/buildbot/config/blender_windows.cmake" + + if platform_config_file_path: + worker.utils.info(f'Trying platform-specific buildbot configuration "{platform_config_file_path}"') + if (Path(builder.blender_dir) / platform_config_file_path).exists(): + worker.utils.info(f'Using platform-specific buildbot configuration "{platform_config_file_path}"') + config_file_path = platform_config_file_path + else: + worker.utils.info(f'Using generic buildbot configuration "{config_file_path}"') + + # Must be first so that we can override some of the options found in the file + options = ["-C", os.path.join(builder.blender_dir, config_file_path)] + + # Optional build as Python module. + if builder.python_module: + bpy_config_file_path = "build_files/cmake/config/bpy_module.cmake" + options += ["-C", os.path.join(builder.blender_dir, bpy_config_file_path)] + options += ["-DWITH_INSTALL_PORTABLE=ON"] + + can_enable_oneapi_binaries = True + if builder.service_env_id != "PROD": + # UATEST machines are too slow currently. + worker.utils.info(f'Disabling oneAPI binaries on "{builder.service_env_id}"') + can_enable_oneapi_binaries = False + if builder.patch_id: + # No enough throughput of the systems to cover AoT oneAPI binaries for patches. + worker.utils.info("Disabling oneAPI binaries for patch build") + can_enable_oneapi_binaries = False + if builder.track_id == "vexp": + # Only enable AoT oneAPI binaries for main and release branches. + worker.utils.info("Disabling oneAPI binaries for branch build") + can_enable_oneapi_binaries = False + + # Add platform specific generator and configs + if builder.platform == "darwin": + if builder.needs_ninja: + options += ["-G", "Ninja"] + else: + options += ["-G", "Unix Makefiles"] + + options += [f"-DCMAKE_OSX_ARCHITECTURES:STRING={builder.architecture}"] + + elif builder.platform == "linux": + if builder.needs_ninja: + options += ["-G", "Ninja"] + else: + options += ["-G", "Unix Makefiles"] + + elif builder.platform == "windows": + if builder.needs_ninja: + # set CC=%LLVM_DIR%\bin\clang-cl + # set CXX=%LLVM_DIR%\bin\clang-cl + # set CFLAGS=-m64 -fmsc-version=1922 + # set CXXFLAGS=-m64 -fmsc-version=1922 + vc_tools_install_dir = os.environ.get("VCToolsInstallDir") + if not vc_tools_install_dir: + raise BaseException("Missing environment variable VCToolsInstallDir") + + vc_tool_install_path = pathlib.PureWindowsPath(vc_tools_install_dir) + if builder.architecture == "arm64": + compiler_file_path="C:/Program Files/LLVM/bin/clang-cl.exe" + compiler_file_path="C:/Program Files/LLVM/bin/clang-cl.exe" + linker_file_path="C:/Program Files/LLVM/bin/lld-link.exe" + else: + vs_tool_install_dir_suffix = "bin/Hostx64/x64" + compiler_file_path = str(vc_tool_install_path / f"{vs_tool_install_dir_suffix}/cl.exe") + linker_file_path = str(vc_tool_install_path / f"{vs_tool_install_dir_suffix}/link.exe") + + options += ["-G", "Ninja"] + # -DWITH_WINDOWS_SCCACHE=On + options += [ + f"-DCMAKE_C_COMPILER:FILEPATH={compiler_file_path}", + f"-DCMAKE_CXX_COMPILER:FILEPATH={compiler_file_path}", + ] + # options += ["-DCMAKE_EXE_LINKER_FLAGS:STRING=/machine:x64"] + options += [f"-DCMAKE_LINKER:FILEPATH={linker_file_path}"] + # Skip the test, it does not work + options += ["-DCMAKE_C_COMPILER_WORKS=1"] + options += ["-DCMAKE_CXX_COMPILER_WORKS=1"] + + else: + if builder.architecture == "arm64": + options += ["-G", "Visual Studio 17 2022", "-A", "arm64"] + else: + options += ["-G", "Visual Studio 16 2019", "-A", "x64"] + + # Add configured overrides + platform_architecure = f"{builder.platform}-{builder.architecture}" + + cmake_overrides: Dict[str, str] = {} + cmake_overrides.update(buildbotConfig["cmake"]["default"]["overrides"]) + cmake_overrides.update(buildbotConfig["cmake"][platform_architecure]["overrides"]) + + # Disallow certain options + restricted_key_patterns = [ + "POSTINSTALL_SCRIPT", + "OPTIX_", + "CMAKE_OSX_ARCHITECTURES", + "CMAKE_BUILD_TYPE", + "CMAKE_INSTALL_PREFIX", + "WITH_GTESTS", + "CUDA", + "WITH_CYCLES", + "CYCLES_CUDA", + ] + + for cmake_key in cmake_overrides.keys(): + for restricted_key_pattern in restricted_key_patterns: + if restricted_key_pattern in cmake_key: + raise Exception(f"CMake key [{cmake_key}] cannot be overriden, aborting") + + for cmake_key, cmake_value in cmake_overrides.items(): + options += [f"-D{cmake_key}={cmake_value}"] + + cmake_build_type = get_cmake_build_type(builder) + options += [f"-DCMAKE_BUILD_TYPE:STRING={cmake_build_type}"] + + if builder.build_configuration == "sanitizer": + # No reliable ASAN on Windows currently. + if builder.platform != "windows": + options += ["-DWITH_COMPILER_ASAN=ON"] + options += ["-DWITH_ASSERT_RELEASE=ON"] + # Avoid buildbot timeouts, see blender/blender#116635. + options += ["-DWITH_UNITY_BUILD=OFF"] + elif builder.build_configuration == "asserts": + options += ["-DWITH_ASSERT_RELEASE=ON"] + + options += [f"-DCMAKE_INSTALL_PREFIX={builder.install_dir}"] + + options += ["-DWITH_INSTALL_COPYRIGHT=ON"] + + options += [f"-DWITH_GTESTS={with_gtests_state}"] + + if builder.platform == "windows": + if builder.architecture != "arm64": + # CUDA + HIP + oneAPI on Windows + options += [f"-DWITH_CYCLES_CUDA_BINARIES={with_gpu_binaries_state}"] + options += [f"-DWITH_CYCLES_HIP_BINARIES={with_gpu_binaries_state}"] + if can_enable_oneapi_binaries: + options += [f"-DWITH_CYCLES_ONEAPI_BINARIES={with_gpu_binaries_state}"] + options += ["-DSYCL_OFFLINE_COMPILER_PARALLEL_JOBS=2"] + else: + options += ["-DWITH_CYCLES_ONEAPI_BINARIES=OFF"] + if "hip" in buildbotConfig: + hip_version = buildbotConfig["hip"]["version"] + else: + hip_version = "5.2.21440" + if "ocloc" in buildbotConfig: + ocloc_version = buildbotConfig["ocloc"]["version"] + else: + ocloc_version = "dev_01" + options += [f"-DHIP_ROOT_DIR=C:/ProgramData/AMD/HIP/hip_sdk_{hip_version}"] + options += ["-DHIP_PERL_DIR=C:/ProgramData/AMD/HIP/strawberry/perl/bin"] + options += [f"-DOCLOC_INSTALL_DIR=C:/ProgramData/Intel/ocloc/ocloc_{ocloc_version}"] + elif builder.platform == "linux": + # CUDA on Linux + options += [f"-DWITH_CYCLES_CUDA_BINARIES={with_gpu_binaries_state}"] + options += [f"-DWITH_CYCLES_HIP_BINARIES={with_gpu_binaries_state}"] + if can_enable_oneapi_binaries: + options += [f"-DWITH_CYCLES_ONEAPI_BINARIES={with_gpu_binaries_state}"] + options += ["-DSYCL_OFFLINE_COMPILER_PARALLEL_JOBS=2"] + else: + options += ["-DWITH_CYCLES_ONEAPI_BINARIES=OFF"] + + # Directory changed to just /opt/rocm in 6.x + rocm_path = pathlib.Path("/opt/rocm/hip") + if not rocm_path.exists(): + rocm_path = pathlib.Path("/opt/rocm") + options += [f"-DHIP_ROOT_DIR:PATH={rocm_path}"] + + # GPU render tests support Linux + NVIDIA currently + if builder.needs_gpu_tests: + with_gpu_tests = True + if builder.needs_gpu_binaries: + options += ["-DCYCLES_TEST_DEVICES=CPU;OPTIX"] + elif builder.platform == "darwin": + # Metal on macOS + if builder.architecture == "arm64": + if builder.needs_gpu_tests: + with_gpu_tests = True + options += ["-DCYCLES_TEST_DEVICES=CPU;METAL"] + + if with_gpu_tests: + # Needs X11 or Wayland, and fails with xvfb to emulate X11. + # options += [f"-DWITH_GPU_DRAW_TESTS=ON"] + options += ["-DWITH_GPU_RENDER_TESTS=ON"] + options += ["-DWITH_GPU_RENDER_TESTS_SILENT=OFF"] + options += ["-DWITH_COMPOSITOR_REALTIME_TESTS=ON"] + + if "optix" in buildbotConfig: + optix_version = buildbotConfig["optix"]["version"] + + if builder.platform == "windows" and builder.architecture != "arm64": + options += [ + f"-DOPTIX_ROOT_DIR:PATH=C:/ProgramData/NVIDIA Corporation/OptiX SDK {optix_version}" + ] + elif builder.platform == "linux": + optix_base_dir = pathlib.Path.home() / ".devops" / "apps" + options += [ + f"-DOPTIX_ROOT_DIR:PATH={optix_base_dir}/NVIDIA-OptiX-SDK-{optix_version}-linux64-x86_64" + ] + + # Blender 4.3 has switched to pre-compiled HIP-RT libraries. + if "hiprt" in buildbotConfig: + hiprt_version = buildbotConfig["hiprt"]["version"] + + if builder.platform == "windows" and builder.architecture != "arm64": + options += [ + f"-DHIPRT_ROOT_DIR:PATH=C:/ProgramData/AMD/HIP/hiprtsdk-{hiprt_version}/hiprt{hiprt_version}" + ] + elif builder.platform == "linux": + hiprt_base_dir = pathlib.Path.home() / ".devops" / "apps" + options += [ + f"-DHIPRT_ROOT_DIR:PATH={hiprt_base_dir}/hiprtsdk-{hiprt_version}/hiprt{hiprt_version}" + ] + + # Enable option to verify enabled libraries and features did not get disabled. + options += ["-DWITH_STRICT_BUILD_OPTIONS=ON"] + + needs_cuda_compile = builder.needs_gpu_binaries + if builder.needs_gpu_binaries: + try: + cuda10_version = buildbotConfig["cuda10"]["version"] + except: + cuda10_version = buildbotConfig["sdks"]["cuda10"]["version"] + + cuda10_folder_version = ".".join(cuda10_version.split(".")[:2]) + + try: + cuda11_version = buildbotConfig["cuda11"]["version"] + except: + cuda11_version = buildbotConfig["sdks"]["cuda11"]["version"] + + cuda11_folder_version = ".".join(cuda11_version.split(".")[:2]) + + try: + cuda12_version = buildbotConfig["cuda12"]["version"] + cuda12_folder_version = ".".join(cuda12_version.split(".")[:2]) + have_cuda12 = True + except: + have_cuda12 = False + + if builder.platform == "windows" and builder.architecture != "arm64": + # CUDA 10 + cuda10_path = pathlib.Path( + f"C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v{cuda10_folder_version}" + ) + if not cuda10_path.exists(): + raise Exception( + f"Was not able to find CUDA path [{cuda10_path}] for version [{cuda10_version}], aborting" + ) + cuda10_file_path = cuda10_path / "bin" / "nvcc.exe" + + options += [f"-DCUDA10_TOOLKIT_ROOT_DIR:PATH={cuda10_path}"] + options += [f"-DCUDA10_NVCC_EXECUTABLE:FILEPATH={cuda10_file_path}"] + + # CUDA 11 + cuda11_path = pathlib.Path( + f"C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v{cuda11_folder_version}" + ) + if not cuda11_path.exists(): + raise Exception( + f"Was not able to find CUDA path [{cuda11_path}] for version [{cuda11_version}], aborting" + ) + cuda11_file_path = cuda11_path / "bin" / "nvcc.exe" + + # CUDA 12 + if have_cuda12: + cuda12_path = pathlib.Path( + f"C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v{cuda12_folder_version}" + ) + if not cuda12_path.exists(): + raise Exception( + f"Was not able to find CUDA path [{cuda12_path}] for version [{cuda12_version}], aborting" + ) + cuda12_file_path = cuda12_path / "bin" / "nvcc.exe" + + options += [f"-DCUDA11_TOOLKIT_ROOT_DIR:PATH={cuda11_path}"] + options += [f"-DCUDA11_NVCC_EXECUTABLE:FILEPATH={cuda11_file_path}"] + + options += [f"-DCUDA_TOOLKIT_ROOT_DIR:PATH={cuda12_path}"] + options += [f"-DCUDA_NVCC_EXECUTABLE:FILEPATH={cuda12_file_path}"] + else: + options += [f"-DCUDA_TOOLKIT_ROOT_DIR:PATH={cuda11_path}"] + options += [f"-DCUDA_NVCC_EXECUTABLE:FILEPATH={cuda11_file_path}"] + + elif builder.platform == "linux": + # CUDA 10 + cuda10_path = pathlib.Path(f"/usr/local/cuda-{cuda10_folder_version}") + if not cuda10_path.exists(): + raise Exception( + f"Was not able to find CUDA path [{cuda10_path}] for version [{cuda10_version}], aborting" + ) + cuda10_file_path = cuda10_path / "bin" / "nvcc" + + # CUDA 11 + cuda11_path = pathlib.Path(f"/usr/local/cuda-{cuda11_folder_version}") + if not cuda11_path.exists(): + raise Exception( + f"Was not able to find CUDA path [{cuda11_path}] for version [{cuda11_version}], aborting" + ) + cuda11_file_path = cuda11_path / "bin" / "nvcc" + + # CUDA 12 + if have_cuda12: + cuda12_path = pathlib.Path(f"/usr/local/cuda-{cuda12_folder_version}") + if not cuda12_path.exists(): + raise Exception( + f"Was not able to find CUDA path [{cuda12_path}] for version [{cuda12_version}], aborting" + ) + cuda12_file_path = cuda12_path / "bin" / "nvcc" + + # CUDA 10, must provide compatible host compiler. + options += [f"-DCUDA10_TOOLKIT_ROOT_DIR:PATH={cuda10_path}"] + + if pathlib.Path( + "/etc/rocky-release" + ).exists(): # We check for Rocky. Version 8 has GCC 8 in /usr/bin + options += [f"-DCUDA10_NVCC_EXECUTABLE:STRING={cuda10_file_path}"] + options += ["-DCUDA_HOST_COMPILER=/usr/bin/gcc"] + else: + # Use new CMake option. + options += [f"-DCUDA10_NVCC_EXECUTABLE:STRING={cuda10_file_path}"] + options += ["-DCUDA_HOST_COMPILER=/opt/rh/devtoolset-8/root/usr/bin/gcc"] + + # CUDA 11 or 12. + if have_cuda12: + options += [f"-DCUDA11_TOOLKIT_ROOT_DIR:PATH={cuda11_path}"] + options += [f"-DCUDA11_NVCC_EXECUTABLE:STRING={cuda11_file_path}"] + + options += [f"-DCUDA_TOOLKIT_ROOT_DIR:PATH={cuda12_path}"] + options += [f"-DCUDA_NVCC_EXECUTABLE:FILEPATH={cuda12_file_path}"] + else: + options += [f"-DCUDA_TOOLKIT_ROOT_DIR:PATH={cuda11_path}"] + options += [f"-DCUDA_NVCC_EXECUTABLE:FILEPATH={cuda11_file_path}"] + + else: + worker.utils.info("Skipping gpu compilation as requested") + + return options + + +def clean_directories(builder: worker.blender.CodeBuilder) -> None: + worker.utils.info(f"Cleaning directory [{builder.install_dir})] from the previous run") + worker.utils.remove_dir(builder.install_dir) + + os.makedirs(builder.build_dir, exist_ok=True) + + worker.utils.info("Remove buildinfo files to re-generate them") + for build_info_file_name in ( + "buildinfo.h", + "buildinfo.h.txt", + ): + full_path = builder.build_dir / "source" / "creator" / build_info_file_name + if full_path.exists(): + worker.utils.info(f"Removing file [{full_path}]") + worker.utils.remove_file(full_path) + + +def cmake_configure(builder: worker.blender.CodeBuilder) -> None: + cmake_cache_file_path = builder.build_dir / "CMakeCache.txt" + if cmake_cache_file_path.exists(): + worker.utils.info("Removing CMake cache") + worker.utils.remove_file(cmake_cache_file_path) + + worker.utils.info("CMake configure options") + cmake_options = get_cmake_options(builder) + cmd = ["cmake", "-S", builder.blender_dir, "-B", builder.build_dir] + list(cmake_options) + builder.call(cmd) + + # This hack does not work as expected, since cmake cache is the always updated, we end up recompiling on each compile step, code, gpu and install + needs_cmake_cache_hack = False + if needs_cmake_cache_hack and pathlib.Path("/usr/lib64/libpthread.a").exists(): + # HACK: The detection for lib pthread does not work on CentOS 7 + worker.utils.warning(f"Hacking file [{cmake_cache_file_path}]") + tmp_cmake_cache_file_path = builder.build_dir / "CMakeCache.txt.tmp" + fin = open(cmake_cache_file_path) + fout = open(tmp_cmake_cache_file_path, "wt") + for line in fin: + # worker.utils.info(line) + if "OpenMP_pthread_LIBRARY:FILEPATH=OpenMP_pthread_LIBRARY-NOTFOUND" in line: + worker.utils.warning( + "Replacing [OpenMP_pthread_LIBRARY-NOTFOUND] to [/usr/lib64/libpthread.a]" + ) + line = line.replace( + "OpenMP_pthread_LIBRARY:FILEPATH=OpenMP_pthread_LIBRARY-NOTFOUND", + "OpenMP_pthread_LIBRARY:FILEPATH=/usr/lib64/libpthread.a", + ) + fout.write(line) + fin.close() + fout.close() + worker.utils.warning(f"Updating [{cmake_cache_file_path}]") + os.replace(tmp_cmake_cache_file_path, cmake_cache_file_path) + + +def cmake_build(builder: worker.blender.CodeBuilder, do_install: bool) -> None: + if builder.track_id in ["vdev", "v430"]: + if builder.platform == "windows": + estimate_gpu_memory_in_mb = 6000 + else: + estimate_gpu_memory_in_mb = 4000 + else: + estimate_gpu_memory_in_mb = 6000 + + estimate_core_memory_in_mb = estimate_gpu_memory_in_mb if builder.needs_gpu_binaries else 1000 + ideal_cpu_count = fetch_ideal_cpu_count(estimate_core_memory_in_mb) + + # Enable verbose building to make ninja to output more often. + # It should help with slow build commands like OneAPI, as well as will help + # troubleshooting situations when the compile-gpu step times out. + needs_verbose = builder.needs_gpu_binaries + + build_type = get_cmake_build_type(builder) + cmd = ["cmake", "--build", builder.build_dir, "--config", build_type] + cmd += ["--parallel", f"{ideal_cpu_count}"] + if do_install: + cmd += ["--target", "install"] + + if needs_verbose: + cmd += ["--verbose"] + + builder.call(cmd) + + +def compile_code(builder: worker.blender.CodeBuilder) -> None: + builder.needs_gpu_binaries = False + builder.setup_build_environment() + clean_directories(builder) + cmake_configure(builder) + cmake_build(builder, False) + + +def compile_gpu(builder: worker.blender.CodeBuilder) -> None: + if builder.platform == "darwin": + worker.utils.info("Compile GPU not required on macOS") + return + + builder.needs_gpu_binaries = True + builder.setup_build_environment() + cmake_configure(builder) + cmake_build(builder, False) + + +def compile_install(builder: worker.blender.CodeBuilder) -> None: + builder.setup_build_environment() + cmake_configure(builder) + cmake_build(builder, True) diff --git a/config/worker/blender/cpack_post.cmake b/config/worker/blender/cpack_post.cmake new file mode 100644 index 0000000..ce44bef --- /dev/null +++ b/config/worker/blender/cpack_post.cmake @@ -0,0 +1,34 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# This is a script which is used as POST-INSTALL one for regular CMake's +# INSTALL target. +# +# It is used by buildbot workers to sign every binary which is going into +# the final bundle. +# + +execute_process( + COMMAND python "${CMAKE_CURRENT_LIST_DIR}/cpack_post.py" "${CMAKE_INSTALL_PREFIX}" + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + RESULT_VARIABLE exit_code +) + +if(NOT exit_code EQUAL "0") + message(FATAL_ERROR "Non-zero exit code of codesign tool") +endif() diff --git a/config/worker/blender/cpack_post.py b/config/worker/blender/cpack_post.py new file mode 100644 index 0000000..e08dbc6 --- /dev/null +++ b/config/worker/blender/cpack_post.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import pathlib +import sys + +sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent.parent)) + +import worker.blender.sign +import worker.utils + +path = pathlib.Path(sys.argv[1]).resolve() + +worker.blender.sign.sign_windows("PROD", path) + +if str(path).find("Unspecified") != -1: + print("Probably running with cpack command, adding Blender path") + blender_path = path.parent / "Blender" + worker.blender.sign.sign_windows("PROD", blender_path) + +print("Codesign for cpack is finished") + +# Only do this for zip +if str(path).find("ZIP") != -1: + new_path = path.parent / path.name.replace("-windows64", "") + package_file_path = new_path.parent / (new_path.name + ".zip") + + worker.utils.call(["7z", "a", "-tzip", package_file_path, path, "-r"]) + worker.utils.call(["7z", "rn", package_file_path, path.name, new_path.name]) diff --git a/config/worker/blender/lint.py b/config/worker/blender/lint.py new file mode 100644 index 0000000..5c0afcd --- /dev/null +++ b/config/worker/blender/lint.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import os +import sys + +import worker.blender +import worker.utils + + +def make_format(builder: worker.blender.CodeBuilder) -> bool: + os.chdir(builder.blender_dir) + + # Always run formatting with scripts from main, for security on unverified patches. + # TODO: how secure is this? How to test formatting issues in the scripts themselves? + # main_files = [makefile, "tools/utils_maintenance", "build_files/windows"] + # for main_file in main_files: + # worker.utils.call(['git', 'checkout', 'origin/main', '--', main_file]) + + # Run format + if builder.platform == "windows": + builder.call(["make.bat", "format"]) + else: + builder.call(["make", "-f", "GNUmakefile", "format"]) + + # Check for changes + diff = worker.utils.check_output(["git", "diff"]) + if len(diff) > 0: + print(diff) + + # Reset + worker.utils.call(["git", "checkout", "HEAD", "--", "."]) + + if len(diff) > 0: + worker.utils.error('Incorrect formatting detected, run "make format" to fix') + return False + + return True + + +def lint(builder: worker.blender.CodeBuilder) -> None: + ok = make_format(builder) + if not ok: + sys.exit(1) diff --git a/config/worker/blender/msix_package.py b/config/worker/blender/msix_package.py new file mode 100644 index 0000000..7941bb8 --- /dev/null +++ b/config/worker/blender/msix_package.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import pathlib +import zipfile + +import worker.utils + + +def pack( + # Version string in the form of 2.83.3.0, this is used in the Store package name + version: str, + # Input file path + input_file_path: pathlib.Path, + # A string in the form of 'CN=PUBLISHER' + publisher: str, + # If set this MSIX is for an LTS release + lts: bool = False, + # If set remove Content folder if it already exists + overwrite: bool = False, + # Don't actually execute commands + dry_run: bool = False, +) -> pathlib.Path: + LTSORNOT = "" + PACKAGETYPE = "" + if lts: + versionparts = version.split(".") + LTSORNOT = f" {versionparts[0]}.{versionparts[1]} LTS" + PACKAGETYPE = f"{versionparts[0]}.{versionparts[1]}LTS" + + output_package_file_name = f"{input_file_path.stem}.msix" + output_package_file_path = pathlib.Path(".", output_package_file_name) + content_folder = pathlib.Path(".", "Content") + content_blender_folder = pathlib.Path(content_folder, "Blender") + content_assets_folder = pathlib.Path(content_folder, "Assets") + assets_original_folder = pathlib.Path(".", "Assets") + + pri_config_file = pathlib.Path(".", "priconfig.xml") + pri_resources_file = pathlib.Path(content_folder, "resources.pri") + + pri_command = [ + "makepri", + "new", + "/pr", + f"{content_folder.absolute()}", + "/cf", + f"{pri_config_file.absolute()}", + "/of", + f"{pri_resources_file.absolute()}", + ] + + msix_command = [ + "makeappx", + "pack", + "/h", + "sha256", + "/d", + f"{content_folder.absolute()}", + "/p", + f"{output_package_file_path.absolute()}", + ] + + if overwrite: + if content_folder.joinpath("Assets").exists(): + worker.utils.remove_dir(content_folder) + content_folder.mkdir(exist_ok=True) + worker.utils.copy_dir(assets_original_folder, content_assets_folder) + + manifest_text = pathlib.Path("AppxManifest.xml.template").read_text() + manifest_text = manifest_text.replace("[VERSION]", version) + manifest_text = manifest_text.replace("[PUBLISHER]", publisher) + manifest_text = manifest_text.replace("[LTSORNOT]", LTSORNOT) + manifest_text = manifest_text.replace("[PACKAGETYPE]", PACKAGETYPE) + pathlib.Path(content_folder, "AppxManifest.xml").write_text(manifest_text) + + worker.utils.info( + f"Extracting files from [{input_file_path}] to [{content_blender_folder.absolute()}]" + ) + + # Extract the files from the ZIP archive, but skip the leading part of paths + # in the ZIP. We want to write the files to the content_blender_folder where + # blender.exe ends up as ./Content/Blender/blender.exe, and not + # ./Content/Blender/blender-2.83.3-windows64/blender.exe + with zipfile.ZipFile(input_file_path, "r") as blender_zip: + for entry in blender_zip.infolist(): + if entry.is_dir(): + continue + entry_location = pathlib.Path(entry.filename) + target_location = content_blender_folder.joinpath(*entry_location.parts[1:]) + pathlib.Path(target_location.parent).mkdir(parents=True, exist_ok=True) + extracted_entry = blender_zip.read(entry) + target_location.write_bytes(extracted_entry) + + worker.utils.info("... extraction complete.") + + worker.utils.info("Generating Package Resource Index (PRI) file") + worker.utils.call(pri_command, dry_run=dry_run) + + worker.utils.info(f"Creating MSIX package using command: {' '.join(msix_command)}") + + # Remove MSIX file if it already exists. Otherwise the MakeAppX tool + # will hang. + worker.utils.remove_file(output_package_file_path) + worker.utils.call(msix_command, dry_run=dry_run) + + if dry_run: + output_package_file_path.write_text("Dry run dummy package file") + + worker.utils.remove_dir(content_folder) + + worker.utils.info("Done.") + + return output_package_file_path diff --git a/config/worker/blender/pack.py b/config/worker/blender/pack.py new file mode 100644 index 0000000..a39a6d8 --- /dev/null +++ b/config/worker/blender/pack.py @@ -0,0 +1,357 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +# Runs on buildbot worker, creating a release package using the build +# system and zipping it into buildbot_upload.zip. This is then uploaded +# to the master in the next buildbot step. + +import hashlib +import json +import os +import sys +import pathlib +import tarfile + +import worker.blender +import worker.utils + +import worker.blender.sign +import worker.blender.bundle_dmg +import worker.blender.version + + +# SemVer based file naming +def get_package_name(builder: worker.blender.CodeBuilder) -> str: + version_info = worker.blender.version.VersionInfo(builder) + + # For release branch we will trim redundant info + branch_id = ( + builder.branch_id.replace("/", "-") + .replace(".", "") + .replace("blender-", "") + .replace("-release", "") + ) + package_name = "bpy" if builder.python_module else "blender" + package_name += f"-{version_info.version}" + package_name += f"-{version_info.risk_id}" + package_name += f"+{branch_id}" + if builder.patch_id: + if builder.patch_id.startswith("D"): + package_name += f"-{builder.patch_id}" + else: + package_name += f"-PR{builder.patch_id}" + + package_name += f".{version_info.hash}" + package_name += f"-{builder.platform}" + package_name += f".{builder.architecture}" + package_name += f"-{builder.build_configuration}" + + return package_name + + +# Generate .sha256 file next to packge +def generate_file_hash(package_file_path: pathlib.Path) -> None: + hash_algorithm = hashlib.sha256() + + mem_array = bytearray(128 * 1024) + mem_view = memoryview(mem_array) + with open(package_file_path, "rb", buffering=0) as f: + while 1: + # https://github.com/python/typeshed/issues/2166 + n = f.readinto(mem_view) # type: ignore + if not n: + break + hash_algorithm.update(mem_view[:n]) + + hash_file_path = (package_file_path.parent) / (package_file_path.name + ".sha256") + hash_text = hash_algorithm.hexdigest() + hash_file_path.write_text(hash_text) + + worker.utils.info(f"Generated hash [{hash_file_path}]") + print(hash_text) + + +# tar cf archive.tar test.c --owner=0 --group=0 +def create_tar_xz(src: pathlib.Path, dest: pathlib.Path, package_name: str) -> None: + # One extra to remove leading os.sep when cleaning root for package_root + ln = len(str(src)) + 1 + flist = list() + + # Create list of tuples containing file and archive name + for root, dirs, files in os.walk(src): + package_root = os.path.join(package_name, root[ln:]) + flist.extend( + [(os.path.join(root, file), os.path.join(package_root, file)) for file in files] + ) + + # Set UID/GID of archived files to 0, otherwise they'd be owned by whatever + # user compiled the package. If root then unpacks it to /usr/local/ you get + # a security issue. + def _fakeroot(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo: + tarinfo.gid = 0 + tarinfo.gname = "root" + tarinfo.uid = 0 + tarinfo.uname = "root" + return tarinfo + + # Silence false positive mypy error. + package = tarfile.open(dest, "w:xz", preset=6) # type: ignore[call-arg] + for entry in flist: + worker.utils.info(f"Adding [{entry[0]}] to archive [{entry[1]}]") + package.add(entry[0], entry[1], recursive=False, filter=_fakeroot) + package.close() + + +def cleanup_files(dirpath: pathlib.Path, extension: str) -> None: + if dirpath.exists(): + for filename in os.listdir(dirpath): + filepath = pathlib.Path(os.path.join(dirpath, filename)) + if filepath.is_file() and filename.endswith(extension): + worker.utils.remove_file(filepath) + + +def pack_mac(builder: worker.blender.CodeBuilder) -> None: + version_info = worker.blender.version.VersionInfo(builder) + + os.chdir(builder.build_dir) + cleanup_files(builder.package_dir, ".dmg") + + package_name = get_package_name(builder) + package_file_name = package_name + ".dmg" + package_file_path = builder.package_dir / package_file_name + + applescript_file_path = pathlib.Path(__file__).parent.resolve() / "blender.applescript" + background_image_file_path = builder.blender_dir / "release" / "darwin" / "background.tif" + + worker.blender.bundle_dmg.bundle( + builder.install_dir, package_file_path, applescript_file_path, background_image_file_path + ) + + # Sign + worker.blender.sign.sign_darwin_files(builder, [package_file_path], "entitlements.plist") + + # Notarize + worker_config = builder.get_worker_config() + team_id = worker_config.sign_code_darwin_team_id + apple_id = worker_config.sign_code_darwin_apple_id + keychain_profile = worker_config.sign_code_darwin_keychain_profile + timeout = "30m" + + if builder.service_env_id == "LOCAL" and not apple_id: + worker.utils.info("Skipping notarization without Apple ID in local build") + return + + # Upload file and wait for completion. + notarize_cmd = [ + "xcrun", + "notarytool", + "submit", + package_file_path, + "--apple-id", + worker.utils.HiddenArgument(apple_id), + "--keychain-profile", + worker.utils.HiddenArgument(keychain_profile), + "--team-id", + worker.utils.HiddenArgument(team_id), + "--timeout", + timeout, + "--wait", + "--output-format", + "json", + ] + + request = worker.utils.check_output(notarize_cmd) + + request_data = json.loads(request) + request_id = request_data["id"] + request_status = request_data["status"] + + # Show logs + worker.utils.call( + ["xcrun", "notarytool", "log", "--keychain-profile", keychain_profile, request_id], + retry_count=5, + retry_wait_time=10.0, + ) + + # Failed? + if request_status != "Accepted": + raise Exception("Notarization failed, aborting") + + # Staple it + worker.utils.call(["xcrun", "stapler", "staple", package_file_path]) + + generate_file_hash(package_file_path) + + +def pack_win(builder: worker.blender.CodeBuilder, pack_format: str) -> None: + os.chdir(builder.build_dir) + + if pack_format == "msi": + cpack_type = "WIX" + else: + cpack_type = "ZIP" + + package_extension = pack_format + cleanup_files(builder.package_dir, f".{package_extension}") + + script_folder_path = pathlib.Path(os.path.realpath(__file__)).parent + + # Will take care of codesigning and correct the folder name in zip + # + # Code signing is done as part of INSTALL target, which makes it possible to sign + # files which are aimed into a bundle and coming from a non-signed source (such as + # libraries SVN). + # + # This is achieved by specifying cpack_post.cmake as a post-install script run + # by cpack. cpack_post.ps1 takes care of actual code signing. + post_script_file_path = script_folder_path / "cpack_post.cmake" + + app_id = "Blender" + final_package_name = get_package_name(builder) + # MSI needs the app id for the Windows menu folder name + # It will also fail if anything else. + cpack_package_name = app_id if pack_format == "msi" else final_package_name + + cmake_cmd = [ + "cmake", + f"-DCPACK_PACKAGE_NAME:STRING={cpack_package_name}", + f"-DCPACK_OVERRIDE_PACKAGENAME:STRING={cpack_package_name}", + # Only works with ZIP, ignored by MSI + # f'-DARCHIVE_FILE:STRING={package_name}', + # f'-DCPACK_PACKAGE_FILE_NAME:STRING={cpack_package_name}', + f"-DCMAKE_INSTALL_PREFIX:PATH={builder.install_dir}", + f"-DPOSTINSTALL_SCRIPT:PATH={post_script_file_path}", + ".", + ] + builder.call(cmake_cmd) + + worker.utils.info("Packaging Blender files") + cpack_cmd = [ + "cpack", + "-G", + cpack_type, + # '--verbose', + "--trace-expand", + "-C", + builder.build_configuration, + "-B", + str(builder.package_dir), # CPACK_PACKAGE_DIRECTORY + "-P", + cpack_package_name, + ] + builder.call(cpack_cmd) + + final_package_file_name = f"{final_package_name}.{package_extension}" + final_package_file_path = builder.package_dir / final_package_file_name + + # HACK: Rename files correctly, packages are appended `-windows64` with no option to rename + bogus_cpack_file_path = ( + builder.package_dir / f"{cpack_package_name}-windows64.{package_extension}" + ) + + if pack_format == "zip": + if bogus_cpack_file_path.exists(): + worker.utils.info(f"Removing bogus file [{bogus_cpack_file_path}]") + worker.utils.remove_file(bogus_cpack_file_path) + + source_cpack_file_path = ( + builder.package_dir + / "_CPack_Packages" + / "Windows" + / "ZIP" + / f"{final_package_file_name}" + ) + worker.utils.info(f"Moving [{source_cpack_file_path}] to [{final_package_file_path}]") + os.rename(source_cpack_file_path, final_package_file_path) + else: + os.rename(bogus_cpack_file_path, final_package_file_path) + version_info = worker.blender.version.VersionInfo(builder) + description = f"Blender {version_info.version}" + worker.blender.sign.sign_windows_files(builder.service_env_id, [final_package_file_path], + description=description) + + generate_file_hash(final_package_file_path) + + +def pack_linux(builder: worker.blender.CodeBuilder) -> None: + blender_executable = builder.install_dir / "blender" + + version_info = worker.blender.version.VersionInfo(builder) + + # Strip all unused symbols from the binaries + worker.utils.info("Stripping binaries") + builder.call(["strip", "--strip-all", blender_executable]) + + worker.utils.info("Stripping python") + + # This should work for 3.0, but for now it is in 3.00 + py_target = builder.install_dir / version_info.short_version + if not os.path.exists(py_target): + # Support older format and current issue with 3.00 + py_target = builder.install_dir / ("%d.%02d" % (version_info.major, version_info.minor)) + + worker.utils.call(["find", py_target, "-iname", "*.so", "-exec", "strip", "-s", "{}", ";"]) + + package_name = get_package_name(builder) + package_file_name = f"{package_name}.tar.xz" + package_file_path = builder.package_dir / package_file_name + + worker.utils.info(f"Creating [{package_file_path}] archive") + + os.makedirs(builder.package_dir, exist_ok=True) + + create_tar_xz(builder.install_dir, package_file_path, package_name) + + generate_file_hash(package_file_path) + + +def pack_python_module(builder: worker.blender.CodeBuilder) -> None: + cleanup_files(builder.package_dir, ".whl") + cleanup_files(builder.package_dir, ".zip") + + package_name = get_package_name(builder) + ".zip" + package_filepath = builder.package_dir / package_name + pack_script = builder.blender_dir / "build_files" / "utils" / "make_bpy_wheel.py" + + # Make wheel + worker.utils.info("Packaging Python Wheel") + cmd = [sys.executable, pack_script, builder.install_dir] + cmd += ["--build-dir", builder.build_dir] + cmd += ["--output-dir", builder.package_dir] + builder.call(cmd) + + # Pack wheel in zip, until pipeline and www can deal with .whl files. + import zipfile + + with zipfile.ZipFile(package_filepath, "w") as zipf: + for whl_name in os.listdir(builder.package_dir): + if whl_name.endswith(".whl"): + whl_filepath = builder.package_dir / whl_name + zipf.write(whl_filepath, arcname=whl_name) + + cleanup_files(builder.package_dir, ".whl") + + generate_file_hash(package_filepath) + + +def pack(builder: worker.blender.CodeBuilder) -> None: + builder.setup_build_environment() + + # Create clean package directory + worker.utils.remove_dir(builder.package_dir) + os.makedirs(builder.package_dir, exist_ok=True) + + # Make sure install directory always exists + os.makedirs(builder.install_dir, exist_ok=True) + + if builder.python_module: + pack_python_module(builder) + elif builder.platform == "darwin": + pack_mac(builder) + elif builder.platform == "windows": + pack_win(builder, "zip") + if builder.track_id not in ["vdev", "vexp"]: + pack_win(builder, "msi") + elif builder.platform == "linux": + pack_linux(builder) diff --git a/config/worker/blender/sign.py b/config/worker/blender/sign.py new file mode 100644 index 0000000..9746c00 --- /dev/null +++ b/config/worker/blender/sign.py @@ -0,0 +1,195 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import pathlib +import sys + +from typing import Optional, Sequence + +import worker.blender +import worker.utils + + +def sign_windows_files( + service_env_id: str, + file_paths: Sequence[pathlib.Path], + description: Optional[str] = None, + certificate_id: str = "", +) -> None: + import conf.worker + + worker_config = conf.worker.get_config(service_env_id) + + # TODO: Rotate them if first 1 fails + timeserver = worker_config.sign_code_windows_time_servers[0] + server_url = worker_config.sign_code_windows_server_url + if not certificate_id: + certificate_id = worker_config.sign_code_windows_certificate + + dry_run = False + if service_env_id == "LOCAL" and not certificate_id: + worker.utils.warning("Performing dry run on LOCAL service environment") + dry_run = True + + cmd_args = [ + sys.executable, + "C:\\tools\\codesign.py", + "--server-url", + worker.utils.HiddenArgument(server_url), + ] + if description: + cmd_args += ["--description", description] + + cmd: worker.utils.CmdSequence = cmd_args + + # Signing one file at a time causes a stampede on servers, resulting in blocking. + # Instead sign in chunks of multiple files. + chunk_size = 25 # Sign how many files at a time + retry_count = 3 + + for i in range(0, len(file_paths), chunk_size): + file_chunks = file_paths[i : i + chunk_size] + worker.utils.call(list(cmd) + list(file_chunks), retry_count=retry_count, dry_run=dry_run) + + +def sign_windows(service_env_id: str, install_path: pathlib.Path) -> None: + # TODO: Why use a junction? Is there some failure with long file paths? + # worker.utils.info("Creating building link") + # temp_build_root_path = pathlib.Path("C:/BlenderTemp") + # os.makedirs(temp_build_root_path, exist_ok=True) + # orig_install_path = install_path + # install_path = temp_build_root_path / install_path.name + + try: + # TODO + # New-Item -type Junction -path install_path -value orig_install_path + + worker.utils.info("Collecting files to process") + file_paths = list(install_path.glob("*.exe")) + file_paths += list(install_path.glob("*.dll")) + file_paths += list(install_path.glob("*.pyd")) + file_paths = [f for f in file_paths if str(f).find("blender.crt") == -1] + for f in file_paths: + print(f) + + sign_windows_files(service_env_id, file_paths) + finally: + # worker.utils.info(f"Removing temporary folder {temp_build_root_path}") + # worker.utils.remove_dir(temp_build_root_path, retry_count=5, retry_wait_time=5.0) + + # TODO: is this really necessary? + # worker.utils.info("Flushing volume cache...") + # Write-VolumeCache -DriveLetter C + + # core_shell_retry_command -retry_count 5 -delay_in_milliseconds 1000 -script_block ` + # worker.utils.info("Junction information...") + # junction = Get-Item -Path install_path + # worker.utils.info(junction | Format-Table) + # worker.utils.info("Attempting to remove...") + # junction.Delete() + # worker.utils.info("Junction deleted!") + pass + + worker.utils.info("End of codesign steps") + + +def sign_darwin_files( + builder: worker.blender.CodeBuilder, + file_paths: Sequence[pathlib.Path], + entitlements_file_name: str +) -> None: + entitlements_path = builder.code_path / "release" / "darwin" / entitlements_file_name + + if not entitlements_path.exists(): + raise Exception(f"File {entitlements_path} not found, aborting") + + worker_config = builder.get_worker_config() + certificate_id = worker_config.sign_code_darwin_certificate + + dry_run = False + if builder.service_env_id == "LOCAL" and not certificate_id: + worker.utils.warning("Performing dry run on LOCAL service environment") + dry_run = True + + keychain_password = worker_config.darwin_keychain_password(builder.service_env_id) + cmd: worker.utils.CmdSequence = [ + "security", + "unlock-keychain", + "-p", + worker.utils.HiddenArgument(keychain_password), + ] + worker.utils.call(cmd, dry_run=dry_run) + + for file_path in file_paths: + if file_path.is_dir() and file_path.suffix != ".app": + continue + + # Remove signature + if file_path.suffix != ".dmg": + worker.utils.call( + ["codesign", "--remove-signature", file_path], exit_on_error=False, dry_run=dry_run + ) + + # Add signature + worker.utils.call( + [ + "codesign", + "--force", + "--timestamp", + "--options", + "runtime", + f"--entitlements={entitlements_path}", + "--sign", + certificate_id, + file_path, + ], + retry_count=3, + dry_run=dry_run, + ) + if file_path.suffix == ".app": + worker.utils.info(f"Vaildating app bundle {file_path}") + worker.utils.call( + ["codesign", "-vvv", "--deep", "--strict", file_path], dry_run=dry_run + ) + + +def sign_darwin(builder: worker.blender.CodeBuilder) -> None: + bundle_path = builder.install_dir / "Blender.app" + + # Executables + sign_path = bundle_path / "Contents" / "MacOS" + worker.utils.info(f"Collecting files to process in {sign_path}") + sign_darwin_files(builder, list(sign_path.rglob("*")), "entitlements.plist") + + # Thumbnailer app extension. + thumbnailer_appex_path = bundle_path / "Contents" / "PlugIns" / "blender-thumbnailer.appex" + if thumbnailer_appex_path.exists(): + sign_path = thumbnailer_appex_path / "Contents" / "MacOS" + worker.utils.info(f"Collecting files to process in {sign_path}") + sign_darwin_files(builder, list(sign_path.rglob("*")), "thumbnailer_entitlements.plist") + + # Shared librarys and Python + sign_path = bundle_path / "Contents" / "Resources" + worker.utils.info(f"Collecting files to process in {sign_path}") + file_paths = list( + set(sign_path.rglob("*.dylib")) + | set(sign_path.rglob("*.so")) + | set(sign_path.rglob("python3.*")) + ) + sign_darwin_files(builder, file_paths, "entitlements.plist") + + # Bundle + worker.utils.info(f"Signing app bundle {bundle_path}") + sign_darwin_files(builder, [bundle_path], "entitlements.plist") + + +def sign(builder: worker.blender.CodeBuilder) -> None: + builder.setup_build_environment() + + if builder.platform == "windows": + sign_windows(builder.service_env_id, builder.install_dir) + elif builder.platform == "darwin": + sign_darwin(builder) + else: + worker.utils.info("No code signing to be done on this platform") diff --git a/config/worker/blender/test.py b/config/worker/blender/test.py new file mode 100644 index 0000000..569f909 --- /dev/null +++ b/config/worker/blender/test.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import os +import shutil + +from typing import List + +import worker.blender +import worker.blender.pack +import worker.blender.compile + + +def get_ctest_arguments(builder: worker.blender.CodeBuilder) -> List[str]: + args = ["--output-on-failure"] + + # GPU tests are currently slow and can cause timeouts. + if not builder.needs_gpu_tests: + args += ["--parallel", "4"] + + args += ["-C", worker.blender.compile.get_cmake_build_type(builder)] + return args + + +def package_for_upload(builder: worker.blender.CodeBuilder, success: bool) -> None: + build_tests_dir = builder.build_dir / "tests" + package_tests_dir = builder.package_dir / "tests" + if not build_tests_dir.exists(): + return + + os.makedirs(package_tests_dir, exist_ok=True) + + # Upload package on failure + if not success: + package_filename = "tests-" + worker.blender.pack.get_package_name(builder) + package_filepath = package_tests_dir / package_filename + shutil.copytree(build_tests_dir, package_filepath) + shutil.make_archive(str(package_filepath), "zip", package_tests_dir, package_filename) + shutil.rmtree(package_filepath) + + # Always upload unpacked folder for main and release tracks, + # when using GPU tests. This is useful for debugging GPU + # differences. + if builder.track_id != "vexp" and builder.needs_gpu_tests: + branch = builder.branch_id.replace("blender-", "").replace("-release", "") + name = f"{branch}-{builder.platform}-{builder.architecture}" + shutil.copytree(build_tests_dir, package_tests_dir / name) + + +def test(builder: worker.blender.CodeBuilder) -> None: + builder.setup_build_environment() + os.chdir(builder.build_dir) + success = False + + try: + builder.call(["ctest"] + get_ctest_arguments(builder)) + success = True + finally: + package_for_upload(builder, success) diff --git a/config/worker/blender/update.py b/config/worker/blender/update.py new file mode 100644 index 0000000..cb5909d --- /dev/null +++ b/config/worker/blender/update.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import os +import sys + +import worker.blender +import worker.utils + + +def _clean_folders(builder: worker.blender.CodeBuilder) -> None: + # Delete build folders. + if builder.needs_full_clean: + worker.utils.remove_dir(builder.build_dir) + else: + worker.utils.remove_dir(builder.build_dir / "Testing") + worker.utils.remove_dir(builder.build_dir / "bin" / "tests") + + # Delete install and packaging folders + worker.utils.remove_dir(builder.install_dir) + worker.utils.remove_dir(builder.package_dir) + + +def update(builder: worker.blender.CodeBuilder) -> None: + _clean_folders(builder) + + builder.update_source() + os.chdir(builder.code_path) + + make_update_path = builder.code_path / "build_files" / "utils" / "make_update.py" + + make_update_text = make_update_path.read_text() + if "def svn_update" in make_update_text: + worker.utils.error("Can't build branch or pull request that uses Subversion libraries.") + worker.utils.error("Merge with latest main or release branch to use Git LFS libraries.") + sys.exit(1) + + # Run make update + cmd = [ + sys.executable, + make_update_path, + "--no-blender", + "--use-linux-libraries", + "--use-tests", + "--architecture", + builder.architecture, + ] + + if builder.track_id not in ("v360", "vexp"): + cmd += ["--prune-destructive"] + + worker.utils.call(cmd) diff --git a/config/worker/blender/version.py b/config/worker/blender/version.py new file mode 100644 index 0000000..24e7fca --- /dev/null +++ b/config/worker/blender/version.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import pathlib +import re + + +import worker.blender + + +class VersionInfo: + def __init__(self, builder: worker.blender.CodeBuilder): + # Get version information + buildinfo_h = builder.build_dir / "source" / "creator" / "buildinfo.h" + blender_h = ( + builder.blender_dir / "source" / "blender" / "blenkernel" / "BKE_blender_version.h" + ) + + version_number = int(self._parse_header_file(blender_h, "BLENDER_VERSION")) + + version_number_patch = int(self._parse_header_file(blender_h, "BLENDER_VERSION_PATCH")) + self.major, self.minor, self.patch = ( + version_number // 100, + version_number % 100, + version_number_patch, + ) + + if self.major >= 3: + self.short_version = "%d.%d" % (self.major, self.minor) + self.version = "%d.%d.%d" % (self.major, self.minor, self.patch) + else: + self.short_version = "%d.%02d" % (self.major, self.minor) + self.version = "%d.%02d.%d" % (self.major, self.minor, self.patch) + + self.version_cycle = self._parse_header_file(blender_h, "BLENDER_VERSION_CYCLE") + if buildinfo_h.exists(): + self.hash = self._parse_header_file(buildinfo_h, "BUILD_HASH")[1:-1] + else: + self.hash = "" + self.risk_id = self.version_cycle.replace("release", "stable").replace("rc", "candidate") + self.is_development_build = self.version_cycle == "alpha" + + def _parse_header_file(self, filename: pathlib.Path, define: str) -> str: + regex = re.compile(r"^#\s*define\s+%s\s+(.*)" % define) + with open(filename, "r") as file: + for l in file: + match = regex.match(l) + if match: + return match.group(1) + + raise BaseException(f"Failed to parse {filename.name} header for {define}") diff --git a/config/worker/code.py b/config/worker/code.py new file mode 100755 index 0000000..36833ea --- /dev/null +++ b/config/worker/code.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import pathlib +import sys + +from collections import OrderedDict + +sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) + +import worker.configure +import worker.utils + +import worker.blender.update +import worker.blender.lint +import worker.blender.compile +import worker.blender.test +import worker.blender.sign +import worker.blender.pack + + +if __name__ == "__main__": + steps: worker.utils.BuilderSteps = OrderedDict() + steps["configure-machine"] = worker.configure.configure_machine + steps["update-code"] = worker.blender.update.update + steps["lint-code"] = worker.blender.lint.lint + steps["compile-code"] = worker.blender.compile.compile_code + steps["compile-gpu"] = worker.blender.compile.compile_gpu + steps["compile-install"] = worker.blender.compile.compile_install + steps["test-code"] = worker.blender.test.test + steps["sign-code-binaries"] = worker.blender.sign.sign + steps["package-code-binaries"] = worker.blender.pack.pack + steps["clean"] = worker.blender.CodeBuilder.clean + + parser = worker.blender.create_argument_parser(steps=steps) + + args = parser.parse_args() + builder = worker.blender.CodeBuilder(args) + builder.setup_track_path() + builder.run(args.step, steps) diff --git a/config/worker/code_benchmark.py b/config/worker/code_benchmark.py new file mode 100755 index 0000000..f7837c5 --- /dev/null +++ b/config/worker/code_benchmark.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import argparse +import pathlib +import sys + +from collections import OrderedDict + +sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) + +import worker.configure +import worker.utils + +import worker.blender +import worker.blender.benchmark +import worker.blender.compile +import worker.blender.update + + +class BenchmarkBuilder(worker.blender.CodeBuilder): + def __init__(self, args: argparse.Namespace): + super().__init__(args) + self.setup_track_path() + + +if __name__ == "__main__": + steps: worker.utils.BuilderSteps = OrderedDict() + steps["configure-machine"] = worker.configure.configure_machine + steps["update-code"] = worker.blender.update.update + steps["compile-code"] = worker.blender.compile.compile_code + steps["compile-gpu"] = worker.blender.compile.compile_gpu + steps["compile-install"] = worker.blender.compile.compile_install + steps["benchmark"] = worker.blender.benchmark.benchmark + steps["clean"] = worker.blender.CodeBuilder.clean + + parser = worker.blender.create_argument_parser(steps=steps) + + args = parser.parse_args() + builder = BenchmarkBuilder(args) + builder.run(args.step, steps) diff --git a/config/worker/code_bpy_deploy.py b/config/worker/code_bpy_deploy.py new file mode 100755 index 0000000..e406ab9 --- /dev/null +++ b/config/worker/code_bpy_deploy.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import pathlib +import sys + +from collections import OrderedDict + +sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) + +import worker.configure +import worker.utils + +import worker.blender +import worker.blender.update + +import worker.deploy +import worker.deploy.pypi + + +if __name__ == "__main__": + steps: worker.utils.BuilderSteps = OrderedDict() + steps["configure-machine"] = worker.configure.configure_machine + steps["update-code"] = worker.blender.update.update + steps["pull"] = worker.deploy.pypi.pull + steps["deliver-pypi"] = worker.deploy.pypi.deliver + steps["clean"] = worker.deploy.CodeDeployBuilder.clean + + parser = worker.blender.create_argument_parser(steps=steps) + + args = parser.parse_args() + builder = worker.deploy.CodeDeployBuilder(args) + builder.run(args.step, steps) diff --git a/config/worker/code_deploy.py b/config/worker/code_deploy.py new file mode 100755 index 0000000..4faab34 --- /dev/null +++ b/config/worker/code_deploy.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import pathlib +import sys + +from collections import OrderedDict + +sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) + +import worker.configure +import worker.utils + +import worker.blender +import worker.blender.update + +import worker.deploy +import worker.deploy.source +import worker.deploy.artifacts +import worker.deploy.monitor + + +if __name__ == "__main__": + steps: worker.utils.BuilderSteps = OrderedDict() + steps["configure-machine"] = worker.configure.configure_machine + steps["update-code"] = worker.blender.update.update + steps["pull-artifacts"] = worker.deploy.artifacts.pull + steps["repackage-artifacts"] = worker.deploy.artifacts.repackage + steps["package-source"] = worker.deploy.source.package + steps["deploy-artifacts"] = worker.deploy.artifacts.deploy + steps["monitor-artifacts"] = worker.deploy.monitor.monitor + steps["clean"] = worker.deploy.CodeDeployBuilder.clean + + parser = worker.blender.create_argument_parser(steps=steps) + + args = parser.parse_args() + builder = worker.deploy.CodeDeployBuilder(args) + builder.run(args.step, steps) diff --git a/config/worker/code_store.py b/config/worker/code_store.py new file mode 100755 index 0000000..0ad5736 --- /dev/null +++ b/config/worker/code_store.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import pathlib +import sys + +from collections import OrderedDict + +sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) + +import worker.configure +import worker.utils + +import worker.blender +import worker.blender.update + +import worker.deploy +import worker.deploy.artifacts +import worker.deploy.snap +import worker.deploy.steam +import worker.deploy.windows + + +def package(builder: worker.deploy.CodeStoreBuilder) -> None: + if builder.store_id == "snap": + worker.deploy.snap.package(builder) + elif builder.store_id == "steam": + worker.deploy.steam.package(builder) + elif builder.store_id == "windows": + builder.setup_build_environment() + worker.deploy.windows.package(builder) + + +def deliver(builder: worker.deploy.CodeStoreBuilder) -> None: + if builder.store_id == "snap": + worker.deploy.snap.deliver(builder) + elif builder.store_id == "steam": + worker.deploy.steam.deliver(builder) + elif builder.store_id == "windows": + worker.deploy.windows.deliver(builder) + + +if __name__ == "__main__": + steps: worker.utils.BuilderSteps = OrderedDict() + steps["configure-machine"] = worker.configure.configure_machine + steps["update-code"] = worker.blender.update.update + steps["pull-artifacts"] = worker.deploy.artifacts.pull + steps["package"] = package + steps["deliver"] = deliver + steps["clean"] = worker.deploy.CodeDeployBuilder.clean + + parser = worker.blender.create_argument_parser(steps=steps) + parser.add_argument("--store-id", type=str, choices=["snap", "steam", "windows"], required=True) + + args = parser.parse_args() + builder = worker.deploy.CodeStoreBuilder(args) + builder.run(args.step, steps) diff --git a/config/worker/configure.py b/config/worker/configure.py new file mode 100644 index 0000000..4476e6c --- /dev/null +++ b/config/worker/configure.py @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import os +import pathlib +import platform +import psutil +import shutil + +from typing import List, Tuple + +import worker.utils + + +def get_os_release() -> str: + if platform.system() == "Darwin": + return "macOS " + platform.mac_ver()[0] + else: + return platform.version() + + +def get_cpu_info() -> str: + if platform.system() == "Darwin": + return worker.utils.check_output(["/usr/sbin/sysctl", "-n", "machdep.cpu.brand_string"]) + elif platform.system() == "Linux": + cpuinfo = pathlib.Path("/proc/cpuinfo").read_text() + for line in cpuinfo.splitlines(): + if line.find("model name") != -1: + return line.split(":")[1].strip() + + return platform.processor() + + +def disk_free_in_gb(builder: worker.utils.Builder) -> float: + _, _, disk_free = shutil.disk_usage(builder.track_path) + return disk_free / (1024.0**3) + + +def get_thread_count(thread_memory_in_GB: float) -> int: + num_threads = psutil.cpu_count() + memory_in_GB = psutil.virtual_memory().total / (1024**3) + + return min(int(memory_in_GB / thread_memory_in_GB), num_threads) + + +def clean(builder: worker.utils.Builder) -> None: + # Remove build folders to make space. + delete_paths: List[pathlib.Path] = [] + optional_delete_paths: List[pathlib.Path] = [] + + branches_config = builder.get_branches_config() + tracks = branches_config.track_major_minor_versions.keys() + + # TODO: don't hardcode these folder and track names + for track in tracks: + track_path = builder.tracks_root_path / ("blender-manual-" + track) + optional_delete_paths += [track_path / "build"] + + for track in tracks: + track_path = builder.tracks_root_path / ("blender-" + track) + delete_paths += [track_path / "build_download"] + delete_paths += [track_path / "build_linux"] + delete_paths += [track_path / "build_darwin"] + delete_paths += [track_path / "build_package"] + delete_paths += [track_path / "build_source"] + delete_paths += [track_path / "build_debug"] + delete_paths += [track_path / "build_arm64_debug"] + delete_paths += [track_path / "build_x86_64_debug"] + delete_paths += [track_path / "build_sanitizer"] + delete_paths += [track_path / "build_arm64_sanitizer"] + delete_paths += [track_path / "build_x86_64_sanitizer"] + delete_paths += [track_path / "install_release"] + delete_paths += [track_path / "install_asserts"] + delete_paths += [track_path / "install_sanitizer"] + delete_paths += [track_path / "install_debug"] + delete_paths += [track_path / "benchmark"] + optional_delete_paths += [track_path / "build_release"] + optional_delete_paths += [track_path / "build_arm64_release"] + optional_delete_paths += [track_path / "build_x86_64_release"] + optional_delete_paths += [track_path / "build_asserts"] + optional_delete_paths += [track_path / "build_arm64_asserts"] + optional_delete_paths += [track_path / "build_x86_64_asserts"] + + for delete_path in delete_paths: + worker.utils.remove_dir(delete_path) + + # Cached build folders only if we are low on disk space + if builder.platform == "darwin": + # On macOS APFS this is not reliable, it makes space on demand. + # This should be ok still. + required_space_gb = 12.0 + else: + required_space_gb = 25.0 + + free_space_gb = disk_free_in_gb(builder) + if free_space_gb < required_space_gb: + worker.utils.warning( + f"Trying to delete cached builds for disk space (free {free_space_gb:.2f} GB)" + ) + sorted_paths: List[Tuple[float, pathlib.Path]] = [] + for delete_path in optional_delete_paths: + try: + sorted_paths += [(os.path.getmtime(delete_path), delete_path)] + except: + pass + + for _, delete_path in sorted(sorted_paths): + worker.utils.remove_dir(delete_path) + if disk_free_in_gb(builder) >= required_space_gb: + break + + # Might be left over from git command hanging + stack_dump_file_path = builder.code_path / "sh.exe.stackdump" + worker.utils.remove_file(stack_dump_file_path) + + +def configure_machine(builder: worker.utils.Builder) -> None: + worker_config = builder.get_worker_config() + + clean(builder) + + # Print system information. + processor = get_cpu_info() + + worker.utils.info("System information") + print(f"System: {platform.system()}") + print(f"Release: {get_os_release()}") + print(f"Version: {platform.version()}") + print(f"Processor: {processor}") + print(f"Cores: {psutil.cpu_count()} logical, {psutil.cpu_count(logical=False)} physical") + print(f"Total Memory: {psutil.virtual_memory().total / (1024**3):.2f} GB") + print(f"Available Memory: {psutil.virtual_memory().available / (1024**3):.2f} GB") + + disk_total, disk_used, disk_free = shutil.disk_usage(builder.track_path) + print( + f"Disk: total {disk_total / (1024**3):.2f} GB, " + f"used {disk_used / (1024**3):.2f} GB, " + f"free {disk_free / (1024**3):.2f} GB" + ) + + # Check dependencies and provision + worker.utils.info("Checking installable software cache") + avilable_software_artifacts = worker_config.software_cache_path.glob("*/*") + for artifact in avilable_software_artifacts: + print(artifact) + + # Check packages + if builder.platform == "linux": + etc_rocky = pathlib.Path("/etc/rocky-release") + + if etc_rocky.exists(): + worker.utils.call(["yum", "updateinfo"]) + worker.utils.call(["yum", "list", "updates"]) + else: + worker.utils.call(["apt", "list", "--upgradable"]) + + elif builder.platform == "windows": + choco_version_str = worker.utils.check_output(["choco", "--version"]) + choco_version = [int(x) for x in choco_version_str.split(".")] + if choco_version[0] >= 2: + # In the newer Chocolatey versions `choco list` behavior got changed + # to only list installed package, and the --localonly flag has been + # removed. + worker.utils.call(["choco", "list"]) + else: + worker.utils.call(["choco", "list", "--lo"]) + worker.utils.call(["choco", "outdated"]) + + # Not an actual command, disabled for now. + # worker.utils.call(["scoop", "list"]) + # worker.utils.call(["scoop", "status"]) + + elif builder.platform == "darwin": + worker.utils.call(["brew", "update"]) + worker.utils.call(["brew", "outdated", "--cask"]) + worker.utils.call(["xcrun", "--show-sdk-path"]) + + # XXX Windows builder debug code + if builder.platform == "windows": + # Ensure the idiff.exe process is stopped. + # It might be hanging there since the previously failed build and it will + # prevent removal of the install directory for the new build (due to held + # open DLLs). + worker.utils.info("Stopping idiff.exe if running") + + dump_folder = pathlib.Path("C:\\tmp\\dump\\") + os.makedirs(dump_folder, exist_ok=True) + + worker.utils.call(["procdump", "idiff.exe", dump_folder], exit_on_error=False) + + for proc in psutil.process_iter(): + if proc.name() == "idiff.exe": + proc.kill() + + for proc in psutil.process_iter(): + if proc.name().lower() in ["blender", "blender.exe", "blender_test", "blender_test.exe"]: + worker.utils.warning("Killing stray Blender process") + proc.kill() diff --git a/config/worker/deploy/__init__.py b/config/worker/deploy/__init__.py new file mode 100644 index 0000000..f96757f --- /dev/null +++ b/config/worker/deploy/__init__.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import argparse +import pathlib + +import worker.blender +import worker.utils + + +class CodeDeployBuilder(worker.blender.CodeBuilder): + def __init__(self, args: argparse.Namespace): + super().__init__(args) + self.platform_ids = ["linux", "darwin", "windows"] + self.setup_track_path() + + track_path: pathlib.Path = self.track_path + + self.download_dir = track_path / "build_download" + self.package_source_dir = track_path / "build_source" + self.store_steam_dir = track_path / "build_store_steam" + self.store_snap_dir = track_path / "build_store_snap" + self.store_windows_dir = track_path / "build_store_windows" + + def clean(self): + worker.utils.remove_dir(self.download_dir) + worker.utils.remove_dir(self.package_dir) + worker.utils.remove_dir(self.package_source_dir) + worker.utils.remove_dir(self.store_steam_dir) + worker.utils.remove_dir(self.store_snap_dir) + worker.utils.remove_dir(self.store_windows_dir) + # Created by make source_archive_complete + worker.utils.remove_dir(self.track_path / "build_linux") + worker.utils.remove_dir(self.track_path / "build_darwin") + + +class CodeStoreBuilder(CodeDeployBuilder): + def __init__(self, args: argparse.Namespace): + super().__init__(args) + self.store_id = args.store_id diff --git a/config/worker/deploy/artifacts.py b/config/worker/deploy/artifacts.py new file mode 100644 index 0000000..240e08b --- /dev/null +++ b/config/worker/deploy/artifacts.py @@ -0,0 +1,251 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import json +import os +import pathlib +import urllib.request + +from typing import Any, Dict + +import worker.blender +import worker.blender.version +import worker.deploy +import worker.utils + + +checksums = ["md5", "sha256"] + + +def pull(builder: worker.deploy.CodeDeployBuilder) -> None: + retry_count = 0 + retry_delay_in_seconds = 30 + timeout_in_seconds = 60 + + pipeline_category = "daily" + if builder.track_id == "vexp": + pipeline_category = "experimental" + + log_path = builder.track_path / "log" + worker.utils.remove_dir(log_path) + os.makedirs(log_path, exist_ok=True) + + worker.utils.info("Cleaning package directory") + worker.utils.remove_dir(builder.package_dir) + os.makedirs(builder.package_dir, exist_ok=True) + + # Fetch builds information. + env_base_url = { + "LOCAL": "https://builder.blender.org", + "UATEST": "https://builder.uatest.blender.org", + "PROD": "https://builder.blender.org", + } + base_url = env_base_url[builder.service_env_id] + + search_url = f"{base_url}/download/{pipeline_category}?format=json&v=1" + + worker.utils.info(f"Fetching build JSON from [{search_url}]") + + builds_response = urllib.request.urlopen(search_url) + # TODO -timeout_sec timeout_in_seconds -retry_interval_sec retry_delay_in_seconds -maximum_retry_count retry_count + builds_json = json.load(builds_response) + + # Get builds matching our version. + worker.utils.info("Processing build JSON") + version_info = worker.blender.version.VersionInfo(builder) + + unique_builds: Dict[Any, Dict[Any, Any]] = {} + for build in builds_json: + if build["version"] != version_info.version: + continue + if build["file_extension"] in checksums: + continue + + # Correct incomplete file extension in JSON. + if build["file_name"].endswith(".tar.xz"): + build["file_extension"] = "tar.xz" + elif build["file_name"].endswith(".tar.gz"): + build["file_extension"] = "tar.gz" + elif build["file_name"].endswith(".tar.bz2"): + build["file_extension"] = "tar.bz2" + + key = (build["platform"], build["architecture"], build["file_extension"]) + if key in unique_builds: + # Prefer more stable builds, to avoid issue when multiple are present. + risk_id_order = ["stable", "candidate", "rc", "beta", "alpha", "edge"] + risk = build["risk_id"] + risk = risk_id_order.index(risk) if risk in risk_id_order else len(risk_id_order) + other_risk = unique_builds[key]["risk_id"] + other_risk = ( + risk_id_order.index(other_risk) + if other_risk in risk_id_order + else len(risk_id_order) + ) + if other_risk <= risk: + continue + else: + print(" ".join(key)) + + unique_builds[key] = build + + builds = list(unique_builds.values()) + + if len(builds) == 0: + raise Exception(f"No builds found for version [{version_info.version}] in [{search_url}]") + + # Download builds. + worker.utils.remove_dir(builder.download_dir) + os.makedirs(builder.download_dir, exist_ok=True) + + for build in builds: + file_uri = build["url"] + file_name = build["file_name"] + + worker.utils.info(f"Pull [{file_name}]") + + download_file_path = builder.download_dir / file_name + + worker.utils.info(f"Download [{file_uri}]") + urllib.request.urlretrieve(file_uri, download_file_path) + # TODO: retry and resume + # -resume -timeout_sec timeout_in_seconds -retry_interval_sec retry_delay_in_seconds -maximum_retry_count retry_count + + # Moving to build_package folder + worker.utils.info(f"Move to [{builder.package_dir}]") + worker.utils.move(download_file_path, builder.package_dir / download_file_path.name) + + worker.utils.remove_dir(builder.download_dir) + + # Write manifest of downloaded packages. + package_manifest = builder.package_dir / "manifest.json" + package_manifest.write_text(json.dumps(builds, indent=2)) + + +def repackage(builder: worker.deploy.CodeDeployBuilder) -> None: + version_info = worker.blender.version.VersionInfo(builder) + + deployable_path = builder.package_dir / "deployable" + worker.utils.remove_dir(deployable_path) + os.makedirs(deployable_path, exist_ok=True) + os.chdir(deployable_path) + + package_manifest = builder.package_dir / "manifest.json" + builds = json.loads(package_manifest.read_text()) + + checksum_file_paths = [] + + # Rename the files and the internal folders for zip and tar.xz files + for build in builds: + file_name = build["file_name"] + file_path = builder.package_dir / file_name + + worker.utils.info(f"Repackaging {file_name}") + + if builder.service_env_id == "PROD" and build["risk_id"] != "stable": + raise Exception( + "Can only repackage and deploy stable versions, found risk id '{build['risk_id']}'" + ) + + version = build["version"] + platform = build["platform"].replace("darwin", "macos") + architecture = build["architecture"].replace("86_", "").replace("amd", "x") + file_extension = build["file_extension"] + + current_folder_name = file_path.name[: -len("." + file_extension)] + new_folder_name = f"blender-{version}-{platform}-{architecture}" + new_file_name = f"{new_folder_name}.{file_extension}" + + source_file_path = file_path + dest_file_path = deployable_path / new_file_name + + worker.utils.info(f"Renaming file [{source_file_path}] to [{dest_file_path}]") + worker.utils.copy_file(source_file_path, dest_file_path) + + if file_extension == "zip": + worker.utils.info(f"Renaming internal folder to [{new_folder_name}]") + worker.utils.call(["7z", "rn", dest_file_path, current_folder_name, new_folder_name]) + elif file_extension == "tar.xz": + worker.utils.info(f"Extracting [{source_file_path}] to [{dest_file_path}]") + worker.utils.call(["tar", "-xf", source_file_path, "--directory", "."]) + + worker.utils.remove_file(dest_file_path) + worker.utils.move( + deployable_path / current_folder_name, deployable_path / new_folder_name + ) + + worker.utils.info(f"Compressing [{new_folder_name}] to [{dest_file_path}]") + cmd = [ + "tar", + "-cv", + "--owner=0", + "--group=0", + "--use-compress-program", + "xz -6", + "-f", + dest_file_path, + new_folder_name, + ] + worker.utils.call(cmd) + worker.utils.remove_dir(deployable_path / new_folder_name) + + checksum_file_paths.append(dest_file_path) + + # Create checksums + worker.utils.info("Creating checksums") + os.chdir(deployable_path) + + for checksum in checksums: + checksum_text = "" + for filepath in checksum_file_paths: + checksum_line = worker.utils.check_output([f"{checksum}sum", filepath.name]).strip() + checksum_text += checksum_line + "\n" + + print(checksum_text) + checksum_filepath = deployable_path / f"blender-{version_info.version}.{checksum}" + checksum_filepath.write_text(checksum_text) + + +def deploy(builder: worker.deploy.CodeDeployBuilder) -> None: + # No testable on UATEST currently. + dry_run = builder.service_env_id not in ("LOCAL", "PROD") + worker_config = builder.get_worker_config() + connect_id = f"{worker_config.download_user}@{worker_config.download_machine}" + + # Copy source + remote_dest_path = pathlib.Path(worker_config.download_source_folder) + change_modes = ["F0444"] + + if builder.service_env_id != "PROD": + # Already assumed to exist on production + worker.utils.call_ssh(connect_id, ["mkdir", "-p", remote_dest_path], dry_run=dry_run) + + for source_path in builder.package_source_dir.iterdir(): + dest_path = f"{connect_id}:{remote_dest_path}/" + worker.utils.info(f"Deploying source package [{source_path}]") + worker.utils.rsync( + source_path, dest_path, change_modes=change_modes, show_names=True, dry_run=dry_run + ) + + worker.utils.call_ssh(connect_id, ["ls", "-al", f"{remote_dest_path}/"], dry_run=dry_run) + + # Copy binaries + version_info = worker.blender.version.VersionInfo(builder) + major_minor_version = version_info.short_version + remote_dest_path = ( + pathlib.Path(worker_config.download_release_folder) / f"Blender{major_minor_version}" + ) + deployable_path = builder.package_dir / "deployable" + change_modes = ["F0444"] + + worker.utils.call_ssh(connect_id, ["mkdir", "-p", remote_dest_path], dry_run=dry_run) + worker.utils.call_ssh(connect_id, ["ls", "-al", f"{remote_dest_path}/"], dry_run=dry_run) + + for source_path in deployable_path.iterdir(): + dest_path = f"{connect_id}:{remote_dest_path}/" + worker.utils.info(f"Deploying binary package [{source_path}]") + worker.utils.rsync( + source_path, dest_path, change_modes=change_modes, show_names=True, dry_run=dry_run + ) + + worker.utils.call_ssh(connect_id, ["ls", "-al", f"{remote_dest_path}/"], dry_run=dry_run) diff --git a/config/worker/deploy/monitor.py b/config/worker/deploy/monitor.py new file mode 100644 index 0000000..206cb63 --- /dev/null +++ b/config/worker/deploy/monitor.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import re +import time +import urllib.request + +import worker.blender.version +import worker.deploy.artifacts +import worker.deploy +import worker.utils + + +def monitor(builder: worker.deploy.CodeDeployBuilder) -> None: + wait_time_in_seconds = 120 + + start_time = time.time() + max_time_hours = 4.0 + + version_info = worker.blender.version.VersionInfo(builder) + + required_base_url = "https://mirror.clarkson.edu/blender/release" + monitored_base_urls = [ + "https://download.blender.org/release", + "https://ftp.nluug.nl/pub/graphics/blender/release", + "https://ftp.halifax.rwth-aachen.de/blender/release", + "https://mirrors.dotsrc.org/blender/blender-release", + "https://mirrors.ocf.berkeley.edu/blender/release", + "https://mirrors.iu13.net/blender/release", + "https://mirrors.aliyun.com/blender/release", + "https://mirrors.sahilister.in/blender/release", + "https://mirror.freedif.org/blender/release", + required_base_url, + ] + + stop_on_required_site_found = False + + branches_config = builder.get_branches_config() + expected_platforms = branches_config.code_official_platform_architectures[builder.track_id] + + expected_file_count = len(worker.deploy.artifacts.checksums) + for expected_platform in expected_platforms: + if expected_platform.startswith("windows"): + expected_file_count += 3 # msi, msix, zip + else: + expected_file_count += 1 + + folder_name = f"Blender{version_info.short_version}" + file_pattern = rf"[Bb]lender-{version_info.version}[\.\-\_a-zA-Z0-9]*" + + while True: + found_site_count = 0 + print("=" * 80) + + # Assume no files are missing + sites_missing_files_count = 0 + + for base_url in monitored_base_urls: + search_url = f"{base_url}/{folder_name}" + print(f"Checking [{search_url}] for version [{version_info.version}]") + + # Header to avoid getting permission denied. + request = urllib.request.Request(search_url, headers={"User-Agent": "Mozilla"}) + + try: + response = urllib.request.urlopen(request, timeout=5.0) + text = response.read().decode("utf-8", "ignore") + except Exception as e: + print(e) + text = "" + + matches = set(re.findall(file_pattern, text)) + found_file_count = len(matches) + for match in matches: + print(f"File [{match}]") + + if len(matches) == expected_file_count: + found_site_count += 1 + elif len(matches) > 0: + sites_missing_files_count += 1 + print("-" * 80) + + can_stop_monitoring = ( + (len(matches) == expected_file_count) + and (base_url == required_base_url) + and (sites_missing_files_count == 0) + ) + + if stop_on_required_site_found and can_stop_monitoring: + print(f"Required site found [{required_base_url}], stopping") + return + + print("") + print("=" * 80) + print(f"Sites [{found_site_count} of {len(monitored_base_urls)}] have all files") + print("=" * 80) + + if found_site_count == len(monitored_base_urls): + break + + remaining_time_hours = max_time_hours - (time.time() - start_time) / 3600.0 + if remaining_time_hours < 0.0: + print("Waited for maximum amount of time, stopping") + break + + print( + f"Waiting {wait_time_in_seconds}s, total wait time remaining {remaining_time_hours:.2f}h" + ) + time.sleep(wait_time_in_seconds) diff --git a/config/worker/deploy/pypi.py b/config/worker/deploy/pypi.py new file mode 100644 index 0000000..51b8ae1 --- /dev/null +++ b/config/worker/deploy/pypi.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import json +import os +import urllib.request +import zipfile + + +import worker.blender +import worker.blender.version +import worker.deploy +import worker.utils + + +def pull(builder: worker.deploy.CodeDeployBuilder) -> None: + version_info = worker.blender.version.VersionInfo(builder) + + worker.utils.info("Cleaning package and download directory") + worker.utils.remove_dir(builder.package_dir) + worker.utils.remove_dir(builder.download_dir) + os.makedirs(builder.package_dir, exist_ok=True) + os.makedirs(builder.download_dir, exist_ok=True) + + # Fetch builds information. + env_base_url = { + "LOCAL": "https://builder.blender.org", + "UATEST": "https://builder.uatest.blender.org", + "PROD": "https://builder.blender.org", + } + base_url = env_base_url[builder.service_env_id] + + search_url = f"{base_url}/download/bpy/?format=json&v=1" + + worker.utils.info(f"Fetching build JSON from [{search_url}]") + + builds_response = urllib.request.urlopen(search_url) + builds_json = json.load(builds_response) + + # Get builds matching our version. + worker.utils.info("Processing build JSON") + + matching_builds = [] + for build in builds_json: + if build["version"] != version_info.version: + continue + if not build["file_name"].endswith(".zip"): + continue + worker.utils.info(f"Found {build['file_name']}") + if build["risk_id"] != "stable": + raise Exception("Can not only deploy stable releases") + matching_builds.append(build) + + # Check expected platforms + branches_config = builder.get_branches_config() + expected_platforms = branches_config.code_official_platform_architectures[builder.track_id] + if len(expected_platforms) != len(matching_builds): + platform_names = "\n".join(expected_platforms) + raise Exception("Unexpected number of builds, expected:\n" + platform_names) + + # Download builds. + for build in matching_builds: + file_uri = build["url"] + file_name = build["file_name"] + + worker.utils.info(f"Download [{file_uri}]") + download_file_path = builder.download_dir / file_name + urllib.request.urlretrieve(file_uri, download_file_path) + + # Unzip. + with zipfile.ZipFile(download_file_path, "r") as zipf: + zipf.extractall(path=builder.package_dir) + + worker.utils.remove_dir(builder.download_dir) + + +def deliver(builder: worker.deploy.CodeDeployBuilder) -> None: + dry_run = builder.service_env_id != "PROD" + wheels = list(builder.package_dir.glob("*.whl")) + + # Check expected platforms + branches_config = builder.get_branches_config() + expected_platforms = branches_config.code_official_platform_architectures[builder.track_id] + wheel_names = "\n".join([wheel.name for wheel in wheels]) + wheel_paths = [str(wheel) for wheel in wheels] + print(wheel_names) + if len(expected_platforms) != len(wheels): + raise Exception("Unexpected number of wheels:\n" + wheel_names) + + # Check wheels + cmd = ["twine", "check"] + wheel_paths + worker.utils.call(cmd) + + # Upload + worker_config = builder.get_worker_config() + env = os.environ.copy() + env["TWINE_USERNAME"] = "__token__" + env["TWINE_PASSWORD"] = worker_config.pypi_token(builder.service_env_id) + env["TWINE_REPOSITORY_URL"] = "https://upload.pypi.org/legacy/" + + cmd = ["twine", "upload", "--verbose", "--non-interactive"] + wheel_paths + worker.utils.call(cmd, env=env, dry_run=dry_run) diff --git a/config/worker/deploy/snap.py b/config/worker/deploy/snap.py new file mode 100644 index 0000000..cb06cb8 --- /dev/null +++ b/config/worker/deploy/snap.py @@ -0,0 +1,161 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import json +import os + +import worker.blender.version +import worker.deploy +import worker.utils + + +def package(builder: worker.deploy.CodeStoreBuilder) -> None: + dry_run = False + if builder.service_env_id == "LOCAL" and not ( + builder.platform == "linux" and worker.utils.is_tool("snapcraft") + ): + worker.utils.warning("Performing dry run on LOCAL service environment") + dry_run = True + elif not builder.platform == "linux": + raise Exception("Can only run snapcraft on Linux, aborting") + + version_info = worker.blender.version.VersionInfo(builder) + + needs_stable_grade = version_info.risk_id in ["candidate", "stable"] + grade = "stable" if needs_stable_grade else "devel" + + # Clean directory + for old_package_file in builder.store_snap_dir.glob("*.tar.xz"): + worker.utils.remove_file(old_package_file) + os.makedirs(builder.store_snap_dir, exist_ok=True) + + # Get input package file path + package_manifest = builder.package_dir / "manifest.json" + builds = json.loads(package_manifest.read_text()) + linux_package_file_path = None + + for build in builds: + if build["platform"] == "linux" and build["file_extension"] == "tar.xz": + linux_package_file_path = builder.package_dir / build["file_name"] + break + if not linux_package_file_path: + raise Exception(f"Linux package not found in [{builder.package_dir}] manifest") + + source_file_path = linux_package_file_path + dest_file_path = builder.store_snap_dir / linux_package_file_path.name + worker.utils.info(f"Copy file [{source_file_path}] -> [{dest_file_path}]") + worker.utils.copy_file(source_file_path, dest_file_path) + + freedesktop_path = builder.code_path / "release" / "freedesktop" + snap_source_root_path = freedesktop_path / "snap" + + blender_icon_file_name = "blender.svg" + snapcraft_template_file_path = snap_source_root_path / "blender-snapcraft-template.yaml" + + worker.utils.info(f"Using snap config file [{snapcraft_template_file_path}]") + snapcraft_text = snapcraft_template_file_path.read_text() + snapcraft_text = snapcraft_text.replace("@VERSION@", version_info.version) + snapcraft_text = snapcraft_text.replace("@GRADE@", grade) + snapcraft_text = snapcraft_text.replace("@ICON_PATH@", f"./{blender_icon_file_name}") + snapcraft_text = snapcraft_text.replace("@PACKAGE_PATH@", f"./{linux_package_file_path.name}") + + snapcraft_file_path = builder.store_snap_dir / "snapcraft.yaml" + worker.utils.info(f"Saving snapcraft config file [{snapcraft_file_path}]") + snapcraft_file_path.write_text(snapcraft_text) + print(snapcraft_text) + + snap_package_file_name = f"blender_{version_info.version}_amd64.snap" + snap_package_file_path = builder.store_snap_dir / snap_package_file_name + if snap_package_file_path.exists(): + worker.utils.info(f"Clearing snap file [{snap_package_file_path}]") + worker.utils.remove_file(snap_package_file_path) + + os.chdir(builder.store_snap_dir) + + # Copy all required files into working folder + source_file_path = freedesktop_path / "icons" / "scalable" / "apps" / blender_icon_file_name + dest_file_path = builder.store_snap_dir / "blender.svg" + worker.utils.info(f"Copy file [{source_file_path}] -> [{dest_file_path}]") + worker.utils.copy_file(source_file_path, dest_file_path) + + source_file_path = snap_source_root_path / "blender-wrapper" + dest_file_path = builder.store_snap_dir / "blender-wrapper" + worker.utils.info(f"Copy file [{source_file_path}] -> [{dest_file_path}]") + worker.utils.copy_file(source_file_path, dest_file_path) + + worker.utils.call(["snapcraft", "clean", "--use-lxd"], dry_run=dry_run) + worker.utils.call(["snapcraft", "--use-lxd"], dry_run=dry_run) + worker.utils.call( + ["review-tools.snap-review", snap_package_file_path, "--allow-classic"], dry_run=dry_run + ) + + if dry_run: + snap_package_file_path.write_text("Dry run dummy package file") + + worker.utils.info("To test the snap package run this command") + print("sudo snap remove blender") + print(f"sudo snap install --dangerous --classic {snap_package_file_path}") + + +def deliver(builder: worker.deploy.CodeStoreBuilder) -> None: + dry_run = False + if builder.service_env_id == "LOCAL": + worker.utils.warning("Performing dry run on LOCAL service environment") + dry_run = True + elif not builder.platform == "linux": + raise Exception("Can only run snapcraft on Linux, aborting") + + version_info = worker.blender.version.VersionInfo(builder) + branches_config = builder.get_branches_config() + is_lts = builder.track_id in branches_config.all_lts_tracks + is_latest = ( + branches_config.track_major_minor_versions[builder.track_id] == version_info.short_version + ) + + # Never push to stable + snap_risk_id = version_info.risk_id.replace("stable", "candidate").replace("alpha", "edge") + if snap_risk_id == "stable": + raise Exception("Delivery to [stable] channel not allowed") + + snap_track_id = version_info.short_version + + if is_lts: + snap_track_id += "lts" + needs_release = True + elif is_latest: + # latest/edge always vdev + snap_track_id = "latest" + needs_release = True + else: + # Push current release under development to beta or candidate + needs_release = True + + # worker.utils.call(["snapcraft", "list-tracks", "blender"], dry_run=dry_run) + snap_package_file_name = f"blender_{version_info.version}_amd64.snap" + snap_package_file_path = builder.store_snap_dir / snap_package_file_name + if not snap_package_file_path.exists(): + raise Exception(f"Snap file [{snap_package_file_path}] missing") + + worker_config = builder.get_worker_config() + env = os.environ.copy() + env["SNAPCRAFT_STORE_CREDENTIALS"] = worker_config.snap_credentials(builder.service_env_id) + + # If this fails, then the permissions were not set correcty with acls + worker.utils.call(["snapcraft", "status", "blender"], dry_run=dry_run, env=env) + + if needs_release: + # Upload and release. + snap_channel = f"{snap_track_id}/{snap_risk_id}" + cmd = ["snapcraft", "upload", "--release", snap_channel, snap_package_file_path] + else: + # Upload only. + snap_channel = "" + cmd = ["snapcraft", "upload", snap_package_file_path] + + # Some api call is making this fail, seems to be status based as we can upload and set channel + worker.utils.call(cmd, retry_count=5, retry_wait_time=120, dry_run=dry_run, env=env) + + if needs_release: + worker.utils.info("To test the snap package run this command") + print(f"sudo snap refresh blender --classic --channel {snap_channel}") diff --git a/config/worker/deploy/source.py b/config/worker/deploy/source.py new file mode 100644 index 0000000..cd58069 --- /dev/null +++ b/config/worker/deploy/source.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import os + +import worker.blender.version +import worker.deploy +import worker.utils + + +def _package(builder: worker.deploy.CodeDeployBuilder, needs_complete: bool = False) -> None: + os.chdir(builder.code_path) + if needs_complete: + worker.utils.call(["make", "source_archive_complete"]) + else: + worker.utils.call(["make", "source_archive"]) + + # The make change scripts writes to a different location since 2.83. + for source_file in builder.code_path.glob("blender-*.tar.xz*"): + worker.utils.move(source_file, builder.package_source_dir / source_file.name) + for source_file in builder.track_path.glob("blender-*.tar.xz*"): + worker.utils.move(source_file, builder.package_source_dir / source_file.name) + + +def package(builder: worker.deploy.CodeDeployBuilder) -> None: + print(f"Cleaning path [{builder.package_source_dir}]") + worker.utils.remove_dir(builder.package_source_dir) + os.makedirs(builder.package_source_dir, exist_ok=True) + + _package(builder, needs_complete=False) + + version_info = worker.blender.version.VersionInfo(builder) + if version_info.patch != 0: + worker.utils.info("Skipping complete source package for patch release") + return + + _package(builder, needs_complete=True) diff --git a/config/worker/deploy/steam.py b/config/worker/deploy/steam.py new file mode 100644 index 0000000..fa96bfe --- /dev/null +++ b/config/worker/deploy/steam.py @@ -0,0 +1,260 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import json +import os +import pathlib +import time + +import worker.blender.version +import worker.deploy +import worker.utils + + +def extract_file( + builder: worker.deploy.CodeStoreBuilder, source_file_path: pathlib.Path, platform: str +) -> None: + worker.utils.info(f"Extracting artifact [{source_file_path}] for Steam") + if not source_file_path.exists(): + raise Exception("File not found, aborting") + + dest_extract_path = builder.store_steam_dir / platform + dest_content_path = dest_extract_path / "content" + worker.utils.remove_dir(dest_extract_path) + worker.utils.remove_dir(dest_content_path) + os.makedirs(dest_extract_path, exist_ok=True) + + if platform == "linux": + worker.utils.info(f"Extract [{source_file_path}] to [{dest_extract_path}]") + cmd = ["tar", "-xf", source_file_path, "--directory", dest_extract_path] + worker.utils.call(cmd) + + # Move any folder there as ./content + for source_content_path in dest_extract_path.iterdir(): + if source_content_path.is_dir(): + worker.utils.info(f"Move [{source_content_path.name}] -> [{dest_content_path}]") + worker.utils.move(source_content_path, dest_content_path) + break + + elif platform == "darwin": + source_content_path = dest_extract_path / "Blender" + if source_content_path.exists(): + worker.utils.info(f"Removing [{source_content_path}]") + worker.utils.remove_dir(source_content_path) + + image_file_path = source_file_path.with_suffix(".img") + + cmd = ["dmg2img", "-v", "-i", source_file_path, "-o", image_file_path] + worker.utils.call(cmd) + + cmd = ["7z", "x", f"-o{dest_extract_path}", image_file_path] + worker.utils.call(cmd) + + os.makedirs(dest_content_path, exist_ok=True) + + worker.utils.remove_file(image_file_path) + + worker.utils.info(f"Move Blender app from [{source_content_path}] -> [{dest_content_path}]") + worker.utils.move(source_content_path / "Blender.app", dest_content_path / "Blender.app") + worker.utils.remove_dir(source_content_path) + elif platform == "windows": + worker.utils.info(f"Extracting zip file [{source_file_path}]") + cmd = ["7z", "x", f"-o{dest_extract_path}", source_file_path] + worker.utils.call(cmd) + + # Move any folder there as ./content + for source_content_path in dest_extract_path.iterdir(): + if source_content_path.is_dir(): + worker.utils.info(f"Move [{source_content_path.name}] -> [{dest_content_path}]") + worker.utils.move(source_content_path, dest_content_path) + break + else: + raise Exception(f"Don't know how to extract for platform [{platform}]") + + +def extract(builder: worker.deploy.CodeStoreBuilder) -> None: + package_manifest = builder.package_dir / "manifest.json" + builds = json.loads(package_manifest.read_text()) + + for build in builds: + if build["file_extension"] not in ["zip", "tar.xz", "dmg"]: + continue + if build["architecture"] == "arm64": + continue + + file_path = builder.package_dir / build["file_name"] + platform = build["platform"] + extract_file(builder, file_path, platform) + + +def build(builder: worker.deploy.CodeStoreBuilder, is_preview: bool) -> None: + dry_run = False + if builder.service_env_id == "LOCAL": + worker.utils.warning("Performing dry run on LOCAL service environment") + dry_run = True + + version_info = worker.blender.version.VersionInfo(builder) + branches_config = builder.get_branches_config() + is_lts = builder.track_id in branches_config.all_lts_tracks + is_latest = branches_config.track_major_minor_versions["vdev"] == version_info.short_version + + track_path = builder.track_path + log_path = builder.track_path / "log" + worker.utils.remove_dir(log_path) + os.makedirs(log_path, exist_ok=True) + + worker_config = builder.get_worker_config() + steam_credentials = worker_config.steam_credentials(builder.service_env_id) + steam_user_id, steam_user_password = steam_credentials + if not steam_user_id or not steam_user_password: + if not dry_run: + raise Exception("Steam user id or password not available, aborting") + + env = os.environ.copy() + env["PATH"] = env["PATH"] + os.pathsep + "/usr/games" + + cmd: worker.utils.CmdSequence = [ + "steamcmd", + "+login", + worker.utils.HiddenArgument(steam_user_id), + worker.utils.HiddenArgument(steam_user_password), + "+quit", + ] + worker.utils.call(cmd, dry_run=dry_run, env=env) + + worker.utils.info("Waiting 5 seconds for next steam command") + time.sleep(5.0) + + steam_app_id = worker_config.steam_app_id + steam_platform_depot_ids = worker_config.steam_platform_depot_ids + + for platform_id in ["linux", "darwin", "windows"]: + worker.utils.info(f"Platform {platform_id}") + + platform_depot_id = steam_platform_depot_ids[platform_id] + + track_build_root_path = builder.store_steam_dir / platform_id + if not track_build_root_path.exists(): + raise Exception(f"Folder {track_build_root_path} does not exist") + + platform_build_file_path = track_build_root_path / "depot_build.vdf" + + source_root_path = track_build_root_path / "content" + if not source_root_path.exists(): + raise Exception(f"Folder {source_root_path} does not exist") + + dest_root_path = track_build_root_path / "output" + + # Steam branches cannot be uper case and no spaces allowed + # Branches are named "daily" and "devtest" on Steam, so rename those. + steam_branch_id = builder.service_env_id.lower() + steam_branch_id = steam_branch_id.replace("prod", "daily") + steam_branch_id = steam_branch_id.replace("uatest", "devtest") + + if is_lts: + # daily-X.X and devtest-X.X branches for LTS. + steam_branch_id = f"{steam_branch_id}-{version_info.short_version}" + elif is_latest: + # daily and devtest branches for main without suffix. + pass + else: + # Not setting this live. + steam_branch_id = "" + + preview = "1" if is_preview else "0" + + app_build_script = f""" +"appbuild" +{{ + "appid" "{steam_app_id}" + "desc" "Blender {version_info.version}" // description for this build + "buildoutput" "{dest_root_path}" // build output folder for .log, .csm & .csd files, relative to location of this file + "contentroot" "{source_root_path}" // root content folder, relative to location of this file + "setlive" "{steam_branch_id}" // branch to set live after successful build, non if empty + "preview" "{preview}" // 1 to enable preview builds, 0 to commit build to steampipe + "local" "" // set to file path of local content server + + "depots" + {{ + "{platform_depot_id}" "{platform_build_file_path}" + }} +}} +""" + + platform_build_script = f""" +"DepotBuildConfig" +{{ + // Set your assigned depot ID here + "DepotID" "{platform_depot_id}" + + // Set a root for all content. + // All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths) + // will be resolved relative to this root. + // If you don't define ContentRoot, then it will be assumed to be + // the location of this script file, which probably isn't what you want + "ContentRoot" "{source_root_path}" + + // include all files recursivley + "FileMapping" + {{ + // This can be a full path, or a path relative to ContentRoot + "LocalPath" "*" + + // This is a path relative to the install folder of your game + "DepotPath" "." + + // If LocalPath contains wildcards, setting this means that all + // matching files within subdirectories of LocalPath will also + // be included. + "recursive" "1" + }} + + // but exclude all symbol files + // This can be a full path, or a path relative to ContentRoot + //"FileExclusion" "*.pdb" +}} +""" + + (track_build_root_path / "app_build.vdf").write_text(app_build_script) + platform_build_file_path.write_text(platform_build_script) + + worker.utils.info( + f"Version [{version_info.version}] for [{platform_id}] in preview [{is_preview}] for steam branch [{steam_branch_id}], building" + ) + + cmd = [ + "steamcmd", + "+login", + worker.utils.HiddenArgument(steam_user_id), + worker.utils.HiddenArgument(steam_user_password), + "+run_app_build", + track_build_root_path / "app_build.vdf", + "+quit", + ] + retry_count = 0 if preview else 3 + + worker.utils.call( + cmd, retry_count=retry_count, retry_wait_time=120, dry_run=dry_run, env=env + ) + + worker.utils.info("Waiting 5 seconds for next steam command") + time.sleep(5.0) + + worker.utils.info( + f"Version [{version_info.version}] for [{platform_id}] in preview [{is_preview}] is done, success" + ) + + +def package(builder: worker.deploy.CodeStoreBuilder) -> None: + worker.utils.remove_dir(builder.store_steam_dir) + os.makedirs(builder.store_steam_dir, exist_ok=True) + + # Extract and prepare content + extract(builder) + build(builder, is_preview=True) + + +def deliver(builder: worker.deploy.CodeStoreBuilder) -> None: + # This will push to the store + build(builder, is_preview=False) diff --git a/config/worker/deploy/windows.py b/config/worker/deploy/windows.py new file mode 100644 index 0000000..8bece72 --- /dev/null +++ b/config/worker/deploy/windows.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import json +import os + +import worker.blender.pack +import worker.blender.sign +import worker.blender.version +import worker.blender.msix_package + +import worker.deploy +import worker.utils + + +def _package_architecture( + builder: worker.deploy.CodeStoreBuilder, architecture: str, dry_run: bool +) -> None: + version_info = worker.blender.version.VersionInfo(builder) + + # Revision with MS Store must be set to 0 + revision_id = 0 + + branches_config = builder.get_branches_config() + is_lts = builder.track_id in branches_config.windows_store_lts_tracks + base_build_number = 0 + + build_number = version_info.patch + base_build_number + worker.utils.info(f"Builder number {build_number}") + + store_version_id = f"{version_info.short_version}.{build_number}.{revision_id}" + worker.utils.info(f"Store version ID {store_version_id}") + + worker.utils.info(f"Cleaning path [{builder.store_windows_dir}]") + worker.utils.remove_dir(builder.store_windows_dir) + os.makedirs(builder.store_windows_dir, exist_ok=True) + + os.chdir(builder.store_windows_dir) + + # Find input zip package. + package_manifest = builder.package_dir / "manifest.json" + builds = json.loads(package_manifest.read_text()) + input_file_path = None + + for build in builds: + if ( + build["platform"] == "windows" + and build["file_extension"] == "zip" + and build["architecture"] == architecture + ): + input_file_path = builder.package_dir / build["file_name"] + break + if not input_file_path: + raise Exception(f"Windows package not found in [{builder.package_dir}] manifest") + + # Copy all required files into working folder + source_path = builder.code_path / "release" / "windows" / "msix" + dest_path = builder.store_windows_dir + worker.utils.info(f"Copying [{source_path}] -> [{dest_path}] for windows store packaging") + + for source_file in source_path.iterdir(): + if source_file.name == "README.md": + continue + if source_file.is_dir(): + worker.utils.copy_dir(source_file, dest_path / source_file.name) + else: + worker.utils.copy_file(source_file, dest_path / source_file.name) + + worker_config = builder.get_worker_config() + + cert_subject = worker_config.windows_store_certificate(builder.service_env_id) + certificate_id = f"CN={cert_subject}" + + msix_filepath = worker.blender.msix_package.pack( + store_version_id, input_file_path, certificate_id, lts=is_lts, dry_run=dry_run + ) + + if worker_config.windows_store_self_sign: + worker.blender.sign.sign_windows_files( + builder.service_env_id, [msix_filepath], certificate_id=certificate_id + ) + + if dry_run: + msix_filepath.write_text("Dry run dummy package file") + + # Clear out all msix files first + for old_msix_filepath in builder.package_dir.glob("*.msix"): + worker.utils.remove_file(old_msix_filepath) + + dest_path = builder.package_dir / msix_filepath.name + worker.utils.info(f"Copying [{msix_filepath}] -> [{dest_path}] for distribution") + worker.utils.copy_file(msix_filepath, dest_path) + worker.blender.pack.generate_file_hash(dest_path) + + +def package(builder: worker.deploy.CodeStoreBuilder) -> None: + dry_run = False + if not builder.platform == "windows": + if builder.service_env_id == "LOCAL": + worker.utils.warning("Performing dry run on LOCAL service environment") + dry_run = True + else: + raise Exception("Can only run this on Windows, aborting") + + branches_config = builder.get_branches_config() + expected_platforms = branches_config.code_official_platform_architectures[builder.track_id] + + for expected_platform in expected_platforms: + if expected_platform.startswith("windows"): + architecture = expected_platform.split("-")[1] + _package_architecture(builder, architecture, dry_run) + + +def deliver(builder: worker.deploy.CodeStoreBuilder) -> None: + worker.utils.info("Windows store delivery not implemented") diff --git a/config/worker/doc_api.py b/config/worker/doc_api.py new file mode 100755 index 0000000..06bf744 --- /dev/null +++ b/config/worker/doc_api.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import argparse +import os +import pathlib +import sys + +from collections import OrderedDict + +sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) + +import worker.configure +import worker.utils + +import worker.blender +import worker.blender.compile +import worker.blender.update +import worker.blender.version + + +class DocApiBuilder(worker.blender.CodeBuilder): + def __init__(self, args: argparse.Namespace): + super().__init__(args) + self.needs_package_delivery = args.needs_package_delivery + self.setup_track_path() + + +def download_api_dump_test_data(local_delivery_path: pathlib.Path) -> None: + import urllib.request + import json + + api_base_url = "https://docs.blender.org/api" + api_dump_index_url = f"{api_base_url}/api_dump_index.json" + + request = urllib.request.Request(api_dump_index_url, headers={"User-Agent": "Mozilla"}) + response = urllib.request.urlopen(request, timeout=5.0) + + api_dump_index_text = response.read().decode("utf-8", "ignore") + api_dump_index_path = local_delivery_path / "api_dump_index.json" + os.makedirs(api_dump_index_path.parent, exist_ok=True) + api_dump_index_path.write_text(api_dump_index_text) + + api_dump_index = json.loads(api_dump_index_text) + for version in api_dump_index.keys(): + api_dump_url = f"{api_base_url}/{version}/api_dump.json" + worker.utils.info(f"Download {api_dump_url}") + + request = urllib.request.Request(api_dump_url, headers={"User-Agent": "Mozilla"}) + response = urllib.request.urlopen(request, timeout=5.0) + + api_dump_text = response.read().decode("utf-8", "ignore") + api_dump_path = local_delivery_path / version / "api_dump.json" + os.makedirs(api_dump_path.parent, exist_ok=True) + api_dump_path.write_text(api_dump_text) + + +def compile_doc(builder: DocApiBuilder) -> None: + # Install requirements + os.chdir(builder.track_path) + doc_api_script_path = builder.code_path / "doc" / "python_api" + worker.utils.call_pipenv( + ["install", "--requirements", doc_api_script_path / "requirements.txt"] + ) + + # Clean build directory + worker.utils.remove_dir(builder.build_doc_path) + os.makedirs(builder.build_doc_path, exist_ok=True) + + os.chdir(doc_api_script_path) + + # Get API dumps data from server. + api_dump_build_path = builder.build_doc_path / "api_dump" + os.makedirs(api_dump_build_path, exist_ok=True) + + api_dump_include_paths = ["api_dump_index.json", "*/", "api_dump.json"] + api_dump_build_path_index = api_dump_build_path / "api_dump_index.json" + + worker_config = builder.get_worker_config() + connect_id = f"{worker_config.docs_user}@{worker_config.docs_machine}" + remote_path = ( + pathlib.Path(worker_config.docs_folder) + / "docs.blender.org" + / "htdocs" + / builder.service_env_id + / "api" + ) + + # Get data from docs.blender.org for local testing. + if builder.service_env_id == "LOCAL": + worker.utils.info("Downloading API dump data from docs.blender.org for testing") + download_api_dump_test_data(remote_path) + + source_path = f"{connect_id}:{remote_path}/" + dest_path = api_dump_build_path + + worker.utils.rsync( + source_path, dest_path, include_paths=api_dump_include_paths, exclude_paths=["*"] + ) + + version = worker.blender.version.VersionInfo(builder).short_version + api_dump_build_path_current_version = api_dump_build_path / version + os.makedirs(api_dump_build_path_current_version, exist_ok=True) + + # Generate API docs + cmd = [ + builder.blender_command_path(), + "--background", + "--factory-startup", + "-noaudio", + "--python", + doc_api_script_path / "sphinx_doc_gen.py", + "--", + "--output", + builder.build_doc_path, + "--api-changelog-generate", + "--api-dump-index-path", + api_dump_build_path_index, + ] + worker.utils.call(cmd) + + num_threads = worker.configure.get_thread_count(thread_memory_in_GB=1.25) + + in_path = builder.build_doc_path / "sphinx-in" + out_path = builder.build_doc_path / "sphinx-out-html" + worker.utils.call(["sphinx-build", "-b", "html", "-j", str(num_threads), in_path, out_path]) + + +def package(builder: DocApiBuilder) -> None: + os.chdir(builder.build_doc_path) + + version = worker.blender.version.VersionInfo(builder).short_version + version_file_label = version.replace(".", "_") + + package_name = f"blender_python_reference_{version_file_label}" + package_file_name = f"{package_name}.zip" + + cmd = ["7z", "a", "-tzip", package_file_name, "./sphinx-out-html", "-r"] + worker.utils.call(cmd) + + cmd = ["7z", "rn", package_file_name, "sphinx-out-html", package_name] + worker.utils.call(cmd) + + +def deliver(builder: DocApiBuilder) -> None: + # Get versions + branches_config = builder.get_branches_config() + version = worker.blender.version.VersionInfo(builder).short_version + dev_version = branches_config.track_major_minor_versions["vdev"] + latest_version = branches_config.doc_stable_major_minor_version + + # Get remote path + worker_config = builder.get_worker_config() + connect_id = f"{worker_config.docs_user}@{worker_config.docs_machine}" + remote_path = ( + pathlib.Path(worker_config.docs_folder) + / "docs.blender.org" + / "htdocs" + / builder.service_env_id + / "api" + ) + + version_remote_path = remote_path / version + worker.utils.call_ssh(connect_id, ["mkdir", "-p", version_remote_path]) + + change_modes = ["D0755", "F0644"] + + # Sync HTML files + source_path = f"{builder.build_doc_path}/sphinx-out-html/" + dest_path = f"{connect_id}:{version_remote_path}/" + worker.utils.rsync( + source_path, dest_path, exclude_paths=[".doctrees"], change_modes=change_modes + ) + + # Put API dumps data on the server. + api_dump_build_path = f"{builder.build_doc_path}/api_dump/" + api_dump_dest_path = f"{connect_id}:{remote_path}/" + worker.utils.rsync(api_dump_build_path, api_dump_dest_path, change_modes=change_modes) + + # Sync zip package + if builder.needs_package_delivery: + version_file_label = version.replace(".", "_") + + package_name = f"blender_python_reference_{version_file_label}" + package_file_name = f"{package_name}.zip" + + source_file_path = builder.build_doc_path / package_file_name + dest_file_path = f"{connect_id}:{version_remote_path}/{package_file_name}" + worker.utils.rsync( + source_file_path, dest_file_path, exclude_paths=[".doctrees"], change_modes=change_modes + ) + + # Create links + if builder.track_id == "vdev": + worker.utils.call_ssh( + connect_id, ["ln", "-svF", remote_path / dev_version, remote_path / "dev"] + ) + worker.utils.call_ssh( + connect_id, ["ln", "-svF", remote_path / dev_version, remote_path / "master"] + ) + worker.utils.call_ssh( + connect_id, ["ln", "-svF", remote_path / dev_version, remote_path / "main"] + ) + worker.utils.call_ssh( + connect_id, ["ln", "-svF", remote_path / latest_version, remote_path / "latest"] + ) + worker.utils.call_ssh( + connect_id, ["ln", "-svF", remote_path / latest_version, remote_path / "current"] + ) + + +if __name__ == "__main__": + steps: worker.utils.BuilderSteps = OrderedDict() + steps["configure-machine"] = worker.configure.configure_machine + steps["update-code"] = worker.blender.update.update + steps["compile-code"] = worker.blender.compile.compile_code + steps["compile-install"] = worker.blender.compile.compile_install + steps["compile"] = compile_doc + steps["package"] = package + steps["deliver"] = deliver + steps["clean"] = worker.blender.CodeBuilder.clean + + parser = worker.blender.create_argument_parser(steps=steps) + parser.add_argument("--needs-package-delivery", action="store_true", required=False) + + args = parser.parse_args() + builder = DocApiBuilder(args) + builder.run(args.step, steps) diff --git a/config/worker/doc_developer.py b/config/worker/doc_developer.py new file mode 100755 index 0000000..50fbd8f --- /dev/null +++ b/config/worker/doc_developer.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import argparse +import os +import pathlib +import sys + +from collections import OrderedDict + +sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) + +import worker.utils + + +class DocDeveloperBuilder(worker.utils.Builder): + def __init__(self, args: argparse.Namespace): + super().__init__(args, "blender", "blender-developer-docs") + self.build_path = self.track_path / "build_developer_docs" + self.output_path = self.build_path / "html" + self.setup_track_path() + + +def update(builder: DocDeveloperBuilder) -> None: + builder.update_source() + + +def compile_doc(builder: DocDeveloperBuilder) -> None: + os.chdir(builder.track_path) + worker.utils.call_pipenv(["install", "--requirements", builder.code_path / "requirements.txt"]) + + worker.utils.remove_dir(builder.output_path) + + os.makedirs(builder.build_path, exist_ok=True) + os.chdir(builder.build_path) + + mkdocs_yml_path = builder.code_path / "mkdocs.yml" + worker.utils.call_pipenv( + ["run", "mkdocs", "build", "-f", mkdocs_yml_path, "-d", builder.output_path] + ) + + +def deliver(builder: DocDeveloperBuilder) -> None: + worker_config = builder.get_worker_config() + + remote_path = f"developer.blender.org/webroot/{builder.service_env_id}/docs" + + connect_id = f"{worker_config.docs_user}@{worker_config.docs_machine}" + server_docs_path = pathlib.Path(worker_config.docs_folder) / pathlib.Path(remote_path) + + change_modes = ["D0755", "F0644"] + source_path = f"{builder.output_path}/" + dest_path = f"{connect_id}:{server_docs_path}/" + + worker.utils.call_ssh(connect_id, ["mkdir", "-p", server_docs_path]) + worker.utils.rsync( + source_path, + dest_path, + change_modes=change_modes, + port=worker_config.docs_port, + delete=True, + delete_path_check=f"/developer.blender.org/webroot/{builder.service_env_id}/docs", + ) + + +if __name__ == "__main__": + steps: worker.utils.BuilderSteps = OrderedDict() + steps["update"] = update + steps["compile"] = compile_doc + steps["deliver"] = deliver + + parser = worker.utils.create_argument_parser(steps=steps) + parser.add_argument("--needs-package-delivery", action="store_true", required=False) + + args = parser.parse_args() + builder = DocDeveloperBuilder(args) + builder.run(args.step, steps) diff --git a/config/worker/doc_manual.py b/config/worker/doc_manual.py new file mode 100755 index 0000000..c15654a --- /dev/null +++ b/config/worker/doc_manual.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import argparse +import json +import os +import pathlib +import re +import sys +import time + +from collections import OrderedDict +from datetime import timedelta +from typing import Optional, Sequence + +sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) + +import worker.configure +import worker.utils + + +class ManualBuilder(worker.utils.Builder): + def __init__(self, args: argparse.Namespace): + super().__init__(args, "blender", "blender-manual") + self.needs_all_locales = args.needs_all_locales + self.needs_package_delivery = args.needs_package_delivery + self.doc_format = args.doc_format + self.build_path = self.track_path / "build" + self.setup_track_path() + + def get_locales(self) -> Sequence[str]: + locales = ["en"] + if self.needs_all_locales: + locale_path = self.code_path / "locale" + locales += [ + item.name for item in locale_path.iterdir() if not item.name.startswith(".") + ] + return locales + + +def update(builder: ManualBuilder) -> None: + builder.update_source() + if builder.needs_all_locales: + worker.utils.update_source( + "blender", "blender-manual-translations", builder.code_path / "locale" + ) + + +def check(builder: ManualBuilder) -> None: + os.chdir(builder.track_path) + worker.utils.call_pipenv(["install", "--pre", "--requirements", builder.code_path / "requirements.txt"]) + + os.chdir(builder.code_path) + + make_cmd = "make.bat" if builder.platform == "windows" else "make" + worker.utils.call_pipenv(["run", make_cmd, "check_structure"]) + # worker.utils.call_pipenv(["run", make_cmd, "check_syntax"]) + # worker.utils.call_pipenv(["run", make_cmd, "check_spelling"]) + + +def compile_doc(builder: ManualBuilder) -> None: + # Install requirements. + os.chdir(builder.track_path) + worker.utils.call_pipenv(["install", "--pre", "--requirements", builder.code_path / "requirements.txt"]) + + # Determine format and locales + locales = builder.get_locales() + doc_format = builder.doc_format + + # Clean build folder + worker.utils.remove_dir(builder.build_path) + os.makedirs(builder.build_path, exist_ok=True) + os.chdir(builder.code_path) + + branches_config = builder.get_branches_config() + + # Check manual version matches track. + conf_file_path = builder.code_path / "manual" / "conf.py" + conf_text = conf_file_path.read_text() + match = re.search(r"blender_version\s*=\s*['\"](.*)['\"]", conf_text) + expected_version = branches_config.track_major_minor_versions[builder.track_id] + found_version = match.groups(0)[0] if match else "nothing" + if found_version != expected_version: + raise Exception( + f"Expected blender_version {expected_version}, but found {found_version} in manual/conf.py" + ) + + def filter_output(line: str) -> Optional[str]: + if line.find("WARNING: unknown mimetype for .doctrees") != -1: + return None + elif line.find("copying images...") != -1: + return None + return line + + # Generate manual + for locale in locales: + start_timestamp = time.time() + worker.utils.info(f"Generating {locale} in {doc_format}") + + num_threads = worker.configure.get_thread_count(thread_memory_in_GB=1.25) + + os.chdir(builder.code_path) + build_output_path = builder.build_path / doc_format / locale + + worker.utils.call_pipenv( + [ + "run", + "sphinx-build", + "-b", + doc_format, + "-j", + str(num_threads), + "-D", + f"language={locale}", + "./manual", + build_output_path, + ], + filter_output=filter_output, + ) + + if doc_format == "epub": + if not build_output_path.rglob("*.epub"): + raise Exception(f"Expected epub files missing in {build_output_path}") + + # Hack appropriate versions.json URL into version_switch.js + worker.utils.info("Replacing URL in version_switch.js") + + version_switch_file_path = build_output_path / "_static" / "js" / "version_switch.js" + versions_file_url = f"https://docs.blender.org/{builder.service_env_id}/versions.json" + + version_switch_text = version_switch_file_path.read_text() + version_switch_text = version_switch_text.replace( + "https://docs.blender.org/versions.json", versions_file_url + ) + version_switch_text = version_switch_text.replace( + "https://docs.blender.org/PROD/versions.json", versions_file_url + ) + version_switch_text = version_switch_text.replace( + "https://docs.blender.org/UATEST/versions.json", versions_file_url + ) + version_switch_file_path.write_text(version_switch_text) + + time_total = time.time() - start_timestamp + time_delta = str(timedelta(seconds=time_total)) + worker.utils.info(f"Generated {locale} in {doc_format} in {time_delta}") + + +def package(builder: ManualBuilder) -> None: + if not builder.needs_package_delivery: + worker.utils.info("No package delivery needed, skipping packaging") + return + + locales = builder.get_locales() + doc_format = builder.doc_format + + os.chdir(builder.build_path) + + compression_option = "" # "-mx=9" + package_file_name = f"blender_manual_{doc_format}.zip" + + build_package_path = builder.build_path / "package" + + for locale in locales: + package_file_path = build_package_path / locale / package_file_name + worker.utils.remove_file(package_file_path) + + source_path = f"{doc_format}/{locale}" + + cmd = [ + "7z", + "a", + "-tzip", + package_file_path, + source_path, + "-r", + "-xr!.doctrees", + compression_option, + ] + worker.utils.call(cmd) + + cmd = [ + "7z", + "rn", + package_file_path, + source_path, + f"blender_manual_{builder.track_id}_{locale}.{doc_format}", + ] + worker.utils.call(cmd) + + +def deliver(builder: ManualBuilder) -> None: + locales = builder.get_locales() + doc_format = builder.doc_format + + # Get versions + branches_config = builder.get_branches_config() + version = branches_config.track_major_minor_versions[builder.track_id] + dev_version = branches_config.track_major_minor_versions["vdev"] + latest_version = branches_config.doc_stable_major_minor_version + + # Get remote paths + worker_config = builder.get_worker_config() + connect_id = f"{worker_config.docs_user}@{worker_config.docs_machine}" + docs_remote_path = ( + pathlib.Path(worker_config.docs_folder) + / "docs.blender.org" + / "htdocs" + / builder.service_env_id + ) + + # Sync each locale + for locale in locales: + worker.utils.info(f"Syncing {locale}") + + # Create directory + remote_path = docs_remote_path / "manual" / locale + version_remote_path = remote_path / version + worker.utils.call_ssh(connect_id, ["mkdir", "-p", version_remote_path]) + + if doc_format == "html": + # Sync html files + source_path = f"{builder.build_path}/{doc_format}/{locale}/" + dest_path = f"{connect_id}:{version_remote_path}/" + # Exclude packaged download files; these get synced with `needs_package_delivery`. + worker.utils.rsync( + source_path, + dest_path, + exclude_paths=[".doctrees", "blender_manual_*.zip"], + delete=True, + delete_path_check=str(version_remote_path) + ) + + # Create links + if builder.track_id == "vdev": + worker.utils.info(f"Creating links for {locale}") + worker.utils.call_ssh( + connect_id, ["ln", "-svF", remote_path / dev_version, remote_path / "dev"] + ) + worker.utils.call_ssh( + connect_id, ["ln", "-svF", remote_path / latest_version, remote_path / "latest"] + ) + + if builder.needs_package_delivery: + # Sync zip package + worker.utils.info(f"Syncing package for {locale}") + build_package_path = builder.build_path / "package" + package_file_name = f"blender_manual_{doc_format}.zip" + source_path = build_package_path / locale / package_file_name + dest_path = f"{connect_id}:{version_remote_path}/{package_file_name}" + worker.utils.rsync(source_path, dest_path, exclude_paths=[".doctrees"]) + + # Create and sync versions.json + worker.utils.info("Creating and syncing versions.json") + + doc_version_labels = branches_config.doc_manual_version_labels + versions_path = builder.build_path / "versions.json" + versions_path.write_text(json.dumps(doc_version_labels, indent=2)) + worker.utils.info(versions_path.read_text()) + + dest_path = f"{connect_id}:{docs_remote_path}/versions.json" + worker.utils.rsync(versions_path, dest_path) + + +def clean(builder: ManualBuilder) -> None: + worker.utils.remove_dir(builder.build_path) + + +if __name__ == "__main__": + steps: worker.utils.BuilderSteps = OrderedDict() + steps["configure-machine"] = worker.configure.configure_machine + steps["update"] = update + steps["check"] = check + steps["compile"] = compile_doc + steps["package"] = package + steps["deliver"] = deliver + steps["clean"] = clean + + parser = worker.utils.create_argument_parser(steps=steps) + parser.add_argument("--needs-all-locales", action="store_true", required=False) + parser.add_argument("--needs-package-delivery", action="store_true", required=False) + parser.add_argument( + "--doc-format", default="html", type=str, required=False, choices=["html", "epub"] + ) + + args = parser.parse_args() + builder = ManualBuilder(args) + builder.run(args.step, steps) diff --git a/config/worker/doc_studio.py b/config/worker/doc_studio.py new file mode 100755 index 0000000..3b6104a --- /dev/null +++ b/config/worker/doc_studio.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import argparse +import os +import pathlib +import sys + +from collections import OrderedDict + +sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) + +import worker.utils + + +class DocStudioBuilder(worker.utils.Builder): + def __init__(self, args: argparse.Namespace): + super().__init__(args, "studio", "blender-studio-tools") + self.setup_track_path() + + +def update(builder: worker.utils.Builder) -> None: + builder.update_source(update_submodules=True) + + +def compile_doc(builder: worker.utils.Builder) -> None: + docs_path = builder.code_path / "docs" + os.chdir(docs_path) + + worker.utils.call(["npm", "install"]) + worker.utils.call(["npm", "run", "docs:build"]) + + +def deliver(builder: worker.utils.Builder) -> None: + dry_run = False + if builder.service_env_id not in ("PROD", "LOCAL"): + worker.utils.warning("Delivery from non-PROD is dry run only") + dry_run = True + + worker_config = builder.get_worker_config() + connect_id = f"{worker_config.studio_user}@{worker_config.studio_machine}" + change_modes = ["D0755", "F0644"] + + if builder.service_env_id == "LOCAL" and builder.platform == "darwin": + worker.utils.warning("rsync change_owner not supported on darwin, ignoring for LOCAL") + change_owner = None + else: + change_owner = "buildbot:www-data" + + # Content of the website. + docs_local_path = builder.code_path / "docs" / ".vitepress" / "dist" + docs_remote_path = pathlib.Path(worker_config.studio_folder) + + docs_source_path = f"{docs_local_path}/" + docs_dest_path = f"{connect_id}:{docs_remote_path}/" + worker.utils.rsync( + docs_source_path, + docs_dest_path, + change_modes=change_modes, + change_owner=change_owner, + port=worker_config.studio_port, + dry_run=dry_run, + ) + + # Downloadable artifacts. + artifacts_local_path = builder.code_path / "dist" + artifacts_remote_path = docs_remote_path / "download" + if artifacts_local_path.exists(): + artifacts_source_path = f"{artifacts_local_path}/" + artifact_dest_path = f"{connect_id}:{artifacts_remote_path}/" + worker.utils.rsync( + artifacts_source_path, + artifact_dest_path, + change_modes=change_modes, + change_owner=change_owner, + port=worker_config.studio_port, + dry_run=dry_run, + ) + else: + worker.utils.info("No downloadable artifacts to be copied over") + + +if __name__ == "__main__": + steps: worker.utils.BuilderSteps = OrderedDict() + steps["update"] = update + steps["compile"] = compile_doc + steps["deliver"] = deliver + + parser = worker.utils.create_argument_parser(steps=steps) + parser.add_argument("--needs-package-delivery", action="store_true", required=False) + + args = parser.parse_args() + builder = DocStudioBuilder(args) + builder.run(args.step, steps) diff --git a/config/worker/utils.py b/config/worker/utils.py new file mode 100644 index 0000000..ba2e77f --- /dev/null +++ b/config/worker/utils.py @@ -0,0 +1,549 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +## Utility functions used by all builders. + +import argparse +import atexit +import logging +import os +import pathlib +import platform +import re +import shutil +import subprocess +import sys +import time + +from collections import OrderedDict +from typing import Any, Callable, Dict, List, Optional, Sequence, Union + +# Logging +_error_pattern = re.compile( + r"(^FATAL:|^ERROR:|^ERROR!|^Unhandled Error|^Traceback| error: | error | Error |FAILED: |ninja: build stopped: subcommand failed|CMake Error|SEGFAULT|Exception: SegFault |The following tests FAILED:|\*\*\*Failed|\*\*\*Exception|\*\*\*Abort|^fatal:)" +) +_warning_pattern = re.compile( + r"(^WARNING:|^WARNING!|^WARN |Warning: | warning: | warning |warning | nvcc warning :|CMake Warning)" +) +_ignore_pattern = re.compile( + r"(SignTool Error: CryptSIPRemoveSignedDataMsg returned error: 0x00000057|unknown mimetype for .*doctree)" +) + +_errors: List[str] = [] +_warnings: List[str] = [] + + +def _print_warning(msg: str) -> None: + print("\033[33m" + msg + "\033[0m", flush=True) + + +def _print_error(msg: str) -> None: + print("\033[31m" + msg + "\033[0m", flush=True) + + +def _print_cmd(msg: str) -> None: + print("\033[32m" + msg + "\033[0m", flush=True) + + +def _exit_handler() -> None: + if len(_warnings): + print("") + print("=" * 80) + print("WARNING Summary:") + print("=" * 80) + for msg in _warnings: + _print_warning(msg) + if len(_errors): + print("") + print("=" * 80) + print("ERROR Summary:") + print("=" * 80) + for msg in _errors: + _print_error(msg) + + +atexit.register(_exit_handler) + + +def info(msg: str) -> None: + print("INFO: " + msg, flush=True) + + +def warning(msg: str) -> None: + _print_warning("WARN: " + msg) + global _warnings + _warnings += [msg] + + +def error(msg: str) -> None: + _print_error("ERROR: " + msg) + global _errors + _errors += [msg] + + +def exception(e: BaseException) -> None: + logging.exception(e) + global _errors + _errors += [str(e)] + + +def _log_cmd(msg: str) -> None: + if re.search(_error_pattern, msg): + if not re.search(_ignore_pattern, msg): + _print_error(msg) + global _errors + _errors += [msg] + return + elif re.search(_warning_pattern, msg): + if not re.search(_ignore_pattern, msg): + _print_warning(msg) + global _warnings + _warnings += [msg] + return + + print(msg.encode('ascii', errors='replace').decode('ascii'), flush=True) + + +# Command execution +class HiddenArgument: + def __init__(self, value: Union[str, pathlib.Path]): + self.value = value + + +CmdArgument = Union[str, pathlib.Path, HiddenArgument, Any] +CmdList = List[CmdArgument] +CmdSequence = Sequence[CmdArgument] +CmdFilterOutput = Optional[Callable[[str], Optional[str]]] +CmdEnvironment = Optional[Dict[str, str]] + + +def _prepare_call(cmd: CmdSequence, dry_run: bool = False) -> Sequence[Union[str, pathlib.Path]]: + real_cmd: List[Union[str, pathlib.Path]] = [] + log_cmd: List[str] = [] + + for arg in cmd: + if isinstance(arg, HiddenArgument): + real_cmd += [arg.value] + else: + log_cmd += [str(arg)] + real_cmd += [arg] + + if dry_run: + info(f"Dry run command in path [{os.getcwd()}]") + else: + info(f"Run command in path [{os.getcwd()}]") + _print_cmd(" ".join(log_cmd)) + + return real_cmd + + +def call( + cmd: CmdSequence, + env: CmdEnvironment = None, + exit_on_error: bool = True, + filter_output: CmdFilterOutput = None, + retry_count: int = 0, + retry_wait_time: float = 1.0, + dry_run: bool = False, +) -> int: + cmd = _prepare_call(cmd, dry_run) + if dry_run: + return 0 + + for try_count in range(0, retry_count + 1): + # Flush to ensure correct order output on Windows. + sys.stdout.flush() + sys.stderr.flush() + + proc = subprocess.Popen( + cmd, + env=env, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + encoding="utf-8", + errors="ignore", + ) + while True: + if not proc.stdout: + break + + line = proc.stdout.readline() + if line: + line_str = line.strip("\n\r") + if filter_output: + line_str_filter = filter_output(line_str) + else: + line_str_filter = line_str + if line_str: + _log_cmd(line_str) + else: + break + + proc.communicate() + + if proc.returncode == 0: + return 0 + + if try_count == retry_count: + if exit_on_error: + sys.exit(proc.returncode) + return proc.returncode + else: + warning("Command failed, retrying") + time.sleep(retry_wait_time) + + return -1 + + +def check_output(cmd: CmdSequence, exit_on_error: bool = True) -> str: + cmd = _prepare_call(cmd) + + # Flush to ensure correct order output on Windows. + sys.stdout.flush() + sys.stderr.flush() + + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True) + except subprocess.CalledProcessError as e: + if exit_on_error: + sys.exit(e.returncode) + output = "" + + return output.strip() + + +def call_pipenv( + cmd: CmdSequence, filter_output: CmdFilterOutput = None, dry_run: bool = False +) -> int: + cmd_prefix: CmdList = ["pipenv"] + return call(cmd_prefix + list(cmd), filter_output=filter_output, dry_run=dry_run) + + +def call_ssh(connect_id: str, cmd: CmdSequence, dry_run: bool = False) -> int: + ssh_cmd = [ + "ssh", + "-o", + "ConnectTimeout=20", + HiddenArgument(connect_id), + " ".join([str(arg) for arg in cmd]), + ] + return call(ssh_cmd, retry_count=3, dry_run=dry_run) + + +def rsync( + source_path: Union[pathlib.Path, str], + dest_path: Union[pathlib.Path, str], + exclude_paths: Sequence[str] = [], + include_paths: Sequence[str] = [], + change_modes: Sequence[str] = [], + change_owner: Optional[str] = None, + show_names: bool = False, + delete: bool = False, + delete_path_check: Optional[str] = None, + dry_run: bool = False, + port: int = 22, + retry_count: int = 3, +) -> int: + # Extra check on path, because delete is risky if pointed at a + # root folder that contains other data. + if delete: + if not delete_path_check: + raise Exception("Rsync: delete requires delete_path_check") + if str(dest_path).find(delete_path_check) == -1: + raise Exception("Rsync: remote path must contain '{delete_path_check}'") + + info_options = "progress0,flist0,name0,stats2" + if show_names: + info_options = "progress0,flist0,name1,stats2" + + cmd: List[Union[str, pathlib.Path, HiddenArgument]] = [ + "rsync", + # SSH options + "-e", + f"ssh -o ConnectTimeout=20 -p {port}", + # The -rlpgoDv options below are equivalent to --archive apart from updating + # the timestamp of the files on the receiving side. This should prevent them + # from getting marked for zfs-snapshots. + "--timeout=60", + "--checksum", + "-rlpgoDv", + "--partial", + ] + if change_owner: + cmd += [f"--chown={change_owner}"] + if delete: + cmd += ["--delete"] + # cmd += [f"--info={info_options}"] + cmd += [f"--include={item}" for item in include_paths] + cmd += [f"--exclude={item}" for item in exclude_paths] + cmd += [f"--chmod={item}" for item in change_modes] + + cmd += [source_path] + cmd += [HiddenArgument(dest_path)] + + return call(cmd, retry_count=retry_count, dry_run=dry_run) + + +def move(path_from: pathlib.Path, path_to: pathlib.Path, dry_run: bool = False) -> None: + if dry_run: + return + # str() works around typing bug in Python 3.6. + shutil.move(str(path_from), path_to) + + +def copy_dir(path_from: pathlib.Path, path_to: pathlib.Path, dry_run: bool = False) -> None: + if dry_run: + return + shutil.copytree(path_from, path_to) + + +def copy_file(path_from: pathlib.Path, path_to: pathlib.Path, dry_run: bool = False) -> None: + if dry_run: + return + shutil.copy2(path_from, path_to) + + +def remove_file( + path: pathlib.Path, retry_count: int = 3, retry_wait_time: float = 5.0, dry_run: bool = False +) -> None: + if not path.exists(): + return + if dry_run: + info(f"Removing {path} (dry run)") + return + + info(f"Removing {path}") + for try_count in range(0, retry_count): + try: + try: + if path.exists(): + path.unlink() + except FileNotFoundError: + pass + return + except: + time.sleep(retry_wait_time) + + # Not using missing_ok yet for Python3.6 compatibility. + try: + if path.exists(): + path.unlink() + except FileNotFoundError: + pass + + +# Retry several times by default, giving it a chance for possible antivirus to release +# a lock on files in the build folder. Happened for example with MSI files on Windows. +def remove_dir( + path: pathlib.Path, retry_count: int = 3, retry_wait_time: float = 5.0, dry_run: bool = False +) -> None: + if not path.exists(): + return + if dry_run: + info(f"Removing {path} (dry run)") + return + + info(f"Removing {path}") + for try_count in range(0, retry_count): + try: + if path.exists(): + shutil.rmtree(path) + return + except: + if platform.system().lower() == "windwos": + # XXX: Windows builder debug. + # Often the `build_package` is failed to be removed because + # of the "Access Denied" error on blender-windows64.msi. + # Run some investigation commands to see what is going on. + if path.name == "build_package": + info("Removal of package artifacts folder failed. Investigating...") + msi_path = ( + path / "_CPack_Packages" / "Windows" / "WIX" / "blender-windows64.msi" + ) + if msi_path.exists(): + info(f"Information about [{msi_path}]") + call(["handle64", msi_path], exit_on_error=False) + call( + ["pwsh", "-command", f"Get-Item {msi_path} | Format-List"], + exit_on_error=False, + ) + call( + ["pwsh", "-command", f"Get-Acl {msi_path} | Format-List"], + exit_on_error=False, + ) + else: + info(f"MSI package file [{msi_path}] does not exist") + + time.sleep(retry_wait_time) + + if path.exists(): + shutil.rmtree(path) + + +def is_tool(name: Union[str, pathlib.Path]) -> bool: + """Check whether `name` is on PATH and marked as executable.""" + return shutil.which(name) is not None + + +# Update source code from git repository. +def update_source( + app_org: str, + app_id: str, + code_path: pathlib.Path, + branch_id: str = "main", + patch_id: Optional[str] = None, + commit_id: Optional[str] = None, + update_submodules: bool = False, +) -> None: + repo_url = f"https://projects.blender.org/{app_org}/{app_id}.git" + + if not code_path.exists(): + # Clone new + info(f"Cloning {repo_url}") + call(["git", "clone", "--progress", repo_url, code_path]) + else: + for index_lock_path in code_path.rglob(".git/index.lock"): + warning("Removing git lock, probably left behind by killed git process") + remove_file(index_lock_path) + for index_lock_path in (code_path / ".git" / "modules").rglob("index.lock"): + warning("Removing submodule git lock, probably left behind by killed git process") + remove_file(index_lock_path) + + os.chdir(code_path) + + # Fix error: "fatal: bad object refs/remotes/origin/HEAD" + call(["git", "remote", "set-head", "origin", "--auto"]) + + # Change to new Gitea URL. + call(["git", "remote", "set-url", "origin", repo_url]) + call(["git", "submodule", "sync"]) + + # Fetch and clean + call(["git", "fetch", "origin", "--prune"]) + call(["git", "clean", "-f", "-d"]) + call(["git", "reset", "--hard"]) + + rebase_merge_path = code_path / ".git" / "rebase-merge" + if rebase_merge_path.exists(): + info(f"Path {rebase_merge_path} exists, removing !") + shutil.rmtree(rebase_merge_path) + + if patch_id: + # Pull request. + pull_request_id = patch_id + branch_name = f"PR{pull_request_id}" + + # Checkout pull request into PR123 branch. + call(["git", "checkout", "main"]) + call(["git", "fetch", "-f", "origin", f"pull/{pull_request_id}/head:{branch_name}"]) + call(["git", "checkout", branch_name]) + + if commit_id and (commit_id != "HEAD"): + call(["git", "reset", "--hard", commit_id]) + else: + # Branch. + call(["git", "checkout", branch_id]) + + if commit_id and (commit_id != "HEAD"): + call(["git", "reset", "--hard", commit_id]) + else: + call(["git", "reset", "--hard", "origin/" + branch_id]) + + if update_submodules: + call(["git", "submodule", "init"]) + + # Resolve potential issues with submodules even if other code + # is responsible for updating them. + call(["git", "submodule", "foreach", "git", "clean", "-f", "-d"]) + call(["git", "submodule", "foreach", "git", "reset", "--hard"]) + + if update_submodules: + call(["git", "submodule", "update"]) + + +# Workaroud missing type info in 3.8. +if sys.version_info >= (3, 9): + BuilderSteps = OrderedDict[str, Callable[[Any], None]] +else: + BuilderSteps = Any + + +class Builder: + def __init__(self, args: argparse.Namespace, app_org: str, app_id: str): + self.service_env_id = args.service_env_id + self.track_id = args.track_id + self.branch_id = args.branch_id + self.patch_id = args.patch_id + self.commit_id = args.commit_id + self.platform = platform.system().lower() + self.architecture = platform.machine().lower() + self.app_org = app_org + self.app_id = app_id + + if not self.branch_id: + branches_config = self.get_branches_config() + self.branch_id = branches_config.track_code_branches[self.track_id] + + self.tracks_root_path = self.get_worker_config().tracks_root_path + self.track_path = self.tracks_root_path / (self.app_id + "-" + self.track_id) + self.code_path = self.track_path / (self.app_id + ".git") + + info(f"Setting up builder paths from [{self.track_path}]") + + def setup_track_path(self) -> None: + # Create track directory if doesn't exist already. + os.makedirs(self.track_path, exist_ok=True) + os.chdir(self.track_path) + + # Clean up any existing pipenv files. + remove_file(self.track_path / "Pipfile") + remove_file(self.track_path / "Pipfile.lock") + remove_file(self.code_path / "Pipfile") + remove_file(self.code_path / "Pipfile.lock") + + def update_source(self, update_submodules: bool = False) -> None: + update_source( + self.app_org, + self.app_id, + self.code_path, + branch_id=self.branch_id, + patch_id=self.patch_id, + commit_id=self.commit_id, + update_submodules=update_submodules, + ) + + def run(self, step: str, steps: BuilderSteps) -> None: + try: + if step == "all": + for func in steps.values(): + func(self) + else: + steps[step](self) + except Exception as e: + exception(e) + sys.exit(1) + + def get_worker_config(self) -> Any: + import conf.worker + + return conf.worker.get_config(self.service_env_id) + + def get_branches_config(self) -> Any: + import conf.branches + + return conf.branches + + +def create_argument_parser(steps: BuilderSteps) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--service-env-id", type=str, required=False, default="LOCAL") + parser.add_argument("--track-id", default="vdev", type=str, required=False) + parser.add_argument("--branch-id", default="", type=str, required=False) + parser.add_argument("--patch-id", default="", type=str, required=False) + parser.add_argument("--commit-id", default="", type=str, required=False) + all_steps = list(steps.keys()) + ["all"] + parser.add_argument("step", choices=all_steps) + return parser