# SPDX-License-Identifier: GPL-2.0-or-later # SPDX-FileCopyrightText: 2011-2024 Blender Authors # import buildbot.plugins from buildbot.plugins import steps as plugins_steps from buildbot.plugins import schedulers as plugins_schedulers import conf.machines devops_git_root_path = "~/git" # Steps that run on the buildbot master. code_pipeline_master_step_names = [ "deduplicate-binaries", "purge-binaries", ] def fetch_property(props, key, default=None): value = default if key in props: value = props[key] return value def fetch_platform_architecture(props): platform_architectures = fetch_property(props, key="platform_architectures") # Find the platform arch for this builder buildername = fetch_property(props, key="buildername") builder_platform_architecture = "-".join(buildername.split("-")[-2:]) found_platform_architecture = None if platform_architectures: for platform_architecture in platform_architectures: if platform_architecture in builder_platform_architecture: found_platform_architecture = platform_architecture break if found_platform_architecture: return found_platform_architecture.split("-") else: return None, None def always_do_step(step): return True def needs_do_doc_pipeline_step(step): if "package" in step.name or "deliver" in step.name: return step.getProperty("needs_package_delivery") else: return True def create_worker_command(script, ENVIRONMENT, track_id, args): # This relative path assume were are in: # ~/.devops/services/buildbot-worker//build # There appears to be no way to expand a tilde here? # # This is assumed to run within the buildbot worker pipenv, # so the python command should match the python version and # available packages. cmd = [ "python", f"../../../../../git/blender-devops/buildbot/worker/{script}", "--track-id", track_id, "--service-env-id", ENVIRONMENT, ] return cmd + list(args) @buildbot.plugins.util.renderer def create_master_command_args( props, ENVIRONMENT, track_id, pipeline_type, step_name, single_platform ): build_configuration = fetch_property( props, key="build_configuration", default="release" ) python_module = fetch_property(props, key="python_module", default=False) args = [ "--pipeline-type", pipeline_type, "--build-configuration", build_configuration, ] if single_platform: # Archive binaries for a single architecture only? platform_id, architecture = fetch_platform_architecture(props) args += ["--platform-id", platform_id, "--architecture", architecture] if python_module: args += ["--python-module"] args += [step_name] # This relative path assume were are in: # ~/.devops/services/buildbot-master # There appears to be no way to expand a tilde here? # # This is assumed to run within the buildbot master pipenv, # so the python command should match the python version and # available packages. cmd = [ "python", "../../../git/blender-devops/buildbot/worker/archive.py", "--track-id", track_id, "--service-env-id", ENVIRONMENT, ] return cmd + list(args) @buildbot.plugins.util.renderer def create_pipeline_worker_command( props, ENVIRONMENT, track_id, script, step_name, variation_property, variation, builder_properties, ): args = [step_name] if variation_property: args += ["--" + variation_property.replace("_", "-"), variation] for builder_prop in builder_properties: if builder_prop.name in props: prop_value = props[builder_prop.name] else: prop_value = builder_prop.default argument_name = "--" + builder_prop.name.replace("_", "-") if isinstance(builder_prop, buildbot.plugins.util.BooleanParameter): if prop_value in ["true", True]: args += [argument_name] else: args += [argument_name, prop_value] if "revision" in props and props["revision"]: args += ["--commit-id", props["revision"]] return create_worker_command(script, ENVIRONMENT, track_id, args) def create_pipeline( ENVIRONMENT, artifact_id, script, steps, tracked_branch_ids, properties, codebase, worker_group_ids, variation_property=None, variations=[""], incremental_properties=None, nightly_properties=None, do_step_if=always_do_step, default_step_timeout_in_seconds=600, tree_stable_timer_in_seconds=180, hour=5, minute=0, ): builders = [] schedulers = [] platform_worker_names = conf.machines.fetch_platform_worker_names(ENVIRONMENT) local_worker_names = conf.machines.fetch_local_worker_names() needs_incremental_schedulers = ( incremental_properties is not None and ENVIRONMENT in ["PROD"] ) needs_nightly_schedulers = nightly_properties is not None and ENVIRONMENT in [ "PROD" ] track_ids = tracked_branch_ids.keys() print(f"*** Creating [{artifact_id}] pipeline") for track_id in track_ids: triggerable_scheduler_names = [] trigger_factory = buildbot.plugins.util.BuildFactory() for worker_group_id, variation in zip(worker_group_ids, variations): if variation: pipeline_builder_name = f"{track_id}-{artifact_id}-{variation}" else: pipeline_builder_name = f"{track_id}-{artifact_id}" pipeline_build_factory = buildbot.plugins.util.BuildFactory() print(f"Creating [{pipeline_builder_name}] pipeline steps") for step in steps: if callable(step): pipeline_build_factory.addStep(step()) continue step_command = create_pipeline_worker_command.withArgs( ENVIRONMENT, track_id, script, step, variation_property, variation, properties, ) pipeline_build_factory.addStep( plugins_steps.ShellCommand( name=step, logEnviron=True, haltOnFailure=True, timeout=default_step_timeout_in_seconds, description="running", descriptionDone="completed", command=step_command, doStepIf=do_step_if, ) ) # Create builder. pipeline_worker_names = platform_worker_names[worker_group_id] if pipeline_worker_names: builder_tags = pipeline_builder_name.split("-") builders += [ buildbot.plugins.util.BuilderConfig( name=pipeline_builder_name, workernames=pipeline_worker_names, tags=builder_tags, factory=pipeline_build_factory, ) ] scheduler_name = f"{pipeline_builder_name}-triggerable" triggerable_scheduler_names += [scheduler_name] schedulers += [ plugins_schedulers.Triggerable( name=scheduler_name, builderNames=[pipeline_builder_name] ) ] # Only create scheduler if we have something to to trigger if triggerable_scheduler_names: trigger_properties = {} for property in properties: if property != variation_property: trigger_properties[property.name] = buildbot.plugins.util.Property( property.name ) trigger_factory.addStep( plugins_steps.Trigger( schedulerNames=triggerable_scheduler_names, waitForFinish=True, updateSourceStamp=False, set_properties=trigger_properties, description="running", descriptionDone="completed", ) ) coordinator_builder_name = f"{track_id}-{artifact_id}-coordinator" builder_tags = coordinator_builder_name.split("-") builders += [ buildbot.plugins.util.BuilderConfig( name=coordinator_builder_name, workernames=local_worker_names, tags=builder_tags, factory=trigger_factory, ) ] coordinator_scheduler_name = f"{track_id}-{artifact_id}-coordinator-force" schedulers += [ plugins_schedulers.ForceScheduler( name=coordinator_scheduler_name, buttonName="Trigger build", builderNames=[coordinator_builder_name], codebases=[ buildbot.plugins.util.CodebaseParameter( codebase="", revision=None, hide=True ) ], properties=properties, ) ] if needs_incremental_schedulers and (track_id in track_ids): incremental_scheduler_name = ( f"{track_id}-{artifact_id}-coordinator-incremental" ) change_filter = buildbot.plugins.util.ChangeFilter( project=[codebase], branch=tracked_branch_ids[track_id] ) schedulers += [ plugins_schedulers.SingleBranchScheduler( name=incremental_scheduler_name, builderNames=[coordinator_builder_name], change_filter=change_filter, properties=incremental_properties, treeStableTimer=tree_stable_timer_in_seconds, ) ] if needs_nightly_schedulers and (track_id in track_ids): nightly_codebases = { codebase: { "repository": "", "branch": tracked_branch_ids[track_id], "revision": None, } } nightly_scheduler_name = f"{track_id}-{artifact_id}-coordinator-nightly" schedulers += [ plugins_schedulers.Nightly( name=nightly_scheduler_name, builderNames=[coordinator_builder_name], codebases=nightly_codebases, properties=nightly_properties, onlyIfChanged=False, hour=hour, minute=minute, ) ] return builders, schedulers