335 lines
11 KiB
Python
335 lines
11 KiB
Python
# SPDX-License-Identifier: GPL-2.0-or-later
|
|
# SPDX-FileCopyrightText: 2011-2024 Blender Authors
|
|
# <pep8 compliant>
|
|
|
|
import buildbot.plugins
|
|
|
|
from buildbot.plugins import steps as plugins_steps
|
|
from buildbot.plugins import schedulers as plugins_schedulers
|
|
|
|
import conf.machines
|
|
|
|
devops_git_root_path = "~/git"
|
|
|
|
# Steps that run on the buildbot master.
|
|
code_pipeline_master_step_names = [
|
|
"deduplicate-binaries",
|
|
"purge-binaries",
|
|
]
|
|
|
|
|
|
def fetch_property(props, key, default=None):
|
|
value = default
|
|
if key in props:
|
|
value = props[key]
|
|
|
|
return value
|
|
|
|
|
|
def fetch_platform_architecture(props):
|
|
platform_architectures = fetch_property(props, key="platform_architectures")
|
|
|
|
# Find the platform arch for this builder
|
|
buildername = fetch_property(props, key="buildername")
|
|
builder_platform_architecture = "-".join(buildername.split("-")[-2:])
|
|
|
|
found_platform_architecture = None
|
|
if platform_architectures:
|
|
for platform_architecture in platform_architectures:
|
|
if platform_architecture in builder_platform_architecture:
|
|
found_platform_architecture = platform_architecture
|
|
break
|
|
|
|
if found_platform_architecture:
|
|
return found_platform_architecture.split("-")
|
|
else:
|
|
return None, None
|
|
|
|
|
|
def always_do_step(step):
|
|
return True
|
|
|
|
|
|
def needs_do_doc_pipeline_step(step):
|
|
if "package" in step.name or "deliver" in step.name:
|
|
return step.getProperty("needs_package_delivery")
|
|
else:
|
|
return True
|
|
|
|
|
|
def create_worker_command(script, devops_env_id, track_id, args):
|
|
# This relative path assume were are in:
|
|
# ~/.devops/services/buildbot-worker/<builder-name>/build
|
|
# There appears to be no way to expand a tilde here?
|
|
#
|
|
# This is assumed to run within the buildbot worker pipenv,
|
|
# so the python command should match the python version and
|
|
# available packages.
|
|
cmd = [
|
|
"python",
|
|
f"../../../../../git/blender-devops/buildbot/worker/{script}",
|
|
"--track-id",
|
|
track_id,
|
|
"--service-env-id",
|
|
devops_env_id,
|
|
]
|
|
|
|
return cmd + list(args)
|
|
|
|
|
|
@buildbot.plugins.util.renderer
|
|
def create_master_command_args(
|
|
props, devops_env_id, track_id, pipeline_type, step_name, single_platform
|
|
):
|
|
build_configuration = fetch_property(props, key="build_configuration", default="release")
|
|
python_module = fetch_property(props, key="python_module", default=False)
|
|
python_module_string = "true" if python_module else "false"
|
|
|
|
args = [
|
|
"--pipeline-type",
|
|
pipeline_type,
|
|
"--build-configuration",
|
|
build_configuration,
|
|
]
|
|
|
|
if single_platform:
|
|
# Archive binaries for a single architecture only?
|
|
platform_id, architecture = fetch_platform_architecture(props)
|
|
args += ["--platform-id", platform_id, "--architecture", architecture]
|
|
|
|
if python_module:
|
|
args += ["--python-module"]
|
|
|
|
args += [step_name]
|
|
|
|
# This relative path assume were are in:
|
|
# ~/.devops/services/buildbot-master
|
|
# There appears to be no way to expand a tilde here?
|
|
#
|
|
# This is assumed to run within the buildbot master pipenv,
|
|
# so the python command should match the python version and
|
|
# available packages.
|
|
cmd = [
|
|
"python",
|
|
"../../../git/blender-devops/buildbot/worker/archive.py",
|
|
"--track-id",
|
|
track_id,
|
|
"--service-env-id",
|
|
devops_env_id,
|
|
]
|
|
|
|
return cmd + list(args)
|
|
|
|
|
|
@buildbot.plugins.util.renderer
|
|
def create_pipeline_worker_command(
|
|
props,
|
|
devops_env_id,
|
|
track_id,
|
|
script,
|
|
step_name,
|
|
variation_property,
|
|
variation,
|
|
builder_properties,
|
|
):
|
|
args = [step_name]
|
|
|
|
if variation_property:
|
|
args += ["--" + variation_property.replace("_", "-"), variation]
|
|
|
|
for builder_prop in builder_properties:
|
|
if builder_prop.name in props:
|
|
prop_value = props[builder_prop.name]
|
|
else:
|
|
prop_value = builder_prop.default
|
|
|
|
argument_name = "--" + builder_prop.name.replace("_", "-")
|
|
if isinstance(builder_prop, buildbot.plugins.util.BooleanParameter):
|
|
if prop_value in ["true", True]:
|
|
args += [argument_name]
|
|
else:
|
|
args += [argument_name, prop_value]
|
|
|
|
if "revision" in props and props["revision"]:
|
|
args += ["--commit-id", props["revision"]]
|
|
|
|
return create_worker_command(script, devops_env_id, track_id, args)
|
|
|
|
|
|
def create_pipeline(
|
|
devops_env_id,
|
|
artifact_id,
|
|
script,
|
|
steps,
|
|
tracked_branch_ids,
|
|
properties,
|
|
codebase,
|
|
worker_group_ids,
|
|
variation_property=None,
|
|
variations=[""],
|
|
incremental_properties=None,
|
|
nightly_properties=None,
|
|
do_step_if=always_do_step,
|
|
default_step_timeout_in_seconds=600,
|
|
tree_stable_timer_in_seconds=180,
|
|
hour=5,
|
|
minute=0,
|
|
):
|
|
builders = []
|
|
schedulers = []
|
|
|
|
platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id)
|
|
local_worker_names = conf.machines.fetch_local_worker_names()
|
|
|
|
needs_incremental_schedulers = incremental_properties is not None and devops_env_id in ["PROD"]
|
|
needs_nightly_schedulers = nightly_properties is not None and devops_env_id in ["PROD"]
|
|
track_ids = tracked_branch_ids.keys()
|
|
|
|
print(f"*** Creating [{artifact_id}] pipeline")
|
|
for track_id in track_ids:
|
|
triggerable_scheduler_names = []
|
|
trigger_factory = buildbot.plugins.util.BuildFactory()
|
|
|
|
for worker_group_id, variation in zip(worker_group_ids, variations):
|
|
if variation:
|
|
pipeline_builder_name = f"{track_id}-{artifact_id}-{variation}"
|
|
else:
|
|
pipeline_builder_name = f"{track_id}-{artifact_id}"
|
|
|
|
pipeline_build_factory = buildbot.plugins.util.BuildFactory()
|
|
|
|
print(f"Creating [{pipeline_builder_name}] pipeline steps")
|
|
for step in steps:
|
|
if callable(step):
|
|
pipeline_build_factory.addStep(step())
|
|
continue
|
|
|
|
step_command = create_pipeline_worker_command.withArgs(
|
|
devops_env_id,
|
|
track_id,
|
|
script,
|
|
step,
|
|
variation_property,
|
|
variation,
|
|
properties,
|
|
)
|
|
|
|
pipeline_build_factory.addStep(
|
|
plugins_steps.ShellCommand(
|
|
name=step,
|
|
logEnviron=True,
|
|
haltOnFailure=True,
|
|
timeout=default_step_timeout_in_seconds,
|
|
description="running",
|
|
descriptionDone="completed",
|
|
command=step_command,
|
|
doStepIf=do_step_if,
|
|
)
|
|
)
|
|
|
|
# Create builder.
|
|
pipeline_worker_names = platform_worker_names[worker_group_id]
|
|
if pipeline_worker_names:
|
|
builder_tags = pipeline_builder_name.split("-")
|
|
|
|
builders += [
|
|
buildbot.plugins.util.BuilderConfig(
|
|
name=pipeline_builder_name,
|
|
workernames=pipeline_worker_names,
|
|
tags=builder_tags,
|
|
factory=pipeline_build_factory,
|
|
)
|
|
]
|
|
|
|
scheduler_name = f"{pipeline_builder_name}-triggerable"
|
|
triggerable_scheduler_names += [scheduler_name]
|
|
|
|
schedulers += [
|
|
plugins_schedulers.Triggerable(
|
|
name=scheduler_name, builderNames=[pipeline_builder_name]
|
|
)
|
|
]
|
|
|
|
# Only create scheduler if we have something to to trigger
|
|
if triggerable_scheduler_names:
|
|
trigger_properties = {}
|
|
for property in properties:
|
|
if property != variation_property:
|
|
trigger_properties[property.name] = buildbot.plugins.util.Property(
|
|
property.name
|
|
)
|
|
|
|
trigger_factory.addStep(
|
|
plugins_steps.Trigger(
|
|
schedulerNames=triggerable_scheduler_names,
|
|
waitForFinish=True,
|
|
updateSourceStamp=False,
|
|
set_properties=trigger_properties,
|
|
description="running",
|
|
descriptionDone="completed",
|
|
)
|
|
)
|
|
|
|
coordinator_builder_name = f"{track_id}-{artifact_id}-coordinator"
|
|
builder_tags = coordinator_builder_name.split("-")
|
|
builders += [
|
|
buildbot.plugins.util.BuilderConfig(
|
|
name=coordinator_builder_name,
|
|
workernames=local_worker_names,
|
|
tags=builder_tags,
|
|
factory=trigger_factory,
|
|
)
|
|
]
|
|
|
|
coordinator_scheduler_name = f"{track_id}-{artifact_id}-coordinator-force"
|
|
schedulers += [
|
|
plugins_schedulers.ForceScheduler(
|
|
name=coordinator_scheduler_name,
|
|
buttonName="Trigger build",
|
|
builderNames=[coordinator_builder_name],
|
|
codebases=[
|
|
buildbot.plugins.util.CodebaseParameter(
|
|
codebase="", revision=None, hide=True
|
|
)
|
|
],
|
|
properties=properties,
|
|
)
|
|
]
|
|
|
|
if needs_incremental_schedulers and (track_id in track_ids):
|
|
incremental_scheduler_name = f"{track_id}-{artifact_id}-coordinator-incremental"
|
|
change_filter = buildbot.plugins.util.ChangeFilter(
|
|
project=[codebase], branch=tracked_branch_ids[track_id]
|
|
)
|
|
schedulers += [
|
|
plugins_schedulers.SingleBranchScheduler(
|
|
name=incremental_scheduler_name,
|
|
builderNames=[coordinator_builder_name],
|
|
change_filter=change_filter,
|
|
properties=incremental_properties,
|
|
treeStableTimer=tree_stable_timer_in_seconds,
|
|
)
|
|
]
|
|
|
|
if needs_nightly_schedulers and (track_id in track_ids):
|
|
nightly_codebases = {
|
|
codebase: {
|
|
"repository": "",
|
|
"branch": tracked_branch_ids[track_id],
|
|
"revision": None,
|
|
}
|
|
}
|
|
nightly_scheduler_name = f"{track_id}-{artifact_id}-coordinator-nightly"
|
|
schedulers += [
|
|
plugins_schedulers.Nightly(
|
|
name=nightly_scheduler_name,
|
|
builderNames=[coordinator_builder_name],
|
|
codebases=nightly_codebases,
|
|
properties=nightly_properties,
|
|
onlyIfChanged=False,
|
|
hour=hour,
|
|
minute=minute,
|
|
)
|
|
]
|
|
|
|
return builders, schedulers
|