Formatting and linting fixes
This commit is contained in:
Bart van der Braak
parent
0a1454d250
commit
d6bce1b39d
32 changed files with 675 additions and 268 deletions
|
|
@ -92,7 +92,9 @@ code_python_module_skip_test_names = ["sign-code-binaries"]
|
|||
code_tracked_branch_ids = conf.branches.code_tracked_branch_ids
|
||||
code_track_ids = list(code_tracked_branch_ids.keys())
|
||||
code_all_platform_architectures = conf.branches.code_all_platform_architectures
|
||||
code_official_platform_architectures = conf.branches.code_official_platform_architectures
|
||||
code_official_platform_architectures = (
|
||||
conf.branches.code_official_platform_architectures
|
||||
)
|
||||
|
||||
code_track_pipeline_types = {}
|
||||
track_properties = {}
|
||||
|
|
@ -211,7 +213,12 @@ scheduler_properties_patch = [
|
|||
default=False,
|
||||
),
|
||||
buildbot.plugins.util.StringParameter(
|
||||
name="pull_revision", label="Pull Revision:", required=False, hide=True, size=80, default=""
|
||||
name="pull_revision",
|
||||
label="Pull Revision:",
|
||||
required=False,
|
||||
hide=True,
|
||||
size=80,
|
||||
default="",
|
||||
),
|
||||
]
|
||||
|
||||
|
|
@ -225,12 +232,20 @@ scheduler_properties = {
|
|||
|
||||
|
||||
@buildbot.plugins.util.renderer
|
||||
def create_code_worker_command_args(props, devops_env_id, track_id, pipeline_type, step_name):
|
||||
def create_code_worker_command_args(
|
||||
props, devops_env_id, track_id, pipeline_type, step_name
|
||||
):
|
||||
commit_id = pipeline.common.fetch_property(props, key="revision", default="HEAD")
|
||||
patch_id = pipeline.common.fetch_property(props, key="patch_id", default="")
|
||||
override_branch_id = pipeline.common.fetch_property(props, key="override_branch_id", default="")
|
||||
python_module = pipeline.common.fetch_property(props, key="python_module", default=False)
|
||||
needs_gpu_tests = pipeline.common.fetch_property(props, key="needs_gpu_tests", default=False)
|
||||
override_branch_id = pipeline.common.fetch_property(
|
||||
props, key="override_branch_id", default=""
|
||||
)
|
||||
python_module = pipeline.common.fetch_property(
|
||||
props, key="python_module", default=False
|
||||
)
|
||||
needs_gpu_tests = pipeline.common.fetch_property(
|
||||
props, key="needs_gpu_tests", default=False
|
||||
)
|
||||
needs_gpu_binaries = pipeline.common.fetch_property(
|
||||
props, key="needs_gpu_binaries", default=False
|
||||
)
|
||||
|
|
@ -279,11 +294,12 @@ def create_code_worker_command_args(props, devops_env_id, track_id, pipeline_typ
|
|||
|
||||
args += [step_name]
|
||||
|
||||
return pipeline.common.create_worker_command("code.py", devops_env_id, track_id, args)
|
||||
return pipeline.common.create_worker_command(
|
||||
"code.py", devops_env_id, track_id, args
|
||||
)
|
||||
|
||||
|
||||
def needs_do_code_pipeline_step(step):
|
||||
build = step.build
|
||||
# Use this to test master steps only, otherwise we be waiting for 30 minutes
|
||||
needs_master_steps_only = False
|
||||
|
||||
|
|
@ -291,9 +307,7 @@ def needs_do_code_pipeline_step(step):
|
|||
is_master_step = step.name in pipeline.common.code_pipeline_master_step_names
|
||||
return is_master_step
|
||||
|
||||
worker = step.worker
|
||||
worker_name = step.getWorkerName()
|
||||
worker_system = worker.worker_system
|
||||
step.getWorkerName()
|
||||
|
||||
is_package_delivery_step = (step.name in code_delivery_step_names) or (
|
||||
step.name in pipeline.common.code_pipeline_master_step_names
|
||||
|
|
@ -337,7 +351,9 @@ class LinkMultipleFileUpload(plugins_steps.MultipleFileUpload):
|
|||
|
||||
def create_deliver_code_binaries_step(worker_config, track_id, pipeline_type):
|
||||
file_size_in_mb = 500 * 1024 * 1024
|
||||
worker_source_path = pathlib.Path(f"../../../../git/blender-{track_id}/build_package")
|
||||
worker_source_path = pathlib.Path(
|
||||
f"../../../../git/blender-{track_id}/build_package"
|
||||
)
|
||||
master_dest_path = pathlib.Path(
|
||||
f"{worker_config.buildbot_download_folder}/{pipeline_type}"
|
||||
).expanduser()
|
||||
|
|
@ -359,7 +375,9 @@ def create_deliver_code_binaries_step(worker_config, track_id, pipeline_type):
|
|||
|
||||
def create_deliver_test_results_step(worker_config, track_id, pipeline_type):
|
||||
file_size_in_mb = 500 * 1024 * 1024
|
||||
worker_source_path = pathlib.Path(f"../../../../git/blender-{track_id}/build_package")
|
||||
worker_source_path = pathlib.Path(
|
||||
f"../../../../git/blender-{track_id}/build_package"
|
||||
)
|
||||
master_dest_path = pathlib.Path(
|
||||
f"{worker_config.buildbot_download_folder}/{pipeline_type}"
|
||||
).expanduser()
|
||||
|
|
@ -456,9 +474,13 @@ def populate(devops_env_id):
|
|||
print(f"Creating [{track_id}] [code] [{pipeline_type}] pipeline steps")
|
||||
for step_name in step_names:
|
||||
if step_name == "deliver-code-binaries":
|
||||
step = create_deliver_code_binaries_step(worker_config, track_id, pipeline_type)
|
||||
step = create_deliver_code_binaries_step(
|
||||
worker_config, track_id, pipeline_type
|
||||
)
|
||||
elif step_name == "deliver-test-results":
|
||||
step = create_deliver_test_results_step(worker_config, track_id, pipeline_type)
|
||||
step = create_deliver_test_results_step(
|
||||
worker_config, track_id, pipeline_type
|
||||
)
|
||||
else:
|
||||
needs_halt_on_failure = True
|
||||
if step_name in code_pipeline_test_step_names:
|
||||
|
|
@ -488,8 +510,14 @@ def populate(devops_env_id):
|
|||
pipeline_build_factory.addStep(step)
|
||||
|
||||
for master_step_name in pipeline.common.code_pipeline_master_step_names:
|
||||
master_step_command = pipeline.common.create_master_command_args.withArgs(
|
||||
devops_env_id, track_id, pipeline_type, master_step_name, single_platform=True
|
||||
master_step_command = (
|
||||
pipeline.common.create_master_command_args.withArgs(
|
||||
devops_env_id,
|
||||
track_id,
|
||||
pipeline_type,
|
||||
master_step_name,
|
||||
single_platform=True,
|
||||
)
|
||||
)
|
||||
|
||||
# Master to archive and purge builds
|
||||
|
|
@ -528,7 +556,9 @@ def populate(devops_env_id):
|
|||
|
||||
# Create builders.
|
||||
for platform_architecture in code_all_platform_architectures[track_id]:
|
||||
print(f"Creating [{track_id}] [{pipeline_type}] [{platform_architecture}] builders")
|
||||
print(
|
||||
f"Creating [{track_id}] [{pipeline_type}] [{platform_architecture}] builders"
|
||||
)
|
||||
|
||||
worker_group_id = f"{platform_architecture}-code"
|
||||
worker_group_id_gpu = f"{platform_architecture}-code-gpu"
|
||||
|
|
@ -544,30 +574,36 @@ def populate(devops_env_id):
|
|||
|
||||
# Assigning different workers for different tracks, specifically Linux builders.
|
||||
suitable_pipeline_worker_names = pipeline_worker_names
|
||||
if platform_architecture == "linux-x86_64" and devops_env_id != "LOCAL":
|
||||
if (
|
||||
platform_architecture == "linux-x86_64"
|
||||
and devops_env_id != "LOCAL"
|
||||
):
|
||||
selector = "rocky"
|
||||
suitable_pipeline_worker_names = [
|
||||
worker for worker in pipeline_worker_names if selector in worker
|
||||
worker
|
||||
for worker in pipeline_worker_names
|
||||
if selector in worker
|
||||
]
|
||||
|
||||
builders += [
|
||||
buildbot.plugins.util.BuilderConfig(
|
||||
name=pipeline_builder_name,
|
||||
workernames=suitable_pipeline_worker_names,
|
||||
nextWorker=partial(next_worker_code, pipeline_worker_names_gpu),
|
||||
nextWorker=partial(
|
||||
next_worker_code, pipeline_worker_names_gpu
|
||||
),
|
||||
tags=pipeline_builder_tags,
|
||||
factory=pipeline_build_factory,
|
||||
)
|
||||
]
|
||||
|
||||
pipeline_scheduler_name = (
|
||||
f"{track_id}-code-{pipeline_type}-{platform_architecture}-triggerable"
|
||||
)
|
||||
pipeline_scheduler_name = f"{track_id}-code-{pipeline_type}-{platform_architecture}-triggerable"
|
||||
triggerable_scheduler_names += [pipeline_scheduler_name]
|
||||
|
||||
schedulers += [
|
||||
plugins_schedulers.Triggerable(
|
||||
name=pipeline_scheduler_name, builderNames=[pipeline_builder_name]
|
||||
name=pipeline_scheduler_name,
|
||||
builderNames=[pipeline_builder_name],
|
||||
)
|
||||
]
|
||||
|
||||
|
|
@ -590,12 +626,15 @@ def populate(devops_env_id):
|
|||
)
|
||||
]
|
||||
|
||||
pipeline_scheduler_name = f"{track_id}-code-{pipeline_type}-lint-triggerable"
|
||||
pipeline_scheduler_name = (
|
||||
f"{track_id}-code-{pipeline_type}-lint-triggerable"
|
||||
)
|
||||
triggerable_scheduler_names += [pipeline_scheduler_name]
|
||||
|
||||
schedulers += [
|
||||
plugins_schedulers.Triggerable(
|
||||
name=pipeline_scheduler_name, builderNames=[pipeline_builder_name]
|
||||
name=pipeline_scheduler_name,
|
||||
builderNames=[pipeline_builder_name],
|
||||
)
|
||||
]
|
||||
|
||||
|
|
@ -603,39 +642,55 @@ def populate(devops_env_id):
|
|||
if triggerable_scheduler_names:
|
||||
trigger_properties = {
|
||||
"python_module": buildbot.plugins.util.Property("python_module"),
|
||||
"needs_full_clean": buildbot.plugins.util.Property("needs_full_clean"),
|
||||
"needs_full_clean": buildbot.plugins.util.Property(
|
||||
"needs_full_clean"
|
||||
),
|
||||
"needs_package_delivery": buildbot.plugins.util.Property(
|
||||
"needs_package_delivery"
|
||||
),
|
||||
"needs_gpu_binaries": buildbot.plugins.util.Property("needs_gpu_binaries"),
|
||||
"needs_gpu_tests": buildbot.plugins.util.Property("needs_gpu_tests"),
|
||||
"needs_skip_tests": buildbot.plugins.util.Property("needs_skip_tests"),
|
||||
"needs_gpu_binaries": buildbot.plugins.util.Property(
|
||||
"needs_gpu_binaries"
|
||||
),
|
||||
"needs_gpu_tests": buildbot.plugins.util.Property(
|
||||
"needs_gpu_tests"
|
||||
),
|
||||
"needs_skip_tests": buildbot.plugins.util.Property(
|
||||
"needs_skip_tests"
|
||||
),
|
||||
"platform_architectures": buildbot.plugins.util.Property(
|
||||
"platform_architectures"
|
||||
),
|
||||
}
|
||||
if pipeline_type == "patch":
|
||||
trigger_properties["patch_id"] = buildbot.plugins.util.Property("patch_id")
|
||||
trigger_properties["revision"] = buildbot.plugins.util.Property("revision")
|
||||
trigger_properties["build_configuration"] = buildbot.plugins.util.Property(
|
||||
"build_configuration"
|
||||
trigger_properties["patch_id"] = buildbot.plugins.util.Property(
|
||||
"patch_id"
|
||||
)
|
||||
trigger_properties["revision"] = buildbot.plugins.util.Property(
|
||||
"revision"
|
||||
)
|
||||
trigger_properties["build_configuration"] = (
|
||||
buildbot.plugins.util.Property("build_configuration")
|
||||
)
|
||||
trigger_factory.addStep(
|
||||
plugins_steps.SetProperties(
|
||||
name="get-revision", properties=gitea.blender.get_patch_revision
|
||||
name="get-revision",
|
||||
properties=gitea.blender.get_patch_revision,
|
||||
)
|
||||
)
|
||||
elif pipeline_type == "experimental":
|
||||
trigger_properties["override_branch_id"] = buildbot.plugins.util.Property(
|
||||
"override_branch_id"
|
||||
trigger_properties["override_branch_id"] = (
|
||||
buildbot.plugins.util.Property("override_branch_id")
|
||||
)
|
||||
trigger_properties["revision"] = buildbot.plugins.util.Property("revision")
|
||||
trigger_properties["build_configuration"] = buildbot.plugins.util.Property(
|
||||
"build_configuration"
|
||||
trigger_properties["revision"] = buildbot.plugins.util.Property(
|
||||
"revision"
|
||||
)
|
||||
trigger_properties["build_configuration"] = (
|
||||
buildbot.plugins.util.Property("build_configuration")
|
||||
)
|
||||
trigger_factory.addStep(
|
||||
plugins_steps.SetProperties(
|
||||
name="get-revision", properties=gitea.blender.get_branch_revision
|
||||
name="get-revision",
|
||||
properties=gitea.blender.get_branch_revision,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -650,7 +705,9 @@ def populate(devops_env_id):
|
|||
)
|
||||
)
|
||||
|
||||
coordinator_builder_name = f"{track_id}-code-{pipeline_type}-coordinator"
|
||||
coordinator_builder_name = (
|
||||
f"{track_id}-code-{pipeline_type}-coordinator"
|
||||
)
|
||||
builder_tags = coordinator_builder_name.split("-")
|
||||
|
||||
builders += [
|
||||
|
|
@ -662,7 +719,9 @@ def populate(devops_env_id):
|
|||
)
|
||||
]
|
||||
|
||||
coordinator_scheduler_name = f"{track_id}-code-{pipeline_type}-coordinator-force"
|
||||
coordinator_scheduler_name = (
|
||||
f"{track_id}-code-{pipeline_type}-coordinator-force"
|
||||
)
|
||||
schedulers += [
|
||||
plugins_schedulers.ForceScheduler(
|
||||
name=coordinator_scheduler_name,
|
||||
|
|
@ -701,7 +760,8 @@ def populate(devops_env_id):
|
|||
}
|
||||
|
||||
change_filter = buildbot.plugins.util.ChangeFilter(
|
||||
project=["blender.git"], branch=code_tracked_branch_ids[track_id]
|
||||
project=["blender.git"],
|
||||
branch=code_tracked_branch_ids[track_id],
|
||||
)
|
||||
schedulers += [
|
||||
plugins_schedulers.SingleBranchScheduler(
|
||||
|
|
@ -724,7 +784,9 @@ def populate(devops_env_id):
|
|||
"needs_package_delivery": True,
|
||||
"needs_gpu_binaries": True,
|
||||
"build_configuration": "release",
|
||||
"platform_architectures": code_all_platform_architectures[track_id],
|
||||
"platform_architectures": code_all_platform_architectures[
|
||||
track_id
|
||||
],
|
||||
}
|
||||
nightly_codebases = {
|
||||
"blender.git": {
|
||||
|
|
|
|||
|
|
@ -41,7 +41,9 @@ scheduler_properties = [
|
|||
def create_deliver_binaries_windows_step(worker_config, track_id, pipeline_type):
|
||||
# Create step for uploading msix to download.blender.org.
|
||||
file_size_in_mb = 500 * 1024 * 1024
|
||||
worker_source_path = pathlib.Path(f"../../../../git/blender-{track_id}/build_package")
|
||||
worker_source_path = pathlib.Path(
|
||||
f"../../../../git/blender-{track_id}/build_package"
|
||||
)
|
||||
master_dest_path = pathlib.Path(
|
||||
f"{worker_config.buildbot_download_folder}/{pipeline_type}"
|
||||
).expanduser()
|
||||
|
|
@ -122,8 +124,14 @@ def populate(devops_env_id):
|
|||
pipeline_build_factory.addStep(step)
|
||||
|
||||
for master_step_name in pipeline.common.code_pipeline_master_step_names:
|
||||
master_step_command = pipeline.common.create_master_command_args.withArgs(
|
||||
devops_env_id, track_id, pipeline_type, master_step_name, single_platform=False
|
||||
master_step_command = (
|
||||
pipeline.common.create_master_command_args.withArgs(
|
||||
devops_env_id,
|
||||
track_id,
|
||||
pipeline_type,
|
||||
master_step_name,
|
||||
single_platform=False,
|
||||
)
|
||||
)
|
||||
|
||||
# Master to archive and purge builds
|
||||
|
|
|
|||
|
|
@ -81,9 +81,10 @@ def create_worker_command(script, devops_env_id, track_id, args):
|
|||
def create_master_command_args(
|
||||
props, devops_env_id, track_id, pipeline_type, step_name, single_platform
|
||||
):
|
||||
build_configuration = fetch_property(props, key="build_configuration", default="release")
|
||||
build_configuration = fetch_property(
|
||||
props, key="build_configuration", default="release"
|
||||
)
|
||||
python_module = fetch_property(props, key="python_module", default=False)
|
||||
python_module_string = "true" if python_module else "false"
|
||||
|
||||
args = [
|
||||
"--pipeline-type",
|
||||
|
|
@ -181,8 +182,12 @@ def create_pipeline(
|
|||
platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id)
|
||||
local_worker_names = conf.machines.fetch_local_worker_names()
|
||||
|
||||
needs_incremental_schedulers = incremental_properties is not None and devops_env_id in ["PROD"]
|
||||
needs_nightly_schedulers = nightly_properties is not None and devops_env_id in ["PROD"]
|
||||
needs_incremental_schedulers = (
|
||||
incremental_properties is not None and devops_env_id in ["PROD"]
|
||||
)
|
||||
needs_nightly_schedulers = nightly_properties is not None and devops_env_id in [
|
||||
"PROD"
|
||||
]
|
||||
track_ids = tracked_branch_ids.keys()
|
||||
|
||||
print(f"*** Creating [{artifact_id}] pipeline")
|
||||
|
|
@ -297,7 +302,9 @@ def create_pipeline(
|
|||
]
|
||||
|
||||
if needs_incremental_schedulers and (track_id in track_ids):
|
||||
incremental_scheduler_name = f"{track_id}-{artifact_id}-coordinator-incremental"
|
||||
incremental_scheduler_name = (
|
||||
f"{track_id}-{artifact_id}-coordinator-incremental"
|
||||
)
|
||||
change_filter = buildbot.plugins.util.ChangeFilter(
|
||||
project=[codebase], branch=tracked_branch_ids[track_id]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -37,7 +37,10 @@ def populate(devops_env_id):
|
|||
["linux-x86_64-general", "linux-x86_64-general"],
|
||||
variation_property="doc_format",
|
||||
variations=["html", "epub"],
|
||||
incremental_properties={"needs_package_delivery": True, "needs_all_locales": False},
|
||||
incremental_properties={
|
||||
"needs_package_delivery": True,
|
||||
"needs_all_locales": False,
|
||||
},
|
||||
nightly_properties={"needs_package_delivery": True, "needs_all_locales": True},
|
||||
tree_stable_timer_in_seconds=15 * 60,
|
||||
do_step_if=pipeline.common.needs_do_doc_pipeline_step,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue