builder.braak.pro/buildbot/config/pipeline/code_store.py

# SPDX-License-Identifier: GPL-2.0-or-later
# SPDX-FileCopyrightText: 2011-2024 Blender Authors
# <pep8 compliant>

# Builders for releasing Blender to stores.

import pathlib

import buildbot.plugins

from buildbot.plugins import steps as plugins_steps
from buildbot.plugins import schedulers as plugins_schedulers

import conf.branches
import conf.worker
import pipeline.common

# Timeouts.
default_step_timeout_in_seconds = 60 * 60

# Tracks.
track_ids = conf.branches.code_store_track_ids
tracked_branch_ids = {}
for track_id in track_ids:
    tracked_branch_ids[track_id] = conf.branches.code_tracked_branch_ids[track_id]

# Properties.
scheduler_properties = [
    buildbot.plugins.util.ChoiceStringParameter(
        name="store_id",
        label="Store:",
        required=True,
        choices=["snap", "steam", "windows"],
        multiple=True,
        strict=True,
        default=["snap", "steam", "windows"],
    ),
]


def create_deliver_binaries_windows_step(worker_config, track_id, pipeline_type):
    # Create step for uploading msix to download.blender.org.
    file_size_in_mb = 500 * 1024 * 1024
    worker_source_path = pathlib.Path(
        f"../../../../git/blender-{track_id}/build_package"
    )
    master_dest_path = pathlib.Path(
        f"{worker_config.buildbot_download_folder}/{pipeline_type}"
    ).expanduser()

    return plugins_steps.MultipleFileUpload(
        name="deliver-binaries",
        maxsize=file_size_in_mb,
        workdir=f"{worker_source_path}",
        glob=True,
        workersrcs=["*.msix*"],
        masterdest=f"{master_dest_path}",
        mode=0o644,
        url=None,
        description="running",
        descriptionDone="completed",
    )


def populate(environment):
    builders = []
    schedulers = []

    platform_worker_names = conf.machines.fetch_platform_worker_names(environment)
    local_worker_names = conf.machines.fetch_local_worker_names()

    worker_config = conf.worker.get_config(environment)

    needs_nightly_schedulers = environment == "PROD"

    pipeline_type = "daily"

    store_ids = ["steam", "snap", "windows"]

    print("*** Creating [code] [store] pipeline")
    for track_id in track_ids:
        triggerable_scheduler_names = []
        trigger_factory = buildbot.plugins.util.BuildFactory()

        for store_id in store_ids:
            # Create build steps.
            pipeline_build_factory = buildbot.plugins.util.BuildFactory()
            step_names = [
                "configure-machine",
                "update-code",
                "pull-artifacts",
                "package",
            ]

            if store_id == "windows":
                step_names += ["deliver-binaries"]
            else:
                step_names += ["deliver"]

            step_names += ["clean"]

            print(f"Creating [{track_id}] [code] [store] [{store_id}] pipeline steps")
            for step_name in step_names:
                if step_name == "deliver-binaries":
                    step = create_deliver_binaries_windows_step(
                        worker_config, track_id, pipeline_type
                    )
                else:
                    args = ["--store-id", store_id, step_name]
                    step_command = pipeline.common.create_worker_command(
                        "code_store.py", environment, track_id, args
                    )

                    step = plugins_steps.ShellCommand(
                        name=step_name,
                        logEnviron=True,
                        haltOnFailure=True,
                        timeout=default_step_timeout_in_seconds,
                        description="running",
                        descriptionDone="completed",
                        command=step_command,
                    )

                pipeline_build_factory.addStep(step)

            for master_step_name in pipeline.common.code_pipeline_master_step_names:
                master_step_command = (
                    pipeline.common.create_master_command_args.withArgs(
                        environment,
                        track_id,
                        pipeline_type,
                        master_step_name,
                        single_platform=False,
                    )
                )

                # Master to archive and purge builds
                pipeline_build_factory.addStep(
                    plugins_steps.MasterShellCommand(
                        name=master_step_name,
                        logEnviron=False,
                        command=master_step_command,
                        description="running",
                        descriptionDone="completed",
                    )
                )

            # Create builders.
            worker_group_id = (
                f"windows-amd64-store-{store_id}"
                if store_id == "windows"
                else f"linux-x86_64-store-{store_id}"
            )
            pipeline_worker_names = platform_worker_names[worker_group_id]
            if pipeline_worker_names:
                pipeline_builder_name = f"{track_id}-code-store-{store_id}"

                builder_tags = pipeline_builder_name.split("-")

                builders += [
                    buildbot.plugins.util.BuilderConfig(
                        name=pipeline_builder_name,
                        workernames=pipeline_worker_names,
                        tags=builder_tags,
                        factory=pipeline_build_factory,
                    )
                ]

                scheduler_name = f"{track_id}-code-store-{store_id}-triggerable"
                triggerable_scheduler_names += [scheduler_name]

                schedulers += [
                    plugins_schedulers.Triggerable(
                        name=scheduler_name, builderNames=[pipeline_builder_name]
                    )
                ]

        # Create coordinator.
        if triggerable_scheduler_names:
            trigger_properties = {}
            trigger_factory.addStep(
                plugins_steps.Trigger(
                    schedulerNames=triggerable_scheduler_names,
                    waitForFinish=True,
                    updateSourceStamp=False,
                    set_properties=trigger_properties,
                    description="running",
                    descriptionDone="completed",
                )
            )

            coordinator_builder_name = f"{track_id}-code-store-coordinator"
            builder_tags = coordinator_builder_name.split("-")

            builders += [
                buildbot.plugins.util.BuilderConfig(
                    name=coordinator_builder_name,
                    workernames=local_worker_names,
                    tags=builder_tags,
                    factory=trigger_factory,
                )
            ]

            coordinator_scheduler_name = f"{track_id}-code-store-coordinator-force"
            schedulers += [
                plugins_schedulers.ForceScheduler(
                    name=coordinator_scheduler_name,
                    buttonName="Trigger store build",
                    builderNames=[coordinator_builder_name],
                    codebases=[
                        buildbot.plugins.util.CodebaseParameter(
                            codebase="", revision=None, hide=True
                        )
                    ],
                    properties=scheduler_properties,
                )
            ]

            if needs_nightly_schedulers and (track_id in track_ids):
                nightly_scheduler_name = f"{track_id}-code-store-coordinator-nightly"
                nightly_properties = {
                    "revision": "HEAD",
                }
                nightly_codebases = {
                    "blender.git": {
                        "repository": "",
                        "branch": tracked_branch_ids[track_id],
                        "revision": None,
                    }
                }
                schedulers += [
                    plugins_schedulers.Nightly(
                        name=nightly_scheduler_name,
                        builderNames=[coordinator_builder_name],
                        codebases=nightly_codebases,
                        properties=nightly_properties,
                        onlyIfChanged=False,
                        hour=5,
                        minute=30,
                    )
                ]

    return builders, schedulers