Add Make and Pipelines for checks

.env.example

@@ -0,0 +1,3 @@
+SERVICE_USER_POSTGRESQL=buildbot
+SERVICE_PASSWORD_POSTGRESQL=changeme!
+BUILDBOT_CONFIG_URL=''

.forgejo/workflows/check.yml

@@ -0,0 +1,14 @@
+name: Run checks
+on: [ push, pull_request ]
+jobs:
+  checks:
+    runs-on: docker
+    env:
+      DIRECTORY: config
+    steps:
+      - uses: actions/checkout@v4
+      - run: cd ${{ env.DIRECTORY }}
+      - run: pip install -r requirements-dev.txt
+      - run: ruff check
+      - run: ruff format
+      - run: myp

.forgejo/workflows/release.yml

@@ -1,3 +1,4 @@
+name: Release and deploy
 on:
   push:
     branches:
@@ -6,18 +7,19 @@ jobs:
   artifact:
     runs-on: docker
     env:
-      ARCHIVE_FOLDER: config
+      DIRECTORY: config
+      PACKAGE_NAME: builder.braak.pro
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Archive
-        run: "tar -czvf ${{ env.ARCHIVE_FOLDER }}.tar.gz ${{ env.ARCHIVE_FOLDER }}"
+        run: "tar -czvf ${{ env.DIRECTORY }}.tar.gz ${{ env.DIRECTORY }}"
       - name: Upload
         run: |
           curl \
             --user "${{ env.GITHUB_REPOSITORY_OWNER }}:${{ secrets.ACCESS_TOKEN }}" \
-            --upload-file "${{ env.ARCHIVE_FOLDER }}.tar.gz" \
-            "${{ env.GITHUB_SERVER_URL }}/api/packages/${{ env.GITHUB_REPOSITORY_OWNER }}/generic/builder.braak.pro/${{ env.GITHUB_REF_NAME }}/${{ env.ARCHIVE_FOLDER }}.tar.gz"
+            --upload-file "${{ env.DIRECTORY }}.tar.gz" \
+            "${{ env.GITHUB_SERVER_URL }}/api/packages/${{ env.GITHUB_REPOSITORY_OWNER }}/generic/${{ env.PACKAGE_NAME }}/${{ env.GITHUB_REF_NAME }}/${{ env.ARCHIVE_FOLDER }}.tar.gz"
       - name: Deploy
         run: |
           curl -X "POST" \

.gitignore

@@ -0,0 +1,2 @@
+.venv
+.env

Makefile

@@ -0,0 +1,16 @@
+
+.PHONY: venv setup check
+help: ## Display this help message
+	@echo "Usage:"
+	@echo "  make <target>"
+	@echo ""
+	@echo "Targets:"
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  %-20s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+setup: ## Create Python virtualenv and install dependencies
+	@if [ ! -f .env ]; then cp .env.example .env; fi
+	@if [ ! -d .venv ]; then python3 -m venv .venv; fi
+	@if [ -d .venv ]; then . .venv/bin/activate && pip3 install -r requirements.txt; fi
+
+check: ## Run linter, formatter and typechecks
+	ruff check && ruff format && mypy config/master.cfg

config/master.cfg

@@ -1,116 +1,11 @@
 # -*- python -*-
 # ex: set filetype=python:
-
+import importlib
 import os
+import sys
 
-from buildbot.plugins import *
+sys.path.insert(0, os.path.expanduser("~/git/blender-devops/buildbot"))
 
-# This is a sample buildmaster config file. It must be installed as
-# 'master.cfg' in your buildmaster's base directory.
-
-# This is the dictionary that the buildmaster pays attention to. We also use
-# a shorter alias to save typing.
-c = BuildmasterConfig = {}
-
-####### WORKERS
-
-# The 'workers' list defines the set of recognized workers. Each element is
-# a Worker object, specifying a unique worker name and password.  The same
-# worker name and password must be configured on the worker.
-
-c['workers'] = [worker.Worker("example-worker", 'pass')]
-
-if 'BUILDBOT_MQ_URL' in os.environ:
-    c['mq'] = {
-        'type' : 'wamp',
-        'router_url': os.environ['BUILDBOT_MQ_URL'],
-        'realm': os.environ.get('BUILDBOT_MQ_REALM', 'buildbot').decode('utf-8'),
-        'debug' : 'BUILDBOT_MQ_DEBUG' in os.environ,
-        'debug_websockets' : 'BUILDBOT_MQ_DEBUG' in os.environ,
-        'debug_lowlevel' : 'BUILDBOT_MQ_DEBUG' in os.environ,
-    }
-# 'protocols' contains information about protocols which master will use for
-# communicating with workers. You must define at least 'port' option that workers
-# could connect to your master with this protocol.
-# 'port' must match the value configured into the workers (with their
-# --master option)
-c['protocols'] = {'pb': {'port': os.environ.get("BUILDBOT_WORKER_PORT", 9989)}}
-
-####### CHANGESOURCES
-
-# the 'change_source' setting tells the buildmaster how it should find out
-# about source code changes.  Here we point to the buildbot clone of pyflakes.
-
-c['change_source'] = []
-c['change_source'].append(changes.GitPoller(
-        'git://github.com/buildbot/pyflakes.git',
-        workdir='gitpoller-workdir', branch='master',
-        pollInterval=300))
-
-####### SCHEDULERS
-
-# Configure the Schedulers, which decide how to react to incoming changes.  In this
-# case, just kick off a 'runtests' build
-
-c['schedulers'] = []
-c['schedulers'].append(schedulers.SingleBranchScheduler(
-                            name="all",
-                            change_filter=util.ChangeFilter(branch='master'),
-                            treeStableTimer=None,
-                            builderNames=["runtests"]))
-c['schedulers'].append(schedulers.ForceScheduler(
-                            name="force",
-                            builderNames=["runtests"]))
-
-####### BUILDERS
-
-# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
-# what steps, and which workers can execute them.  Note that any particular build will
-# only take place on one worker.
-
-factory = util.BuildFactory()
-# check out the source
-factory.addStep(steps.Git(repourl='http://github.com/buildbot/pyflakes.git', mode='incremental'))
-# run the tests (note that this will require that 'trial' is installed)
-factory.addStep(steps.ShellCommand(command=["trial", "pyflakes"]))
-
-c['builders'] = []
-c['builders'].append(
-    util.BuilderConfig(name="runtests",
-      workernames=["example-worker"],
-      factory=factory))
-
-####### REPORTER TARGETS
-
-# 'services' is a list of Reporter Targets. The results of each build will be
-# pushed to these targets. buildbot/reporters/*.py has a variety to choose from,
-# like IRC bots.
-
-c['services'] = []
-
-####### PROJECT IDENTITY
-
-# the 'title' string will appear at the top of this buildbot installation's
-# home pages (linked to the 'titleURL').
-
-c['title'] = "Blender"
-c['titleURL'] = "https://blender.org/download"
-
-# the 'buildbotURL' string should point to the location where the buildbot's
-# internal web server is visible. This typically uses the port number set in
-# the 'www' entry below, but with an externally-visible host name which the
-# buildbot cannot figure out without some help.
-
-c['buildbotURL'] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/")
-
-# minimalistic config to activate new web UI
-c['www'] = dict(port=os.environ.get("BUILDBOT_WEB_PORT", 8010),
-                plugins=dict(waterfall_view={}, console_view={}))
-
-####### DB URL
-
-c['db'] = {
-    # This specifies what database buildbot uses to store its state.  You can leave
-    # this at its default for all but the largest installations.
-    'db_url' : os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ),
-}
+import setup
+importlib.reload(setup)
+BuildmasterConfig = setup.setup()

config/setup.py

@@ -0,0 +1,169 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# SPDX-FileCopyrightText: 2011-2024 Blender Authors
+# <pep8 compliant>
+
+import importlib
+import os
+import sys
+import re
+import pathlib
+
+import buildbot.plugins
+
+from typing import Any, Dict, List
+
+sys.path.append(str(pathlib.Path(__file__).resolve().parent))
+
+import conf.auth
+import conf.branches
+import conf.machines
+import conf.worker
+
+import gitea.blender
+
+import pipeline
+
+# We need to do this when we reload (SIGHUP) the buildbot server process.
+importlib.reload(conf.auth)
+importlib.reload(conf.branches)
+importlib.reload(conf.machines)
+importlib.reload(conf.worker)
+importlib.reload(gitea.blender)
+importlib.reload(pipeline)
+
+devops_env_id = os.environ.get("DEVOPS_ENV_ID", default="LOCAL")
+devops_host_id = os.environ.get("DEVOPS_HOST_ID", default="localhost")
+
+
+def setup() -> Dict[str, Any]:
+    ####### MAIN CONFIGURATION
+    c = {}
+
+    # Change Source
+    c["change_source"] = pipeline.change_sources()
+
+    # Workers
+    print("*** Creating platform workers")
+    platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id)
+    workers: List[buildbot.plugins.worker.Worker] = []
+    configured_worker_names = set()
+    for worker_names in platform_worker_names.values():
+        for worker_name in worker_names:
+            if worker_name in configured_worker_names:
+                print(f"Skipping {worker_name}, already configured")
+                continue
+            configured_worker_names.add(worker_name)
+            workers += [
+                buildbot.plugins.worker.Worker(
+                    worker_name,
+                    conf.machines.get_worker_password(devops_env_id, worker_name),
+                    max_builds=1,
+                    keepalive_interval=3600,
+                )
+            ]
+
+    print("*** Creating local workers")
+    local_worker_names = conf.machines.fetch_local_worker_names()
+    for worker_name in local_worker_names:
+        workers += [buildbot.plugins.worker.LocalWorker(worker_name)]
+
+    c["workers"] = workers
+
+    # Builders and Schedulers
+    builders, schedulers = pipeline.populate(devops_env_id)
+    c["builders"] = builders
+    c["schedulers"] = schedulers
+
+    ####### BUILDBOT SERVICES
+
+    # 'services' is a list of BuildbotService items like reporter targets. The
+    # status of each build will be pushed to these targets. buildbot/reporters/*.py
+    # has a variety to choose from, like IRC bots.
+
+    gitea_status_service = gitea.blender.setup_service(devops_env_id)
+    if gitea_status_service:
+        c["services"] = [gitea_status_service]
+    else:
+        c["services"] = []
+
+    ####### PROJECT IDENTITY
+
+    # the 'title' string will appear at the top of this buildbot installation's
+    # home pages (linked to the 'titleURL').
+
+    c["title"] = f"Bot - {devops_env_id}"
+    c["titleURL"] = "https://projects.blender.org"
+
+    # the 'buildbotURL' string should point to the location where the buildbot's
+    # internal web server is visible. This typically uses the port number set in
+    # the 'www' entry below, but with an externally-visible host name which the
+    # buildbot cannot figure out without some help.
+    c["buildbotURL"] = f"http://{devops_host_id}:8010/"
+
+    if devops_env_id != "LOCAL":
+        c["buildbotURL"] = f"http://{devops_host_id}:8000/admin/"
+
+        if devops_env_id == "PROD":
+            c["buildbotURL"] = "https://builder.blender.org/admin/"
+        if devops_env_id == "UATEST":
+            c["buildbotURL"] = "https://builder.uatest.blender.org/admin/"
+
+    # Minimalistic config to activate new web UI
+    c["www"] = dict(
+        port=8010, plugins=dict(waterfall_view={}, console_view={}, grid_view={})
+    )
+
+    # Database
+    if devops_env_id == "LOCAL":
+        c["db"] = {"db_url": "sqlite:///state.sqlite"}
+    else:
+        # PostgreSQL database, as recommended for production environment.
+        c["db"] = {"db_url": "postgresql://buildbot@127.0.0.1/buildbot"}
+
+    c["buildbotNetUsageData"] = None
+
+    # Authentication
+    c["www"]["auth"] = conf.auth.fetch_authentication(devops_env_id)
+
+    # Authorization
+    c["www"]["authz"] = conf.auth.fetch_authorization(devops_env_id)
+
+    # Disable UI - does not work
+    c["www"]["plugins"] = {
+        "waterfall_view": False,
+        "console_view": False,
+        "grid_view": False,
+    }
+
+    # UI Defaults
+    c["www"]["ui_default_config"] = {
+        "Grid.fullChanges": True,
+        "Grid.leftToRight": True,
+        "Grid.revisionLimit": 10,
+        "Builders.buildFetchLimit": 400,
+        "LogPreview.loadlines": 100,
+        "LogPreview.maxlines": 100,
+        "ChangeBuilds.buildsFetchLimit": 10,
+    }
+
+    # Validation
+    c["validation"] = {
+        "branch": re.compile(r"^[\w.+/~-]*$"),
+        "revision": re.compile(r"^[ \w\.\-\/]*$"),
+        "property_name": re.compile(r"^[\w\.\-\/\~:]*$"),
+        "property_value": re.compile(r"^[\w\.\-\/\~:]*$"),
+    }
+
+    # Rev link
+    c["revlink"] = buildbot.plugins.util.RevlinkMatch(
+        [r"https://projects.blender.org/([^/]*)/([^/]*?)(?:\.git)?$"],
+        r"https://projects.blender.org/\1/\2/commit/%s",
+    )
+
+    # Port for workers to connectto
+    c["protocols"] = {"pb": {"port": 9989}}
+
+    # Disable collapsing requests
+    c["collapseRequests"] = False
+
+    return c

docker-compose.override.yml

@@ -0,0 +1,5 @@
+services:
+  buildbot-master:
+    env_file: .env
+    volumes:
+      - ./config:/buildbot/config

mypy.ini

@@ -0,0 +1,3 @@
+[mypy]
+warn_unused_configs = True
+ignore_missing_imports = True

requirements.txt

@@ -0,0 +1,2 @@
+ruff
+mypy