From 1c908c4b825ab9b4a268cfb80fb3d9315568e4bd Mon Sep 17 00:00:00 2001 From: Bart van der Braak Date: Tue, 19 Nov 2024 16:51:43 +0100 Subject: [PATCH] Add Make and Pipelines for checks --- .env.example | 3 + .forgejo/workflows/check.yml | 14 ++ .../workflows/{publish.yml => release.yml} | 10 +- .gitignore | 2 + Makefile | 16 ++ config/master.cfg | 117 +----------- config/setup.py | 169 ++++++++++++++++++ docker-compose.override.yml | 5 + mypy.ini | 3 + requirements.txt | 2 + 10 files changed, 226 insertions(+), 115 deletions(-) create mode 100644 .env.example create mode 100644 .forgejo/workflows/check.yml rename .forgejo/workflows/{publish.yml => release.yml} (61%) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 config/setup.py create mode 100644 docker-compose.override.yml create mode 100644 mypy.ini create mode 100644 requirements.txt diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..fd89b15 --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +SERVICE_USER_POSTGRESQL=buildbot +SERVICE_PASSWORD_POSTGRESQL=changeme! +BUILDBOT_CONFIG_URL='' \ No newline at end of file diff --git a/.forgejo/workflows/check.yml b/.forgejo/workflows/check.yml new file mode 100644 index 0000000..8f68941 --- /dev/null +++ b/.forgejo/workflows/check.yml @@ -0,0 +1,14 @@ +name: Run checks +on: [ push, pull_request ] +jobs: + checks: + runs-on: docker + env: + DIRECTORY: config + steps: + - uses: actions/checkout@v4 + - run: cd ${{ env.DIRECTORY }} + - run: pip install -r requirements-dev.txt + - run: ruff check + - run: ruff format + - run: myp \ No newline at end of file diff --git a/.forgejo/workflows/publish.yml b/.forgejo/workflows/release.yml similarity index 61% rename from .forgejo/workflows/publish.yml rename to .forgejo/workflows/release.yml index 1e2c061..68a5349 100644 --- a/.forgejo/workflows/publish.yml +++ b/.forgejo/workflows/release.yml @@ -1,3 +1,4 @@ +name: Release and deploy on: push: branches: @@ -6,18 +7,19 @@ jobs: artifact: runs-on: docker env: - ARCHIVE_FOLDER: config + DIRECTORY: config + PACKAGE_NAME: builder.braak.pro steps: - name: Checkout uses: actions/checkout@v4 - name: Archive - run: "tar -czvf ${{ env.ARCHIVE_FOLDER }}.tar.gz ${{ env.ARCHIVE_FOLDER }}" + run: "tar -czvf ${{ env.DIRECTORY }}.tar.gz ${{ env.DIRECTORY }}" - name: Upload run: | curl \ --user "${{ env.GITHUB_REPOSITORY_OWNER }}:${{ secrets.ACCESS_TOKEN }}" \ - --upload-file "${{ env.ARCHIVE_FOLDER }}.tar.gz" \ - "${{ env.GITHUB_SERVER_URL }}/api/packages/${{ env.GITHUB_REPOSITORY_OWNER }}/generic/builder.braak.pro/${{ env.GITHUB_REF_NAME }}/${{ env.ARCHIVE_FOLDER }}.tar.gz" + --upload-file "${{ env.DIRECTORY }}.tar.gz" \ + "${{ env.GITHUB_SERVER_URL }}/api/packages/${{ env.GITHUB_REPOSITORY_OWNER }}/generic/${{ env.PACKAGE_NAME }}/${{ env.GITHUB_REF_NAME }}/${{ env.ARCHIVE_FOLDER }}.tar.gz" - name: Deploy run: | curl -X "POST" \ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c2eabec --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.venv +.env \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cc95b6a --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ + +.PHONY: venv setup check +help: ## Display this help message + @echo "Usage:" + @echo " make " + @echo "" + @echo "Targets:" + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-20s %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +setup: ## Create Python virtualenv and install dependencies + @if [ ! -f .env ]; then cp .env.example .env; fi + @if [ ! -d .venv ]; then python3 -m venv .venv; fi + @if [ -d .venv ]; then . .venv/bin/activate && pip3 install -r requirements.txt; fi + +check: ## Run linter, formatter and typechecks + ruff check && ruff format && mypy config/master.cfg \ No newline at end of file diff --git a/config/master.cfg b/config/master.cfg index a11d998..8d4475a 100644 --- a/config/master.cfg +++ b/config/master.cfg @@ -1,116 +1,11 @@ # -*- python -*- # ex: set filetype=python: - +import importlib import os +import sys -from buildbot.plugins import * +sys.path.insert(0, os.path.expanduser("~/git/blender-devops/buildbot")) -# This is a sample buildmaster config file. It must be installed as -# 'master.cfg' in your buildmaster's base directory. - -# This is the dictionary that the buildmaster pays attention to. We also use -# a shorter alias to save typing. -c = BuildmasterConfig = {} - -####### WORKERS - -# The 'workers' list defines the set of recognized workers. Each element is -# a Worker object, specifying a unique worker name and password. The same -# worker name and password must be configured on the worker. - -c['workers'] = [worker.Worker("example-worker", 'pass')] - -if 'BUILDBOT_MQ_URL' in os.environ: - c['mq'] = { - 'type' : 'wamp', - 'router_url': os.environ['BUILDBOT_MQ_URL'], - 'realm': os.environ.get('BUILDBOT_MQ_REALM', 'buildbot').decode('utf-8'), - 'debug' : 'BUILDBOT_MQ_DEBUG' in os.environ, - 'debug_websockets' : 'BUILDBOT_MQ_DEBUG' in os.environ, - 'debug_lowlevel' : 'BUILDBOT_MQ_DEBUG' in os.environ, - } -# 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their -# --master option) -c['protocols'] = {'pb': {'port': os.environ.get("BUILDBOT_WORKER_PORT", 9989)}} - -####### CHANGESOURCES - -# the 'change_source' setting tells the buildmaster how it should find out -# about source code changes. Here we point to the buildbot clone of pyflakes. - -c['change_source'] = [] -c['change_source'].append(changes.GitPoller( - 'git://github.com/buildbot/pyflakes.git', - workdir='gitpoller-workdir', branch='master', - pollInterval=300)) - -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming changes. In this -# case, just kick off a 'runtests' build - -c['schedulers'] = [] -c['schedulers'].append(schedulers.SingleBranchScheduler( - name="all", - change_filter=util.ChangeFilter(branch='master'), - treeStableTimer=None, - builderNames=["runtests"])) -c['schedulers'].append(schedulers.ForceScheduler( - name="force", - builderNames=["runtests"])) - -####### BUILDERS - -# The 'builders' list defines the Builders, which tell Buildbot how to perform a build: -# what steps, and which workers can execute them. Note that any particular build will -# only take place on one worker. - -factory = util.BuildFactory() -# check out the source -factory.addStep(steps.Git(repourl='http://github.com/buildbot/pyflakes.git', mode='incremental')) -# run the tests (note that this will require that 'trial' is installed) -factory.addStep(steps.ShellCommand(command=["trial", "pyflakes"])) - -c['builders'] = [] -c['builders'].append( - util.BuilderConfig(name="runtests", - workernames=["example-worker"], - factory=factory)) - -####### REPORTER TARGETS - -# 'services' is a list of Reporter Targets. The results of each build will be -# pushed to these targets. buildbot/reporters/*.py has a variety to choose from, -# like IRC bots. - -c['services'] = [] - -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). - -c['title'] = "Blender" -c['titleURL'] = "https://blender.org/download" - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. - -c['buildbotURL'] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/") - -# minimalistic config to activate new web UI -c['www'] = dict(port=os.environ.get("BUILDBOT_WEB_PORT", 8010), - plugins=dict(waterfall_view={}, console_view={})) - -####### DB URL - -c['db'] = { - # This specifies what database buildbot uses to store its state. You can leave - # this at its default for all but the largest installations. - 'db_url' : os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ), -} \ No newline at end of file +import setup +importlib.reload(setup) +BuildmasterConfig = setup.setup() diff --git a/config/setup.py b/config/setup.py new file mode 100644 index 0000000..a4fdb9e --- /dev/null +++ b/config/setup.py @@ -0,0 +1,169 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# SPDX-FileCopyrightText: 2011-2024 Blender Authors +# + +import importlib +import os +import sys +import re +import pathlib + +import buildbot.plugins + +from typing import Any, Dict, List + +sys.path.append(str(pathlib.Path(__file__).resolve().parent)) + +import conf.auth +import conf.branches +import conf.machines +import conf.worker + +import gitea.blender + +import pipeline + +# We need to do this when we reload (SIGHUP) the buildbot server process. +importlib.reload(conf.auth) +importlib.reload(conf.branches) +importlib.reload(conf.machines) +importlib.reload(conf.worker) +importlib.reload(gitea.blender) +importlib.reload(pipeline) + +devops_env_id = os.environ.get("DEVOPS_ENV_ID", default="LOCAL") +devops_host_id = os.environ.get("DEVOPS_HOST_ID", default="localhost") + + +def setup() -> Dict[str, Any]: + ####### MAIN CONFIGURATION + c = {} + + # Change Source + c["change_source"] = pipeline.change_sources() + + # Workers + print("*** Creating platform workers") + platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id) + workers: List[buildbot.plugins.worker.Worker] = [] + configured_worker_names = set() + for worker_names in platform_worker_names.values(): + for worker_name in worker_names: + if worker_name in configured_worker_names: + print(f"Skipping {worker_name}, already configured") + continue + configured_worker_names.add(worker_name) + workers += [ + buildbot.plugins.worker.Worker( + worker_name, + conf.machines.get_worker_password(devops_env_id, worker_name), + max_builds=1, + keepalive_interval=3600, + ) + ] + + print("*** Creating local workers") + local_worker_names = conf.machines.fetch_local_worker_names() + for worker_name in local_worker_names: + workers += [buildbot.plugins.worker.LocalWorker(worker_name)] + + c["workers"] = workers + + # Builders and Schedulers + builders, schedulers = pipeline.populate(devops_env_id) + c["builders"] = builders + c["schedulers"] = schedulers + + ####### BUILDBOT SERVICES + + # 'services' is a list of BuildbotService items like reporter targets. The + # status of each build will be pushed to these targets. buildbot/reporters/*.py + # has a variety to choose from, like IRC bots. + + gitea_status_service = gitea.blender.setup_service(devops_env_id) + if gitea_status_service: + c["services"] = [gitea_status_service] + else: + c["services"] = [] + + ####### PROJECT IDENTITY + + # the 'title' string will appear at the top of this buildbot installation's + # home pages (linked to the 'titleURL'). + + c["title"] = f"Bot - {devops_env_id}" + c["titleURL"] = "https://projects.blender.org" + + # the 'buildbotURL' string should point to the location where the buildbot's + # internal web server is visible. This typically uses the port number set in + # the 'www' entry below, but with an externally-visible host name which the + # buildbot cannot figure out without some help. + c["buildbotURL"] = f"http://{devops_host_id}:8010/" + + if devops_env_id != "LOCAL": + c["buildbotURL"] = f"http://{devops_host_id}:8000/admin/" + + if devops_env_id == "PROD": + c["buildbotURL"] = "https://builder.blender.org/admin/" + if devops_env_id == "UATEST": + c["buildbotURL"] = "https://builder.uatest.blender.org/admin/" + + # Minimalistic config to activate new web UI + c["www"] = dict( + port=8010, plugins=dict(waterfall_view={}, console_view={}, grid_view={}) + ) + + # Database + if devops_env_id == "LOCAL": + c["db"] = {"db_url": "sqlite:///state.sqlite"} + else: + # PostgreSQL database, as recommended for production environment. + c["db"] = {"db_url": "postgresql://buildbot@127.0.0.1/buildbot"} + + c["buildbotNetUsageData"] = None + + # Authentication + c["www"]["auth"] = conf.auth.fetch_authentication(devops_env_id) + + # Authorization + c["www"]["authz"] = conf.auth.fetch_authorization(devops_env_id) + + # Disable UI - does not work + c["www"]["plugins"] = { + "waterfall_view": False, + "console_view": False, + "grid_view": False, + } + + # UI Defaults + c["www"]["ui_default_config"] = { + "Grid.fullChanges": True, + "Grid.leftToRight": True, + "Grid.revisionLimit": 10, + "Builders.buildFetchLimit": 400, + "LogPreview.loadlines": 100, + "LogPreview.maxlines": 100, + "ChangeBuilds.buildsFetchLimit": 10, + } + + # Validation + c["validation"] = { + "branch": re.compile(r"^[\w.+/~-]*$"), + "revision": re.compile(r"^[ \w\.\-\/]*$"), + "property_name": re.compile(r"^[\w\.\-\/\~:]*$"), + "property_value": re.compile(r"^[\w\.\-\/\~:]*$"), + } + + # Rev link + c["revlink"] = buildbot.plugins.util.RevlinkMatch( + [r"https://projects.blender.org/([^/]*)/([^/]*?)(?:\.git)?$"], + r"https://projects.blender.org/\1/\2/commit/%s", + ) + + # Port for workers to connectto + c["protocols"] = {"pb": {"port": 9989}} + + # Disable collapsing requests + c["collapseRequests"] = False + + return c diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000..cae2114 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,5 @@ +services: + buildbot-master: + env_file: .env + volumes: + - ./config:/buildbot/config \ No newline at end of file diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..f7d3a54 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +warn_unused_configs = True +ignore_missing_imports = True \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e3b95ac --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +ruff +mypy \ No newline at end of file