bartvdbraak/builder.braak.pro
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages 1 Wiki Activity Actions

Implement authentication via Gitea
Some checks failed
/ checks (pull_request) Failing after 14s
Details

Browse source
This commit is contained in:
Bart van der Braak 2024-11-20 23:59:35 +01:00
parent 0f45b38432
commit edb56e96dc
22 changed files with 134 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -1,7 +1,6 @@
# Hidden files
.venv
.env.production
.env.staging
.env*
# Python
__pycache__

5
Makefile
View file

@ -19,4 +19,7 @@ check: ## Check linting, formatting and types
format: ## Autofix linting and formatting issues
ruff check --fix
ruff format
ruff format
docker: ## Spin up and force recreation of Docker containers
docker compose up --force-recreate --detach

14
buildbot/config/conf/auth.py
View file

@ -7,8 +7,8 @@ import importlib
import buildbot.plugins
def _get_auth_config(ENVIRONMENT: str):
if ENVIRONMENT == "LOCAL":
def _get_auth_config(environment: str):
if environment == "LOCAL":
import conf.local.auth
importlib.reload(conf.local.auth)
@ -20,13 +20,13 @@ def _get_auth_config(ENVIRONMENT: str):
return conf.production.auth
def fetch_authentication(ENVIRONMENT: str):
auth_config = _get_auth_config(ENVIRONMENT)
return auth_config.get_authentication(ENVIRONMENT)
def fetch_authentication(environment: str):
auth_config = _get_auth_config(environment)
return auth_config.get_authentication(environment)
def fetch_authorization(ENVIRONMENT: str):
auth_config = _get_auth_config(ENVIRONMENT)
def fetch_authorization(environment: str):
auth_config = _get_auth_config(environment)
admin_usernames = auth_config.admin_usernames
deploy_dev_usernames = auth_config.deploy_dev_usernames

35
buildbot/config/conf/local/auth.py
View file

@ -2,7 +2,10 @@
# SPDX-FileCopyrightText: 2011-2024 Blender Authors
# <pep8 compliant>
import buildbot.plugins
# import buildbot.plugins
import os
from buildbot.www.oauth2 import OAuth2Auth
from urllib.parse import urljoin
# Buildbot admin with access to everything.
admin_usernames = [
@ -19,10 +22,30 @@ trusted_dev_usernames = [
"admin",
]
gitea_endpoint = os.environ.get("GITEA_ENDPOINT", default="")
gitea_client_id = os.environ.get("GITEA_CLIENT_ID", default="")
gitea_client_secret = os.environ.get("GITEA_CLIENT_SECRET", default="")
def get_authentication(ENVIRONMENT: str):
class LocalEnvAuth(buildbot.plugins.util.CustomAuth):
def check_credentials(self, user, password):
return user.decode() == "admin" and password.decode() == "admin"
return LocalEnvAuth()
def get_authentication(environment: str):
class GiteaAuth(OAuth2Auth):
name = "projects.blender.org"
faIcon = "fa-cogs"
AUTH_URL = "login/oauth/authorize"
TOKEN_URL = "login/oauth/access_token"
def __init__(self, endpoint, client_id, client_secret, **kwargs):
super(GiteaAuth, self).__init__(client_id, client_secret, **kwargs)
self.resourceEndpoint = endpoint
self.authUri = urljoin(endpoint, self.AUTH_URL)
self.tokenUri = urljoin(endpoint, self.TOKEN_URL)
def getUserInfoFromOAuthClient(self, c):
return self.get(c, "/api/v1/user")
# class LocalEnvAuth(buildbot.plugins.util.CustomAuth):
# def check_credentials(self, user, password):
# return user.decode() == "admin" and password.decode() == "admin"
return GiteaAuth(gitea_endpoint, gitea_client_id, gitea_client_secret)

14
buildbot/config/conf/machines.py
View file

@ -5,8 +5,8 @@
import importlib
def _get_config(ENVIRONMENT: str):
if ENVIRONMENT == "LOCAL":
def _get_config(environment: str):
if environment == "LOCAL":
import conf.local.machines
importlib.reload(conf.local.machines)
@ -18,13 +18,13 @@ def _get_config(ENVIRONMENT: str):
return conf.production.machines
def fetch_platform_worker_names(ENVIRONMENT: str):
machines_config = _get_config(ENVIRONMENT)
return machines_config.get_worker_names(ENVIRONMENT)
def fetch_platform_worker_names(environment: str):
machines_config = _get_config(environment)
return machines_config.get_worker_names(environment)
def get_worker_password(ENVIRONMENT: str, worker_name: str) -> str:
machines_config = _get_config(ENVIRONMENT)
def get_worker_password(environment: str, worker_name: str) -> str:
machines_config = _get_config(environment)
return machines_config.get_worker_password(worker_name)

4
buildbot/config/conf/worker.py
View file

@ -7,8 +7,8 @@ import importlib
from typing import Any
def get_config(ENVIRONMENT: str) -> Any:
if ENVIRONMENT == "LOCAL":
def get_config(environment: str) -> Any:
if environment == "LOCAL":
import conf.local.worker
importlib.reload(conf.local.worker)

6
buildbot/config/gitea/blender.py
View file

@ -22,12 +22,12 @@ gitea_api_token = None
gitea_status_service = None
def setup_service(ENVIRONMENT: str):
def setup_service(environment: str):
import conf.worker
importlib.reload(conf.worker)
worker_config = conf.worker.get_config(ENVIRONMENT)
gitea_api_token = worker_config.gitea_api_token(ENVIRONMENT)
worker_config = conf.worker.get_config(environment)
gitea_api_token = worker_config.gitea_api_token(environment)
if gitea_api_token:
log.msg("Found Gitea API token, enabling status push")

4
buildbot/config/pipeline/__init__.py
View file

@ -23,7 +23,7 @@ importlib.reload(pipeline.common)
importlib.reload(conf.branches)
def populate(ENVIRONMENT):
def populate(environment):
pipelines_modules = [
pipeline.code,
pipeline.code_benchmark,
@ -41,7 +41,7 @@ def populate(ENVIRONMENT):
for pipelines_module in pipelines_modules:
importlib.reload(pipelines_module)
b, s = pipelines_module.populate(ENVIRONMENT)
b, s = pipelines_module.populate(environment)
builders += b
schedulers += s

22
buildbot/config/pipeline/code.py
View file

@ -233,7 +233,7 @@ scheduler_properties = {
@buildbot.plugins.util.renderer
def create_code_worker_command_args(
props, ENVIRONMENT, track_id, pipeline_type, step_name
props, environment, track_id, pipeline_type, step_name
):
commit_id = pipeline.common.fetch_property(props, key="revision", default="HEAD")
patch_id = pipeline.common.fetch_property(props, key="patch_id", default="")
@ -294,7 +294,7 @@ def create_code_worker_command_args(
args += [step_name]
return pipeline.common.create_worker_command("code.py", ENVIRONMENT, track_id, args)
return pipeline.common.create_worker_command("code.py", environment, track_id, args)
def needs_do_code_pipeline_step(step):
@ -449,17 +449,17 @@ class PlatformTrigger(plugins_steps.Trigger):
return schedulers
def populate(ENVIRONMENT):
def populate(environment):
builders = []
schedulers = []
platform_worker_names = conf.machines.fetch_platform_worker_names(ENVIRONMENT)
platform_worker_names = conf.machines.fetch_platform_worker_names(environment)
local_worker_names = conf.machines.fetch_local_worker_names()
worker_config = conf.worker.get_config(ENVIRONMENT)
worker_config = conf.worker.get_config(environment)
needs_incremental_schedulers = ENVIRONMENT in ["PROD"]
needs_nightly_schedulers = ENVIRONMENT in ["PROD"]
needs_incremental_schedulers = environment in ["PROD"]
needs_nightly_schedulers = environment in ["PROD"]
print("*** Creating [code] pipeline")
for track_id in code_track_ids:
@ -491,7 +491,7 @@ def populate(ENVIRONMENT):
step_timeout_in_seconds = compile_gpu_step_timeout_in_seconds
step_command = create_code_worker_command_args.withArgs(
ENVIRONMENT, track_id, pipeline_type, step_name
environment, track_id, pipeline_type, step_name
)
step = buildbot.plugins.steps.ShellCommand(
@ -510,7 +510,7 @@ def populate(ENVIRONMENT):
for master_step_name in pipeline.common.code_pipeline_master_step_names:
master_step_command = (
pipeline.common.create_master_command_args.withArgs(
ENVIRONMENT,
environment,
track_id,
pipeline_type,
master_step_name,
@ -534,7 +534,7 @@ def populate(ENVIRONMENT):
pipeline_lint_factory = buildbot.plugins.util.BuildFactory()
for step_name in code_pipeline_lint_step_names:
step_command = create_code_worker_command_args.withArgs(
ENVIRONMENT, track_id, pipeline_type, step_name
environment, track_id, pipeline_type, step_name
)
pipeline_lint_factory.addStep(
@ -574,7 +574,7 @@ def populate(ENVIRONMENT):
suitable_pipeline_worker_names = pipeline_worker_names
if (
platform_architecture == "linux-x86_64"
and ENVIRONMENT != "LOCAL"
and environment != "LOCAL"
):
selector = "rocky"
suitable_pipeline_worker_names = [

10
buildbot/config/pipeline/code_benchmark.py
View file

@ -26,8 +26,8 @@ class LinkMultipleFileUpload(plugins_steps.MultipleFileUpload):
return
def create_deliver_step(ENVIRONMENT):
worker_config = conf.worker.get_config(ENVIRONMENT)
def create_deliver_step(environment):
worker_config = conf.worker.get_config(environment)
file_size_in_mb = 500 * 1024 * 1024
worker_source_path = pathlib.Path("../../../../git/blender-vdev/build_package")
@ -48,7 +48,7 @@ def create_deliver_step(ENVIRONMENT):
)
def populate(ENVIRONMENT):
def populate(environment):
properties = [
buildbot.plugins.util.StringParameter(
name="commit_id",
@ -68,7 +68,7 @@ def populate(ENVIRONMENT):
]
return pipeline.common.create_pipeline(
ENVIRONMENT,
environment,
"code-benchmark",
"code_benchmark.py",
[
@ -78,7 +78,7 @@ def populate(ENVIRONMENT):
"compile-gpu",
"compile-install",
"benchmark",
partial(create_deliver_step, ENVIRONMENT),
partial(create_deliver_step, environment),
"clean",
],
{"vdev": "main"},

4
buildbot/config/pipeline/code_bpy_deploy.py
View file

@ -9,11 +9,11 @@ import conf.branches
import pipeline.common
def populate(ENVIRONMENT):
def populate(environment):
properties = []
return pipeline.common.create_pipeline(
ENVIRONMENT,
environment,
"code-bpy-deploy",
"code_bpy_deploy.py",
[

4
buildbot/config/pipeline/code_deploy.py
View file

@ -10,7 +10,7 @@ import conf.branches
import pipeline.common
def populate(ENVIRONMENT):
def populate(environment):
properties = [
buildbot.plugins.util.BooleanParameter(
name="needs_full_clean",
@ -22,7 +22,7 @@ def populate(ENVIRONMENT):
]
return pipeline.common.create_pipeline(
ENVIRONMENT,
environment,
"code-artifacts-deploy",
"code_deploy.py",
[

12
buildbot/config/pipeline/code_store.py
View file

@ -62,16 +62,16 @@ def create_deliver_binaries_windows_step(worker_config, track_id, pipeline_type)
)
def populate(ENVIRONMENT):
def populate(environment):
builders = []
schedulers = []
platform_worker_names = conf.machines.fetch_platform_worker_names(ENVIRONMENT)
platform_worker_names = conf.machines.fetch_platform_worker_names(environment)
local_worker_names = conf.machines.fetch_local_worker_names()
worker_config = conf.worker.get_config(ENVIRONMENT)
worker_config = conf.worker.get_config(environment)
needs_nightly_schedulers = ENVIRONMENT == "PROD"
needs_nightly_schedulers = environment == "PROD"
pipeline_type = "daily"
@ -108,7 +108,7 @@ def populate(ENVIRONMENT):
else:
args = ["--store-id", store_id, step_name]
step_command = pipeline.common.create_worker_command(
"code_store.py", ENVIRONMENT, track_id, args
"code_store.py", environment, track_id, args
)
step = plugins_steps.ShellCommand(
@ -126,7 +126,7 @@ def populate(ENVIRONMENT):
for master_step_name in pipeline.common.code_pipeline_master_step_names:
master_step_command = (
pipeline.common.create_master_command_args.withArgs(
ENVIRONMENT,
environment,
track_id,
pipeline_type,
master_step_name,

22
buildbot/config/pipeline/common.py
View file

@ -57,7 +57,7 @@ def needs_do_doc_pipeline_step(step):
return True
def create_worker_command(script, ENVIRONMENT, track_id, args):
def create_worker_command(script, environment, track_id, args):
# This relative path assume were are in:
# ~/.devops/services/buildbot-worker/<builder-name>/build
# There appears to be no way to expand a tilde here?
@ -71,7 +71,7 @@ def create_worker_command(script, ENVIRONMENT, track_id, args):
"--track-id",
track_id,
"--service-env-id",
ENVIRONMENT,
environment,
]
return cmd + list(args)
@ -79,7 +79,7 @@ def create_worker_command(script, ENVIRONMENT, track_id, args):
@buildbot.plugins.util.renderer
def create_master_command_args(
props, ENVIRONMENT, track_id, pipeline_type, step_name, single_platform
props, environment, track_id, pipeline_type, step_name, single_platform
):
build_configuration = fetch_property(
props, key="build_configuration", default="release"
@ -116,7 +116,7 @@ def create_master_command_args(
"--track-id",
track_id,
"--service-env-id",
ENVIRONMENT,
environment,
]
return cmd + list(args)
@ -125,7 +125,7 @@ def create_master_command_args(
@buildbot.plugins.util.renderer
def create_pipeline_worker_command(
props,
ENVIRONMENT,
environment,
track_id,
script,
step_name,
@ -154,11 +154,11 @@ def create_pipeline_worker_command(
if "revision" in props and props["revision"]:
args += ["--commit-id", props["revision"]]
return create_worker_command(script, ENVIRONMENT, track_id, args)
return create_worker_command(script, environment, track_id, args)
def create_pipeline(
ENVIRONMENT,
environment,
artifact_id,
script,
steps,
@ -179,13 +179,13 @@ def create_pipeline(
builders = []
schedulers = []
platform_worker_names = conf.machines.fetch_platform_worker_names(ENVIRONMENT)
platform_worker_names = conf.machines.fetch_platform_worker_names(environment)
local_worker_names = conf.machines.fetch_local_worker_names()
needs_incremental_schedulers = (
incremental_properties is not None and ENVIRONMENT in ["PROD"]
incremental_properties is not None and environment in ["PROD"]
)
needs_nightly_schedulers = nightly_properties is not None and ENVIRONMENT in [
needs_nightly_schedulers = nightly_properties is not None and environment in [
"PROD"
]
track_ids = tracked_branch_ids.keys()
@ -210,7 +210,7 @@ def create_pipeline(
continue
step_command = create_pipeline_worker_command.withArgs(
ENVIRONMENT,
environment,
track_id,
script,
step,

4
buildbot/config/pipeline/doc_api.py
View file

@ -8,7 +8,7 @@ import conf.branches
import pipeline.common
def populate(ENVIRONMENT):
def populate(environment):
properties = [
buildbot.plugins.util.BooleanParameter(
name="needs_full_clean",
@ -27,7 +27,7 @@ def populate(ENVIRONMENT):
]
return pipeline.common.create_pipeline(
ENVIRONMENT,
environment,
"doc-api",
"doc_api.py",
[

4
buildbot/config/pipeline/doc_developer.py
View file

@ -7,7 +7,7 @@ import buildbot.plugins
import pipeline.common
def populate(ENVIRONMENT):
def populate(environment):
properties = [
buildbot.plugins.util.BooleanParameter(
name="needs_package_delivery",
@ -19,7 +19,7 @@ def populate(ENVIRONMENT):
]
return pipeline.common.create_pipeline(
ENVIRONMENT,
environment,
"doc-developer",
"doc_developer.py",
["update", "compile", "deliver"],

4
buildbot/config/pipeline/doc_manual.py
View file

@ -8,7 +8,7 @@ import conf.branches
import pipeline.common
def populate(ENVIRONMENT):
def populate(environment):
properties = [
buildbot.plugins.util.BooleanParameter(
name="needs_package_delivery",
@ -27,7 +27,7 @@ def populate(ENVIRONMENT):
]
return pipeline.common.create_pipeline(
ENVIRONMENT,
environment,
"doc-manual",
"doc_manual.py",
["configure-machine", "update", "compile", "package", "deliver", "clean"],

4
buildbot/config/pipeline/doc_studio.py
View file

@ -7,7 +7,7 @@ import buildbot.plugins
import pipeline.common
def populate(ENVIRONMENT):
def populate(environment):
properties = [
buildbot.plugins.util.BooleanParameter(
name="needs_package_delivery",
@ -19,7 +19,7 @@ def populate(ENVIRONMENT):
]
return pipeline.common.create_pipeline(
ENVIRONMENT,
environment,
"doc-studio-tools",
"doc_studio.py",
["update", "compile", "deliver"],

32
buildbot/config/setup.py
View file

@ -31,7 +31,7 @@ importlib.reload(conf.worker)
importlib.reload(gitea.blender)
importlib.reload(pipeline)
environment = os.environ.get("BUILDBOT_environment", default="LOCAL")
environment = os.environ.get("BUILDBOT_ENVIRONMENT", default="LOCAL")
def setup() -> Dict[str, Any]:
@ -90,7 +90,7 @@ def setup() -> Dict[str, Any]:
# the 'title' string will appear at the top of this buildbot installation's
# home pages (linked to the 'titleURL').
c["title"] = f"Blender Buildbot - {environment}"
c["title"] = "Builder"
c["titleURL"] = "https://projects.blender.org"
# the 'buildbotURL' string should point to the location where the buildbot's
@ -103,6 +103,20 @@ def setup() -> Dict[str, Any]:
c["www"] = dict(
port=os.environ.get("BUILDBOT_WEB_PORT", 8010),
plugins=dict(waterfall_view={}, console_view={}, grid_view={}),
theme={
"bb-sidebar-background-color": "#1F2226", # Eerie Black 2
"bb-sidebar-header-background-color": "#202327", # Eerie Black
"bb-sidebar-header-text-color": "#9fa3a8", # Dim Gray (Lighter gray for text)
"bb-sidebar-title-text-color": "#9fa3a8", # Dim Gray (Titles)
"bb-sidebar-footer-background-color": "#292d32", # Jet
"bb-sidebar-button-text-color": "#9fa3a8", # Dim Gray (Button text)
"bb-sidebar-button-hover-background-color": "#292d32", # Jet (Button hover background)
"bb-sidebar-button-hover-text-color": "#3dabf5", # Light blue for hover text
"bb-sidebar-button-current-background-color": "#292d32", # Jet (Current button background)
"bb-sidebar-button-current-text-color": "#3dabf5", # Light blue for current button text
"bb-sidebar-stripe-hover-color": "#3695D5", # Celestial Blue
"bb-sidebar-stripe-current-color": "#084F7E", # Indigo Dye
},
)
# Database
@ -116,7 +130,19 @@ def setup() -> Dict[str, Any]:
c["www"]["auth"] = conf.auth.fetch_authentication(environment)
# Authorization
c["www"]["authz"] = conf.auth.fetch_authorization(environment)
# c["www"]["authz"] = conf.auth.fetch_authorization(environment)
c["www"]["authz"] = buildbot.plugins.util.Authz(
allowRules=[
buildbot.plugins.util.AnyControlEndpointMatcher(
role="Admins"
), # Organization teams
],
roleMatchers=[
buildbot.plugins.util.RolesFromGroups(
groupPrefix="test-org/"
) # Gitea organization
],
)
# Disable UI - does not work
c["www"]["plugins"] = {

2
buildbot/config/worker/blender/__init__.py
View file

@ -47,7 +47,7 @@ class CodeBuilder(worker.utils.Builder):
# Call command with in compiler environment.
def call(
self, cmd: worker.utils.CmdSequence, env: worker.utils.CmdEnvironment = None
self, cmd: worker.utils.CmdSequence, env: worker.utils.Cmdenvironment = None
) -> int:
cmd_prefix: worker.utils.CmdList = []

4
buildbot/config/worker/utils.py
View file

@ -115,7 +115,7 @@ CmdArgument = Union[str, pathlib.Path, HiddenArgument, Any]
CmdList = List[CmdArgument]
CmdSequence = Sequence[CmdArgument]
CmdFilterOutput = Optional[Callable[[str], Optional[str]]]
CmdEnvironment = Optional[Dict[str, str]]
Cmdenvironment = Optional[Dict[str, str]]
def _prepare_call(
@ -142,7 +142,7 @@ def _prepare_call(
def call(
cmd: CmdSequence,
env: CmdEnvironment = None,
env: Cmdenvironment = None,
exit_on_error: bool = True,
filter_output: CmdFilterOutput = None,
retry_count: int = 0,

3
docker-compose.yml
View file

@ -1,6 +1,7 @@
services:
buildbot-master:
image: 'buildbot/buildbot-master:${BUILDBOT_IMAGE_TAG:-v4.1.0}'
env_file: .env
hostname: buildbot-master
restart: unless-stopped
ports:
@ -59,7 +60,7 @@ services:
- 'BUILDBOT_ENVIRONMENT=${BUILDBOT_ENVIRONMENT:-LOCAL}'
- 'WORKERNAME=${WORKERNAME:-localhost}'
- 'WORKERPASS=${WORKERPASS:-localhost}'
- 'WORKER_ENVIRONMENT_BLACKLIST=${WORKER_ENVIRONMENT_BLACKLIST:-DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST}'
- 'WORKER_environment_BLACKLIST=${WORKER_environment_BLACKLIST:-DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_environment_BLACKLIST}'
healthcheck:
test:
- CMD