bartvdbraak/builder.braak.pro
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages 1 Wiki Activity Actions

Change environment variable setup
All checks were successful
/ checks (pull_request) Successful in 15s
Details

Browse source
This commit is contained in:
Bart van der Braak 2024-11-20 02:29:30 +01:00
parent d6bce1b39d
commit 77ae214d24
23 changed files with 113 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.env.example
View file

@ -1,3 +0,0 @@
SERVICE_USER_POSTGRESQL=buildbot
SERVICE_PASSWORD_POSTGRESQL=changeme!
BUILDBOT_CONFIG_URL=''

4
.env.local Normal file
View file

@ -0,0 +1,4 @@
SERVICE_USER_POSTGRESQL=buildbot
SERVICE_PASSWORD_POSTGRESQL=changeme!
GITEA_URL=https://projects.blender.org
BUILDBOT_WEB_URL=http://localhost:8010/

3
.gitignore vendored
View file

@ -1,2 +1,3 @@
.venv
.env
.env.staging
.env.production

14
config/conf/auth.py
View file

@ -7,8 +7,8 @@ import importlib
import buildbot.plugins
def _get_auth_config(devops_env_id: str):
if devops_env_id == "LOCAL":
def _get_auth_config(ENVIRONMENT: str):
if ENVIRONMENT == "LOCAL":
import conf.local.auth
importlib.reload(conf.local.auth)
@ -20,13 +20,13 @@ def _get_auth_config(devops_env_id: str):
return conf.production.auth
def fetch_authentication(devops_env_id: str):
auth_config = _get_auth_config(devops_env_id)
return auth_config.get_authentication(devops_env_id)
def fetch_authentication(ENVIRONMENT: str):
auth_config = _get_auth_config(ENVIRONMENT)
return auth_config.get_authentication(ENVIRONMENT)
def fetch_authorization(devops_env_id: str):
auth_config = _get_auth_config(devops_env_id)
def fetch_authorization(ENVIRONMENT: str):
auth_config = _get_auth_config(ENVIRONMENT)
admin_usernames = auth_config.admin_usernames
deploy_dev_usernames = auth_config.deploy_dev_usernames

2
config/conf/local/auth.py
View file

@ -20,7 +20,7 @@ trusted_dev_usernames = [
]
def get_authentication(devops_env_id: str):
def get_authentication(ENVIRONMENT: str):
class LocalEnvAuth(buildbot.plugins.util.CustomAuth):
def check_credentials(self, user, password):
return user.decode() == "admin" and password.decode() == "admin"

2
config/conf/local/machines.py
View file

@ -27,5 +27,5 @@ def get_worker_password(worker_name: str) -> str:
return "localhost"
def get_worker_names(devops_env_id: str):
def get_worker_names(ENVIRONMENT: str):
return _worker_names

14
config/conf/machines.py
View file

@ -5,8 +5,8 @@
import importlib
def _get_config(devops_env_id: str):
if devops_env_id == "LOCAL":
def _get_config(ENVIRONMENT: str):
if ENVIRONMENT == "LOCAL":
import conf.local.machines
importlib.reload(conf.local.machines)
@ -18,13 +18,13 @@ def _get_config(devops_env_id: str):
return conf.production.machines
def fetch_platform_worker_names(devops_env_id: str):
machines_config = _get_config(devops_env_id)
return machines_config.get_worker_names(devops_env_id)
def fetch_platform_worker_names(ENVIRONMENT: str):
machines_config = _get_config(ENVIRONMENT)
return machines_config.get_worker_names(ENVIRONMENT)
def get_worker_password(devops_env_id: str, worker_name: str) -> str:
machines_config = _get_config(devops_env_id)
def get_worker_password(ENVIRONMENT: str, worker_name: str) -> str:
machines_config = _get_config(ENVIRONMENT)
return machines_config.get_worker_password(worker_name)

4
config/conf/worker.py
View file

@ -7,8 +7,8 @@ import importlib
from typing import Any
def get_config(devops_env_id: str) -> Any:
if devops_env_id == "LOCAL":
def get_config(ENVIRONMENT: str) -> Any:
if ENVIRONMENT == "LOCAL":
import conf.local.worker
importlib.reload(conf.local.worker)

6
config/gitea/blender.py
View file

@ -22,12 +22,12 @@ gitea_api_token = None
gitea_status_service = None
def setup_service(devops_env_id: str):
def setup_service(ENVIRONMENT: str):
import conf.worker
importlib.reload(conf.worker)
worker_config = conf.worker.get_config(devops_env_id)
gitea_api_token = worker_config.gitea_api_token(devops_env_id)
worker_config = conf.worker.get_config(ENVIRONMENT)
gitea_api_token = worker_config.gitea_api_token(ENVIRONMENT)
if gitea_api_token:
log.msg("Found Gitea API token, enabling status push")

4
config/pipeline/__init__.py
View file

@ -23,7 +23,7 @@ importlib.reload(pipeline.common)
importlib.reload(conf.branches)
def populate(devops_env_id):
def populate(ENVIRONMENT):
pipelines_modules = [
pipeline.code,
pipeline.code_benchmark,
@ -41,7 +41,7 @@ def populate(devops_env_id):
for pipelines_module in pipelines_modules:
importlib.reload(pipelines_module)
b, s = pipelines_module.populate(devops_env_id)
b, s = pipelines_module.populate(ENVIRONMENT)
builders += b
schedulers += s

24
config/pipeline/code.py
View file

@ -233,7 +233,7 @@ scheduler_properties = {
@buildbot.plugins.util.renderer
def create_code_worker_command_args(
props, devops_env_id, track_id, pipeline_type, step_name
props, ENVIRONMENT, track_id, pipeline_type, step_name
):
commit_id = pipeline.common.fetch_property(props, key="revision", default="HEAD")
patch_id = pipeline.common.fetch_property(props, key="patch_id", default="")
@ -294,9 +294,7 @@ def create_code_worker_command_args(
args += [step_name]
return pipeline.common.create_worker_command(
"code.py", devops_env_id, track_id, args
)
return pipeline.common.create_worker_command("code.py", ENVIRONMENT, track_id, args)
def needs_do_code_pipeline_step(step):
@ -451,17 +449,17 @@ class PlatformTrigger(plugins_steps.Trigger):
return schedulers
def populate(devops_env_id):
def populate(ENVIRONMENT):
builders = []
schedulers = []
platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id)
platform_worker_names = conf.machines.fetch_platform_worker_names(ENVIRONMENT)
local_worker_names = conf.machines.fetch_local_worker_names()
worker_config = conf.worker.get_config(devops_env_id)
worker_config = conf.worker.get_config(ENVIRONMENT)
needs_incremental_schedulers = devops_env_id in ["PROD"]
needs_nightly_schedulers = devops_env_id in ["PROD"]
needs_incremental_schedulers = ENVIRONMENT in ["PROD"]
needs_nightly_schedulers = ENVIRONMENT in ["PROD"]
print("*** Creating [code] pipeline")
for track_id in code_track_ids:
@ -493,7 +491,7 @@ def populate(devops_env_id):
step_timeout_in_seconds = compile_gpu_step_timeout_in_seconds
step_command = create_code_worker_command_args.withArgs(
devops_env_id, track_id, pipeline_type, step_name
ENVIRONMENT, track_id, pipeline_type, step_name
)
step = buildbot.plugins.steps.ShellCommand(
@ -512,7 +510,7 @@ def populate(devops_env_id):
for master_step_name in pipeline.common.code_pipeline_master_step_names:
master_step_command = (
pipeline.common.create_master_command_args.withArgs(
devops_env_id,
ENVIRONMENT,
track_id,
pipeline_type,
master_step_name,
@ -536,7 +534,7 @@ def populate(devops_env_id):
pipeline_lint_factory = buildbot.plugins.util.BuildFactory()
for step_name in code_pipeline_lint_step_names:
step_command = create_code_worker_command_args.withArgs(
devops_env_id, track_id, pipeline_type, step_name
ENVIRONMENT, track_id, pipeline_type, step_name
)
pipeline_lint_factory.addStep(
@ -576,7 +574,7 @@ def populate(devops_env_id):
suitable_pipeline_worker_names = pipeline_worker_names
if (
platform_architecture == "linux-x86_64"
and devops_env_id != "LOCAL"
and ENVIRONMENT != "LOCAL"
):
selector = "rocky"
suitable_pipeline_worker_names = [

10
config/pipeline/code_benchmark.py
View file

@ -26,8 +26,8 @@ class LinkMultipleFileUpload(plugins_steps.MultipleFileUpload):
return
def create_deliver_step(devops_env_id):
worker_config = conf.worker.get_config(devops_env_id)
def create_deliver_step(ENVIRONMENT):
worker_config = conf.worker.get_config(ENVIRONMENT)
file_size_in_mb = 500 * 1024 * 1024
worker_source_path = pathlib.Path("../../../../git/blender-vdev/build_package")
@ -48,7 +48,7 @@ def create_deliver_step(devops_env_id):
)
def populate(devops_env_id):
def populate(ENVIRONMENT):
properties = [
buildbot.plugins.util.StringParameter(
name="commit_id",
@ -68,7 +68,7 @@ def populate(devops_env_id):
]
return pipeline.common.create_pipeline(
devops_env_id,
ENVIRONMENT,
"code-benchmark",
"code_benchmark.py",
[
@ -78,7 +78,7 @@ def populate(devops_env_id):
"compile-gpu",
"compile-install",
"benchmark",
partial(create_deliver_step, devops_env_id),
partial(create_deliver_step, ENVIRONMENT),
"clean",
],
{"vdev": "main"},

4
config/pipeline/code_bpy_deploy.py
View file

@ -9,11 +9,11 @@ import conf.branches
import pipeline.common
def populate(devops_env_id):
def populate(ENVIRONMENT):
properties = []
return pipeline.common.create_pipeline(
devops_env_id,
ENVIRONMENT,
"code-bpy-deploy",
"code_bpy_deploy.py",
[

4
config/pipeline/code_deploy.py
View file

@ -10,7 +10,7 @@ import conf.branches
import pipeline.common
def populate(devops_env_id):
def populate(ENVIRONMENT):
properties = [
buildbot.plugins.util.BooleanParameter(
name="needs_full_clean",
@ -22,7 +22,7 @@ def populate(devops_env_id):
]
return pipeline.common.create_pipeline(
devops_env_id,
ENVIRONMENT,
"code-artifacts-deploy",
"code_deploy.py",
[

12
config/pipeline/code_store.py
View file

@ -62,16 +62,16 @@ def create_deliver_binaries_windows_step(worker_config, track_id, pipeline_type)
)
def populate(devops_env_id):
def populate(ENVIRONMENT):
builders = []
schedulers = []
platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id)
platform_worker_names = conf.machines.fetch_platform_worker_names(ENVIRONMENT)
local_worker_names = conf.machines.fetch_local_worker_names()
worker_config = conf.worker.get_config(devops_env_id)
worker_config = conf.worker.get_config(ENVIRONMENT)
needs_nightly_schedulers = devops_env_id == "PROD"
needs_nightly_schedulers = ENVIRONMENT == "PROD"
pipeline_type = "daily"
@ -108,7 +108,7 @@ def populate(devops_env_id):
else:
args = ["--store-id", store_id, step_name]
step_command = pipeline.common.create_worker_command(
"code_store.py", devops_env_id, track_id, args
"code_store.py", ENVIRONMENT, track_id, args
)
step = plugins_steps.ShellCommand(
@ -126,7 +126,7 @@ def populate(devops_env_id):
for master_step_name in pipeline.common.code_pipeline_master_step_names:
master_step_command = (
pipeline.common.create_master_command_args.withArgs(
devops_env_id,
ENVIRONMENT,
track_id,
pipeline_type,
master_step_name,

22
config/pipeline/common.py
View file

@ -57,7 +57,7 @@ def needs_do_doc_pipeline_step(step):
return True
def create_worker_command(script, devops_env_id, track_id, args):
def create_worker_command(script, ENVIRONMENT, track_id, args):
# This relative path assume were are in:
# ~/.devops/services/buildbot-worker/<builder-name>/build
# There appears to be no way to expand a tilde here?
@ -71,7 +71,7 @@ def create_worker_command(script, devops_env_id, track_id, args):
"--track-id",
track_id,
"--service-env-id",
devops_env_id,
ENVIRONMENT,
]
return cmd + list(args)
@ -79,7 +79,7 @@ def create_worker_command(script, devops_env_id, track_id, args):
@buildbot.plugins.util.renderer
def create_master_command_args(
props, devops_env_id, track_id, pipeline_type, step_name, single_platform
props, ENVIRONMENT, track_id, pipeline_type, step_name, single_platform
):
build_configuration = fetch_property(
props, key="build_configuration", default="release"
@ -116,7 +116,7 @@ def create_master_command_args(
"--track-id",
track_id,
"--service-env-id",
devops_env_id,
ENVIRONMENT,
]
return cmd + list(args)
@ -125,7 +125,7 @@ def create_master_command_args(
@buildbot.plugins.util.renderer
def create_pipeline_worker_command(
props,
devops_env_id,
ENVIRONMENT,
track_id,
script,
step_name,
@ -154,11 +154,11 @@ def create_pipeline_worker_command(
if "revision" in props and props["revision"]:
args += ["--commit-id", props["revision"]]
return create_worker_command(script, devops_env_id, track_id, args)
return create_worker_command(script, ENVIRONMENT, track_id, args)
def create_pipeline(
devops_env_id,
ENVIRONMENT,
artifact_id,
script,
steps,
@ -179,13 +179,13 @@ def create_pipeline(
builders = []
schedulers = []
platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id)
platform_worker_names = conf.machines.fetch_platform_worker_names(ENVIRONMENT)
local_worker_names = conf.machines.fetch_local_worker_names()
needs_incremental_schedulers = (
incremental_properties is not None and devops_env_id in ["PROD"]
incremental_properties is not None and ENVIRONMENT in ["PROD"]
)
needs_nightly_schedulers = nightly_properties is not None and devops_env_id in [
needs_nightly_schedulers = nightly_properties is not None and ENVIRONMENT in [
"PROD"
]
track_ids = tracked_branch_ids.keys()
@ -210,7 +210,7 @@ def create_pipeline(
continue
step_command = create_pipeline_worker_command.withArgs(
devops_env_id,
ENVIRONMENT,
track_id,
script,
step,

4
config/pipeline/doc_api.py
View file

@ -8,7 +8,7 @@ import conf.branches
import pipeline.common
def populate(devops_env_id):
def populate(ENVIRONMENT):
properties = [
buildbot.plugins.util.BooleanParameter(
name="needs_full_clean",
@ -27,7 +27,7 @@ def populate(devops_env_id):
]
return pipeline.common.create_pipeline(
devops_env_id,
ENVIRONMENT,
"doc-api",
"doc_api.py",
[

4
config/pipeline/doc_developer.py
View file

@ -7,7 +7,7 @@ import buildbot.plugins
import pipeline.common
def populate(devops_env_id):
def populate(ENVIRONMENT):
properties = [
buildbot.plugins.util.BooleanParameter(
name="needs_package_delivery",
@ -19,7 +19,7 @@ def populate(devops_env_id):
]
return pipeline.common.create_pipeline(
devops_env_id,
ENVIRONMENT,
"doc-developer",
"doc_developer.py",
["update", "compile", "deliver"],

4
config/pipeline/doc_manual.py
View file

@ -8,7 +8,7 @@ import conf.branches
import pipeline.common
def populate(devops_env_id):
def populate(ENVIRONMENT):
properties = [
buildbot.plugins.util.BooleanParameter(
name="needs_package_delivery",
@ -27,7 +27,7 @@ def populate(devops_env_id):
]
return pipeline.common.create_pipeline(
devops_env_id,
ENVIRONMENT,
"doc-manual",
"doc_manual.py",
["configure-machine", "update", "compile", "package", "deliver", "clean"],

4
config/pipeline/doc_studio.py
View file

@ -7,7 +7,7 @@ import buildbot.plugins
import pipeline.common
def populate(devops_env_id):
def populate(ENVIRONMENT):
properties = [
buildbot.plugins.util.BooleanParameter(
name="needs_package_delivery",
@ -19,7 +19,7 @@ def populate(devops_env_id):
]
return pipeline.common.create_pipeline(
devops_env_id,
ENVIRONMENT,
"doc-studio-tools",
"doc_studio.py",
["update", "compile", "deliver"],

44
config/setup.py
View file

@ -31,12 +31,11 @@ importlib.reload(conf.worker)
importlib.reload(gitea.blender)
importlib.reload(pipeline)
devops_env_id = os.environ.get("DEVOPS_ENV_ID", default="LOCAL")
devops_host_id = os.environ.get("DEVOPS_HOST_ID", default="localhost")
ENVIRONMENT = os.environ.get("ENVIRONMENT", default="LOCAL")
def setup() -> Dict[str, Any]:
####### MAIN - configuration
####### CONFIGURATION
c = {}
# Change Source
@ -44,7 +43,7 @@ def setup() -> Dict[str, Any]:
# Workers
print("*** Creating platform workers")
platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id)
platform_worker_names = conf.machines.fetch_platform_worker_names(ENVIRONMENT)
workers: List[buildbot.plugins.worker.Worker] = []
configured_worker_names = set()
for worker_names in platform_worker_names.values():
@ -56,7 +55,7 @@ def setup() -> Dict[str, Any]:
workers += [
buildbot.plugins.worker.Worker(
worker_name,
conf.machines.get_worker_password(devops_env_id, worker_name),
conf.machines.get_worker_password(ENVIRONMENT, worker_name),
max_builds=1,
keepalive_interval=3600,
)
@ -70,7 +69,7 @@ def setup() -> Dict[str, Any]:
c["workers"] = workers
# Builders and Schedulers
builders, schedulers = pipeline.populate(devops_env_id)
builders, schedulers = pipeline.populate(ENVIRONMENT)
c["builders"] = builders
c["schedulers"] = schedulers
@ -80,7 +79,7 @@ def setup() -> Dict[str, Any]:
# status of each build will be pushed to these targets. buildbot/reporters/*.py
# has a variety to choose from, like IRC bots.
gitea_status_service = gitea.blender.setup_service(devops_env_id)
gitea_status_service = gitea.blender.setup_service(ENVIRONMENT)
if gitea_status_service:
c["services"] = [gitea_status_service]
else:
@ -91,42 +90,33 @@ def setup() -> Dict[str, Any]:
# the 'title' string will appear at the top of this buildbot installation's
# home pages (linked to the 'titleURL').
c["title"] = f"Bot - {devops_env_id}"
c["title"] = f"Blender Buildbot - {ENVIRONMENT}"
c["titleURL"] = "https://projects.blender.org"
# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server is visible. This typically uses the port number set in
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.
c["buildbotURL"] = f"http://{devops_host_id}:8010/"
if devops_env_id != "LOCAL":
c["buildbotURL"] = f"http://{devops_host_id}:8000/admin/"
if devops_env_id == "PROD":
c["buildbotURL"] = "https://builder.blender.org/admin/"
if devops_env_id == "UATEST":
c["buildbotURL"] = "https://builder.uatest.blender.org/admin/"
c["buildbotURL"] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/")
# Minimalistic config to activate new web UI
c["www"] = dict(
port=8010, plugins=dict(waterfall_view={}, console_view={}, grid_view={})
port=os.environ.get("BUILDBOT_WEB_PORT", 8010),
plugins=dict(waterfall_view={}, console_view={}, grid_view={}),
)
# Database
if devops_env_id == "LOCAL":
c["db"] = {"db_url": "sqlite:///state.sqlite"}
else:
# PostgreSQL database, as recommended for production environment.
c["db"] = {"db_url": "postgresql://buildbot@127.0.0.1/buildbot"}
c["db"] = {
"db_url": os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ)
}
c["buildbotNetUsageData"] = None
# Authentication
c["www"]["auth"] = conf.auth.fetch_authentication(devops_env_id)
c["www"]["auth"] = conf.auth.fetch_authentication(ENVIRONMENT)
# Authorization
c["www"]["authz"] = conf.auth.fetch_authorization(devops_env_id)
c["www"]["authz"] = conf.auth.fetch_authorization(ENVIRONMENT)
# Disable UI - does not work
c["www"]["plugins"] = {
@ -160,8 +150,8 @@ def setup() -> Dict[str, Any]:
r"https://projects.blender.org/\1/\2/commit/%s",
)
# Port for workers to connectto
c["protocols"] = {"pb": {"port": 9989}}
# Port for workers to connect to
c["protocols"] = {"pb": {"port": os.environ.get("BUILDBOT_WORKER_PORT", 9989)}}
# Disable collapsing requests
c["collapseRequests"] = False

23
docker-compose.override.yml
View file

@ -1,5 +1,24 @@
services:
buildbot-master:
env_file: .env
env_file: .env.local
volumes:
- ./config:/buildbot/config
- ./config:/buildbot/config
buildbot-worker:
image: 'buildbot/buildbot-worker:${BUILDBOT_IMAGE_TAG:-v4.1.0}'
restart: unless-stopped
environment:
- 'BUILDMASTER=${BUILDMASTER:-buildbot-master}'
- 'BUILDMASTER_PORT=${BUILDBOT_WORKER_PORT:-9989}'
- 'WORKERNAME=${WORKERNAME:-example-worker}'
- 'WORKERPASS=${WORKERPASS:-pass}'
- 'WORKER_ENVIRONMENT_BLACKLIST=${WORKER_ENVIRONMENT_BLACKLIST:-DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST}'
healthcheck:
test:
- CMD
- curl
- '-f'
- 'http://$${BUILDMASTER}:$${BUILDMASTER_PORT}'
interval: 5s
timeout: 20s
retries: 10

20
docker-compose.yml
View file

@ -18,7 +18,7 @@ services:
- CMD
- curl
- '-f'
- 'http://localhost:$${BUILDBOT_WEB_PORT}'
- '$${BUILDBOT_WEB_URL}'
interval: 2s
timeout: 10s
retries: 15
@ -41,23 +41,5 @@ services:
- 'POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRESQL}'
- 'POSTGRES_USER=${SERVICE_USER_POSTGRESQL}'
- 'POSTGRES_DB=${POSTGRES_DB:-buildbot}'
buildbot-worker:
image: 'buildbot/buildbot-worker:${BUILDBOT_IMAGE_TAG:-v4.1.0}'
restart: unless-stopped
environment:
- 'BUILDMASTER=${BUILDMASTER:-buildbot-master}'
- 'BUILDMASTER_PORT=${BUILDBOT_WORKER_PORT:-9989}'
- 'WORKERNAME=${WORKERNAME:-example-worker}'
- 'WORKERPASS=${WORKERPASS:-pass}'
- 'WORKER_ENVIRONMENT_BLACKLIST=${WORKER_ENVIRONMENT_BLACKLIST:-DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST}'
healthcheck:
test:
- CMD
- curl
- '-f'
- 'http://$${BUILDMASTER}:$${BUILDMASTER_PORT}'
interval: 5s
timeout: 20s
retries: 10
volumes:
buildbot-db: {}