Change environment variable setup
All checks were successful
/ checks (pull_request) Successful in 15s
All checks were successful
/ checks (pull_request) Successful in 15s
This commit is contained in:
parent
d6bce1b39d
commit
77ae214d24
23 changed files with 113 additions and 122 deletions
|
@ -1,3 +0,0 @@
|
||||||
SERVICE_USER_POSTGRESQL=buildbot
|
|
||||||
SERVICE_PASSWORD_POSTGRESQL=changeme!
|
|
||||||
BUILDBOT_CONFIG_URL=''
|
|
4
.env.local
Normal file
4
.env.local
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
SERVICE_USER_POSTGRESQL=buildbot
|
||||||
|
SERVICE_PASSWORD_POSTGRESQL=changeme!
|
||||||
|
GITEA_URL=https://projects.blender.org
|
||||||
|
BUILDBOT_WEB_URL=http://localhost:8010/
|
3
.gitignore
vendored
3
.gitignore
vendored
|
@ -1,2 +1,3 @@
|
||||||
.venv
|
.venv
|
||||||
.env
|
.env.staging
|
||||||
|
.env.production
|
|
@ -7,8 +7,8 @@ import importlib
|
||||||
import buildbot.plugins
|
import buildbot.plugins
|
||||||
|
|
||||||
|
|
||||||
def _get_auth_config(devops_env_id: str):
|
def _get_auth_config(ENVIRONMENT: str):
|
||||||
if devops_env_id == "LOCAL":
|
if ENVIRONMENT == "LOCAL":
|
||||||
import conf.local.auth
|
import conf.local.auth
|
||||||
|
|
||||||
importlib.reload(conf.local.auth)
|
importlib.reload(conf.local.auth)
|
||||||
|
@ -20,13 +20,13 @@ def _get_auth_config(devops_env_id: str):
|
||||||
return conf.production.auth
|
return conf.production.auth
|
||||||
|
|
||||||
|
|
||||||
def fetch_authentication(devops_env_id: str):
|
def fetch_authentication(ENVIRONMENT: str):
|
||||||
auth_config = _get_auth_config(devops_env_id)
|
auth_config = _get_auth_config(ENVIRONMENT)
|
||||||
return auth_config.get_authentication(devops_env_id)
|
return auth_config.get_authentication(ENVIRONMENT)
|
||||||
|
|
||||||
|
|
||||||
def fetch_authorization(devops_env_id: str):
|
def fetch_authorization(ENVIRONMENT: str):
|
||||||
auth_config = _get_auth_config(devops_env_id)
|
auth_config = _get_auth_config(ENVIRONMENT)
|
||||||
|
|
||||||
admin_usernames = auth_config.admin_usernames
|
admin_usernames = auth_config.admin_usernames
|
||||||
deploy_dev_usernames = auth_config.deploy_dev_usernames
|
deploy_dev_usernames = auth_config.deploy_dev_usernames
|
||||||
|
|
|
@ -20,7 +20,7 @@ trusted_dev_usernames = [
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def get_authentication(devops_env_id: str):
|
def get_authentication(ENVIRONMENT: str):
|
||||||
class LocalEnvAuth(buildbot.plugins.util.CustomAuth):
|
class LocalEnvAuth(buildbot.plugins.util.CustomAuth):
|
||||||
def check_credentials(self, user, password):
|
def check_credentials(self, user, password):
|
||||||
return user.decode() == "admin" and password.decode() == "admin"
|
return user.decode() == "admin" and password.decode() == "admin"
|
||||||
|
|
|
@ -27,5 +27,5 @@ def get_worker_password(worker_name: str) -> str:
|
||||||
return "localhost"
|
return "localhost"
|
||||||
|
|
||||||
|
|
||||||
def get_worker_names(devops_env_id: str):
|
def get_worker_names(ENVIRONMENT: str):
|
||||||
return _worker_names
|
return _worker_names
|
||||||
|
|
|
@ -5,8 +5,8 @@
|
||||||
import importlib
|
import importlib
|
||||||
|
|
||||||
|
|
||||||
def _get_config(devops_env_id: str):
|
def _get_config(ENVIRONMENT: str):
|
||||||
if devops_env_id == "LOCAL":
|
if ENVIRONMENT == "LOCAL":
|
||||||
import conf.local.machines
|
import conf.local.machines
|
||||||
|
|
||||||
importlib.reload(conf.local.machines)
|
importlib.reload(conf.local.machines)
|
||||||
|
@ -18,13 +18,13 @@ def _get_config(devops_env_id: str):
|
||||||
return conf.production.machines
|
return conf.production.machines
|
||||||
|
|
||||||
|
|
||||||
def fetch_platform_worker_names(devops_env_id: str):
|
def fetch_platform_worker_names(ENVIRONMENT: str):
|
||||||
machines_config = _get_config(devops_env_id)
|
machines_config = _get_config(ENVIRONMENT)
|
||||||
return machines_config.get_worker_names(devops_env_id)
|
return machines_config.get_worker_names(ENVIRONMENT)
|
||||||
|
|
||||||
|
|
||||||
def get_worker_password(devops_env_id: str, worker_name: str) -> str:
|
def get_worker_password(ENVIRONMENT: str, worker_name: str) -> str:
|
||||||
machines_config = _get_config(devops_env_id)
|
machines_config = _get_config(ENVIRONMENT)
|
||||||
return machines_config.get_worker_password(worker_name)
|
return machines_config.get_worker_password(worker_name)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -7,8 +7,8 @@ import importlib
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
def get_config(devops_env_id: str) -> Any:
|
def get_config(ENVIRONMENT: str) -> Any:
|
||||||
if devops_env_id == "LOCAL":
|
if ENVIRONMENT == "LOCAL":
|
||||||
import conf.local.worker
|
import conf.local.worker
|
||||||
|
|
||||||
importlib.reload(conf.local.worker)
|
importlib.reload(conf.local.worker)
|
||||||
|
|
|
@ -22,12 +22,12 @@ gitea_api_token = None
|
||||||
gitea_status_service = None
|
gitea_status_service = None
|
||||||
|
|
||||||
|
|
||||||
def setup_service(devops_env_id: str):
|
def setup_service(ENVIRONMENT: str):
|
||||||
import conf.worker
|
import conf.worker
|
||||||
|
|
||||||
importlib.reload(conf.worker)
|
importlib.reload(conf.worker)
|
||||||
worker_config = conf.worker.get_config(devops_env_id)
|
worker_config = conf.worker.get_config(ENVIRONMENT)
|
||||||
gitea_api_token = worker_config.gitea_api_token(devops_env_id)
|
gitea_api_token = worker_config.gitea_api_token(ENVIRONMENT)
|
||||||
|
|
||||||
if gitea_api_token:
|
if gitea_api_token:
|
||||||
log.msg("Found Gitea API token, enabling status push")
|
log.msg("Found Gitea API token, enabling status push")
|
||||||
|
|
|
@ -23,7 +23,7 @@ importlib.reload(pipeline.common)
|
||||||
importlib.reload(conf.branches)
|
importlib.reload(conf.branches)
|
||||||
|
|
||||||
|
|
||||||
def populate(devops_env_id):
|
def populate(ENVIRONMENT):
|
||||||
pipelines_modules = [
|
pipelines_modules = [
|
||||||
pipeline.code,
|
pipeline.code,
|
||||||
pipeline.code_benchmark,
|
pipeline.code_benchmark,
|
||||||
|
@ -41,7 +41,7 @@ def populate(devops_env_id):
|
||||||
|
|
||||||
for pipelines_module in pipelines_modules:
|
for pipelines_module in pipelines_modules:
|
||||||
importlib.reload(pipelines_module)
|
importlib.reload(pipelines_module)
|
||||||
b, s = pipelines_module.populate(devops_env_id)
|
b, s = pipelines_module.populate(ENVIRONMENT)
|
||||||
builders += b
|
builders += b
|
||||||
schedulers += s
|
schedulers += s
|
||||||
|
|
||||||
|
|
|
@ -233,7 +233,7 @@ scheduler_properties = {
|
||||||
|
|
||||||
@buildbot.plugins.util.renderer
|
@buildbot.plugins.util.renderer
|
||||||
def create_code_worker_command_args(
|
def create_code_worker_command_args(
|
||||||
props, devops_env_id, track_id, pipeline_type, step_name
|
props, ENVIRONMENT, track_id, pipeline_type, step_name
|
||||||
):
|
):
|
||||||
commit_id = pipeline.common.fetch_property(props, key="revision", default="HEAD")
|
commit_id = pipeline.common.fetch_property(props, key="revision", default="HEAD")
|
||||||
patch_id = pipeline.common.fetch_property(props, key="patch_id", default="")
|
patch_id = pipeline.common.fetch_property(props, key="patch_id", default="")
|
||||||
|
@ -294,9 +294,7 @@ def create_code_worker_command_args(
|
||||||
|
|
||||||
args += [step_name]
|
args += [step_name]
|
||||||
|
|
||||||
return pipeline.common.create_worker_command(
|
return pipeline.common.create_worker_command("code.py", ENVIRONMENT, track_id, args)
|
||||||
"code.py", devops_env_id, track_id, args
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def needs_do_code_pipeline_step(step):
|
def needs_do_code_pipeline_step(step):
|
||||||
|
@ -451,17 +449,17 @@ class PlatformTrigger(plugins_steps.Trigger):
|
||||||
return schedulers
|
return schedulers
|
||||||
|
|
||||||
|
|
||||||
def populate(devops_env_id):
|
def populate(ENVIRONMENT):
|
||||||
builders = []
|
builders = []
|
||||||
schedulers = []
|
schedulers = []
|
||||||
|
|
||||||
platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id)
|
platform_worker_names = conf.machines.fetch_platform_worker_names(ENVIRONMENT)
|
||||||
local_worker_names = conf.machines.fetch_local_worker_names()
|
local_worker_names = conf.machines.fetch_local_worker_names()
|
||||||
|
|
||||||
worker_config = conf.worker.get_config(devops_env_id)
|
worker_config = conf.worker.get_config(ENVIRONMENT)
|
||||||
|
|
||||||
needs_incremental_schedulers = devops_env_id in ["PROD"]
|
needs_incremental_schedulers = ENVIRONMENT in ["PROD"]
|
||||||
needs_nightly_schedulers = devops_env_id in ["PROD"]
|
needs_nightly_schedulers = ENVIRONMENT in ["PROD"]
|
||||||
|
|
||||||
print("*** Creating [code] pipeline")
|
print("*** Creating [code] pipeline")
|
||||||
for track_id in code_track_ids:
|
for track_id in code_track_ids:
|
||||||
|
@ -493,7 +491,7 @@ def populate(devops_env_id):
|
||||||
step_timeout_in_seconds = compile_gpu_step_timeout_in_seconds
|
step_timeout_in_seconds = compile_gpu_step_timeout_in_seconds
|
||||||
|
|
||||||
step_command = create_code_worker_command_args.withArgs(
|
step_command = create_code_worker_command_args.withArgs(
|
||||||
devops_env_id, track_id, pipeline_type, step_name
|
ENVIRONMENT, track_id, pipeline_type, step_name
|
||||||
)
|
)
|
||||||
|
|
||||||
step = buildbot.plugins.steps.ShellCommand(
|
step = buildbot.plugins.steps.ShellCommand(
|
||||||
|
@ -512,7 +510,7 @@ def populate(devops_env_id):
|
||||||
for master_step_name in pipeline.common.code_pipeline_master_step_names:
|
for master_step_name in pipeline.common.code_pipeline_master_step_names:
|
||||||
master_step_command = (
|
master_step_command = (
|
||||||
pipeline.common.create_master_command_args.withArgs(
|
pipeline.common.create_master_command_args.withArgs(
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
track_id,
|
track_id,
|
||||||
pipeline_type,
|
pipeline_type,
|
||||||
master_step_name,
|
master_step_name,
|
||||||
|
@ -536,7 +534,7 @@ def populate(devops_env_id):
|
||||||
pipeline_lint_factory = buildbot.plugins.util.BuildFactory()
|
pipeline_lint_factory = buildbot.plugins.util.BuildFactory()
|
||||||
for step_name in code_pipeline_lint_step_names:
|
for step_name in code_pipeline_lint_step_names:
|
||||||
step_command = create_code_worker_command_args.withArgs(
|
step_command = create_code_worker_command_args.withArgs(
|
||||||
devops_env_id, track_id, pipeline_type, step_name
|
ENVIRONMENT, track_id, pipeline_type, step_name
|
||||||
)
|
)
|
||||||
|
|
||||||
pipeline_lint_factory.addStep(
|
pipeline_lint_factory.addStep(
|
||||||
|
@ -576,7 +574,7 @@ def populate(devops_env_id):
|
||||||
suitable_pipeline_worker_names = pipeline_worker_names
|
suitable_pipeline_worker_names = pipeline_worker_names
|
||||||
if (
|
if (
|
||||||
platform_architecture == "linux-x86_64"
|
platform_architecture == "linux-x86_64"
|
||||||
and devops_env_id != "LOCAL"
|
and ENVIRONMENT != "LOCAL"
|
||||||
):
|
):
|
||||||
selector = "rocky"
|
selector = "rocky"
|
||||||
suitable_pipeline_worker_names = [
|
suitable_pipeline_worker_names = [
|
||||||
|
|
|
@ -26,8 +26,8 @@ class LinkMultipleFileUpload(plugins_steps.MultipleFileUpload):
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
def create_deliver_step(devops_env_id):
|
def create_deliver_step(ENVIRONMENT):
|
||||||
worker_config = conf.worker.get_config(devops_env_id)
|
worker_config = conf.worker.get_config(ENVIRONMENT)
|
||||||
|
|
||||||
file_size_in_mb = 500 * 1024 * 1024
|
file_size_in_mb = 500 * 1024 * 1024
|
||||||
worker_source_path = pathlib.Path("../../../../git/blender-vdev/build_package")
|
worker_source_path = pathlib.Path("../../../../git/blender-vdev/build_package")
|
||||||
|
@ -48,7 +48,7 @@ def create_deliver_step(devops_env_id):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def populate(devops_env_id):
|
def populate(ENVIRONMENT):
|
||||||
properties = [
|
properties = [
|
||||||
buildbot.plugins.util.StringParameter(
|
buildbot.plugins.util.StringParameter(
|
||||||
name="commit_id",
|
name="commit_id",
|
||||||
|
@ -68,7 +68,7 @@ def populate(devops_env_id):
|
||||||
]
|
]
|
||||||
|
|
||||||
return pipeline.common.create_pipeline(
|
return pipeline.common.create_pipeline(
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
"code-benchmark",
|
"code-benchmark",
|
||||||
"code_benchmark.py",
|
"code_benchmark.py",
|
||||||
[
|
[
|
||||||
|
@ -78,7 +78,7 @@ def populate(devops_env_id):
|
||||||
"compile-gpu",
|
"compile-gpu",
|
||||||
"compile-install",
|
"compile-install",
|
||||||
"benchmark",
|
"benchmark",
|
||||||
partial(create_deliver_step, devops_env_id),
|
partial(create_deliver_step, ENVIRONMENT),
|
||||||
"clean",
|
"clean",
|
||||||
],
|
],
|
||||||
{"vdev": "main"},
|
{"vdev": "main"},
|
||||||
|
|
|
@ -9,11 +9,11 @@ import conf.branches
|
||||||
import pipeline.common
|
import pipeline.common
|
||||||
|
|
||||||
|
|
||||||
def populate(devops_env_id):
|
def populate(ENVIRONMENT):
|
||||||
properties = []
|
properties = []
|
||||||
|
|
||||||
return pipeline.common.create_pipeline(
|
return pipeline.common.create_pipeline(
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
"code-bpy-deploy",
|
"code-bpy-deploy",
|
||||||
"code_bpy_deploy.py",
|
"code_bpy_deploy.py",
|
||||||
[
|
[
|
||||||
|
|
|
@ -10,7 +10,7 @@ import conf.branches
|
||||||
import pipeline.common
|
import pipeline.common
|
||||||
|
|
||||||
|
|
||||||
def populate(devops_env_id):
|
def populate(ENVIRONMENT):
|
||||||
properties = [
|
properties = [
|
||||||
buildbot.plugins.util.BooleanParameter(
|
buildbot.plugins.util.BooleanParameter(
|
||||||
name="needs_full_clean",
|
name="needs_full_clean",
|
||||||
|
@ -22,7 +22,7 @@ def populate(devops_env_id):
|
||||||
]
|
]
|
||||||
|
|
||||||
return pipeline.common.create_pipeline(
|
return pipeline.common.create_pipeline(
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
"code-artifacts-deploy",
|
"code-artifacts-deploy",
|
||||||
"code_deploy.py",
|
"code_deploy.py",
|
||||||
[
|
[
|
||||||
|
|
|
@ -62,16 +62,16 @@ def create_deliver_binaries_windows_step(worker_config, track_id, pipeline_type)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def populate(devops_env_id):
|
def populate(ENVIRONMENT):
|
||||||
builders = []
|
builders = []
|
||||||
schedulers = []
|
schedulers = []
|
||||||
|
|
||||||
platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id)
|
platform_worker_names = conf.machines.fetch_platform_worker_names(ENVIRONMENT)
|
||||||
local_worker_names = conf.machines.fetch_local_worker_names()
|
local_worker_names = conf.machines.fetch_local_worker_names()
|
||||||
|
|
||||||
worker_config = conf.worker.get_config(devops_env_id)
|
worker_config = conf.worker.get_config(ENVIRONMENT)
|
||||||
|
|
||||||
needs_nightly_schedulers = devops_env_id == "PROD"
|
needs_nightly_schedulers = ENVIRONMENT == "PROD"
|
||||||
|
|
||||||
pipeline_type = "daily"
|
pipeline_type = "daily"
|
||||||
|
|
||||||
|
@ -108,7 +108,7 @@ def populate(devops_env_id):
|
||||||
else:
|
else:
|
||||||
args = ["--store-id", store_id, step_name]
|
args = ["--store-id", store_id, step_name]
|
||||||
step_command = pipeline.common.create_worker_command(
|
step_command = pipeline.common.create_worker_command(
|
||||||
"code_store.py", devops_env_id, track_id, args
|
"code_store.py", ENVIRONMENT, track_id, args
|
||||||
)
|
)
|
||||||
|
|
||||||
step = plugins_steps.ShellCommand(
|
step = plugins_steps.ShellCommand(
|
||||||
|
@ -126,7 +126,7 @@ def populate(devops_env_id):
|
||||||
for master_step_name in pipeline.common.code_pipeline_master_step_names:
|
for master_step_name in pipeline.common.code_pipeline_master_step_names:
|
||||||
master_step_command = (
|
master_step_command = (
|
||||||
pipeline.common.create_master_command_args.withArgs(
|
pipeline.common.create_master_command_args.withArgs(
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
track_id,
|
track_id,
|
||||||
pipeline_type,
|
pipeline_type,
|
||||||
master_step_name,
|
master_step_name,
|
||||||
|
|
|
@ -57,7 +57,7 @@ def needs_do_doc_pipeline_step(step):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def create_worker_command(script, devops_env_id, track_id, args):
|
def create_worker_command(script, ENVIRONMENT, track_id, args):
|
||||||
# This relative path assume were are in:
|
# This relative path assume were are in:
|
||||||
# ~/.devops/services/buildbot-worker/<builder-name>/build
|
# ~/.devops/services/buildbot-worker/<builder-name>/build
|
||||||
# There appears to be no way to expand a tilde here?
|
# There appears to be no way to expand a tilde here?
|
||||||
|
@ -71,7 +71,7 @@ def create_worker_command(script, devops_env_id, track_id, args):
|
||||||
"--track-id",
|
"--track-id",
|
||||||
track_id,
|
track_id,
|
||||||
"--service-env-id",
|
"--service-env-id",
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
]
|
]
|
||||||
|
|
||||||
return cmd + list(args)
|
return cmd + list(args)
|
||||||
|
@ -79,7 +79,7 @@ def create_worker_command(script, devops_env_id, track_id, args):
|
||||||
|
|
||||||
@buildbot.plugins.util.renderer
|
@buildbot.plugins.util.renderer
|
||||||
def create_master_command_args(
|
def create_master_command_args(
|
||||||
props, devops_env_id, track_id, pipeline_type, step_name, single_platform
|
props, ENVIRONMENT, track_id, pipeline_type, step_name, single_platform
|
||||||
):
|
):
|
||||||
build_configuration = fetch_property(
|
build_configuration = fetch_property(
|
||||||
props, key="build_configuration", default="release"
|
props, key="build_configuration", default="release"
|
||||||
|
@ -116,7 +116,7 @@ def create_master_command_args(
|
||||||
"--track-id",
|
"--track-id",
|
||||||
track_id,
|
track_id,
|
||||||
"--service-env-id",
|
"--service-env-id",
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
]
|
]
|
||||||
|
|
||||||
return cmd + list(args)
|
return cmd + list(args)
|
||||||
|
@ -125,7 +125,7 @@ def create_master_command_args(
|
||||||
@buildbot.plugins.util.renderer
|
@buildbot.plugins.util.renderer
|
||||||
def create_pipeline_worker_command(
|
def create_pipeline_worker_command(
|
||||||
props,
|
props,
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
track_id,
|
track_id,
|
||||||
script,
|
script,
|
||||||
step_name,
|
step_name,
|
||||||
|
@ -154,11 +154,11 @@ def create_pipeline_worker_command(
|
||||||
if "revision" in props and props["revision"]:
|
if "revision" in props and props["revision"]:
|
||||||
args += ["--commit-id", props["revision"]]
|
args += ["--commit-id", props["revision"]]
|
||||||
|
|
||||||
return create_worker_command(script, devops_env_id, track_id, args)
|
return create_worker_command(script, ENVIRONMENT, track_id, args)
|
||||||
|
|
||||||
|
|
||||||
def create_pipeline(
|
def create_pipeline(
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
artifact_id,
|
artifact_id,
|
||||||
script,
|
script,
|
||||||
steps,
|
steps,
|
||||||
|
@ -179,13 +179,13 @@ def create_pipeline(
|
||||||
builders = []
|
builders = []
|
||||||
schedulers = []
|
schedulers = []
|
||||||
|
|
||||||
platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id)
|
platform_worker_names = conf.machines.fetch_platform_worker_names(ENVIRONMENT)
|
||||||
local_worker_names = conf.machines.fetch_local_worker_names()
|
local_worker_names = conf.machines.fetch_local_worker_names()
|
||||||
|
|
||||||
needs_incremental_schedulers = (
|
needs_incremental_schedulers = (
|
||||||
incremental_properties is not None and devops_env_id in ["PROD"]
|
incremental_properties is not None and ENVIRONMENT in ["PROD"]
|
||||||
)
|
)
|
||||||
needs_nightly_schedulers = nightly_properties is not None and devops_env_id in [
|
needs_nightly_schedulers = nightly_properties is not None and ENVIRONMENT in [
|
||||||
"PROD"
|
"PROD"
|
||||||
]
|
]
|
||||||
track_ids = tracked_branch_ids.keys()
|
track_ids = tracked_branch_ids.keys()
|
||||||
|
@ -210,7 +210,7 @@ def create_pipeline(
|
||||||
continue
|
continue
|
||||||
|
|
||||||
step_command = create_pipeline_worker_command.withArgs(
|
step_command = create_pipeline_worker_command.withArgs(
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
track_id,
|
track_id,
|
||||||
script,
|
script,
|
||||||
step,
|
step,
|
||||||
|
|
|
@ -8,7 +8,7 @@ import conf.branches
|
||||||
import pipeline.common
|
import pipeline.common
|
||||||
|
|
||||||
|
|
||||||
def populate(devops_env_id):
|
def populate(ENVIRONMENT):
|
||||||
properties = [
|
properties = [
|
||||||
buildbot.plugins.util.BooleanParameter(
|
buildbot.plugins.util.BooleanParameter(
|
||||||
name="needs_full_clean",
|
name="needs_full_clean",
|
||||||
|
@ -27,7 +27,7 @@ def populate(devops_env_id):
|
||||||
]
|
]
|
||||||
|
|
||||||
return pipeline.common.create_pipeline(
|
return pipeline.common.create_pipeline(
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
"doc-api",
|
"doc-api",
|
||||||
"doc_api.py",
|
"doc_api.py",
|
||||||
[
|
[
|
||||||
|
|
|
@ -7,7 +7,7 @@ import buildbot.plugins
|
||||||
import pipeline.common
|
import pipeline.common
|
||||||
|
|
||||||
|
|
||||||
def populate(devops_env_id):
|
def populate(ENVIRONMENT):
|
||||||
properties = [
|
properties = [
|
||||||
buildbot.plugins.util.BooleanParameter(
|
buildbot.plugins.util.BooleanParameter(
|
||||||
name="needs_package_delivery",
|
name="needs_package_delivery",
|
||||||
|
@ -19,7 +19,7 @@ def populate(devops_env_id):
|
||||||
]
|
]
|
||||||
|
|
||||||
return pipeline.common.create_pipeline(
|
return pipeline.common.create_pipeline(
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
"doc-developer",
|
"doc-developer",
|
||||||
"doc_developer.py",
|
"doc_developer.py",
|
||||||
["update", "compile", "deliver"],
|
["update", "compile", "deliver"],
|
||||||
|
|
|
@ -8,7 +8,7 @@ import conf.branches
|
||||||
import pipeline.common
|
import pipeline.common
|
||||||
|
|
||||||
|
|
||||||
def populate(devops_env_id):
|
def populate(ENVIRONMENT):
|
||||||
properties = [
|
properties = [
|
||||||
buildbot.plugins.util.BooleanParameter(
|
buildbot.plugins.util.BooleanParameter(
|
||||||
name="needs_package_delivery",
|
name="needs_package_delivery",
|
||||||
|
@ -27,7 +27,7 @@ def populate(devops_env_id):
|
||||||
]
|
]
|
||||||
|
|
||||||
return pipeline.common.create_pipeline(
|
return pipeline.common.create_pipeline(
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
"doc-manual",
|
"doc-manual",
|
||||||
"doc_manual.py",
|
"doc_manual.py",
|
||||||
["configure-machine", "update", "compile", "package", "deliver", "clean"],
|
["configure-machine", "update", "compile", "package", "deliver", "clean"],
|
||||||
|
|
|
@ -7,7 +7,7 @@ import buildbot.plugins
|
||||||
import pipeline.common
|
import pipeline.common
|
||||||
|
|
||||||
|
|
||||||
def populate(devops_env_id):
|
def populate(ENVIRONMENT):
|
||||||
properties = [
|
properties = [
|
||||||
buildbot.plugins.util.BooleanParameter(
|
buildbot.plugins.util.BooleanParameter(
|
||||||
name="needs_package_delivery",
|
name="needs_package_delivery",
|
||||||
|
@ -19,7 +19,7 @@ def populate(devops_env_id):
|
||||||
]
|
]
|
||||||
|
|
||||||
return pipeline.common.create_pipeline(
|
return pipeline.common.create_pipeline(
|
||||||
devops_env_id,
|
ENVIRONMENT,
|
||||||
"doc-studio-tools",
|
"doc-studio-tools",
|
||||||
"doc_studio.py",
|
"doc_studio.py",
|
||||||
["update", "compile", "deliver"],
|
["update", "compile", "deliver"],
|
||||||
|
|
|
@ -31,12 +31,11 @@ importlib.reload(conf.worker)
|
||||||
importlib.reload(gitea.blender)
|
importlib.reload(gitea.blender)
|
||||||
importlib.reload(pipeline)
|
importlib.reload(pipeline)
|
||||||
|
|
||||||
devops_env_id = os.environ.get("DEVOPS_ENV_ID", default="LOCAL")
|
ENVIRONMENT = os.environ.get("ENVIRONMENT", default="LOCAL")
|
||||||
devops_host_id = os.environ.get("DEVOPS_HOST_ID", default="localhost")
|
|
||||||
|
|
||||||
|
|
||||||
def setup() -> Dict[str, Any]:
|
def setup() -> Dict[str, Any]:
|
||||||
####### MAIN - configuration
|
####### CONFIGURATION
|
||||||
c = {}
|
c = {}
|
||||||
|
|
||||||
# Change Source
|
# Change Source
|
||||||
|
@ -44,7 +43,7 @@ def setup() -> Dict[str, Any]:
|
||||||
|
|
||||||
# Workers
|
# Workers
|
||||||
print("*** Creating platform workers")
|
print("*** Creating platform workers")
|
||||||
platform_worker_names = conf.machines.fetch_platform_worker_names(devops_env_id)
|
platform_worker_names = conf.machines.fetch_platform_worker_names(ENVIRONMENT)
|
||||||
workers: List[buildbot.plugins.worker.Worker] = []
|
workers: List[buildbot.plugins.worker.Worker] = []
|
||||||
configured_worker_names = set()
|
configured_worker_names = set()
|
||||||
for worker_names in platform_worker_names.values():
|
for worker_names in platform_worker_names.values():
|
||||||
|
@ -56,7 +55,7 @@ def setup() -> Dict[str, Any]:
|
||||||
workers += [
|
workers += [
|
||||||
buildbot.plugins.worker.Worker(
|
buildbot.plugins.worker.Worker(
|
||||||
worker_name,
|
worker_name,
|
||||||
conf.machines.get_worker_password(devops_env_id, worker_name),
|
conf.machines.get_worker_password(ENVIRONMENT, worker_name),
|
||||||
max_builds=1,
|
max_builds=1,
|
||||||
keepalive_interval=3600,
|
keepalive_interval=3600,
|
||||||
)
|
)
|
||||||
|
@ -70,7 +69,7 @@ def setup() -> Dict[str, Any]:
|
||||||
c["workers"] = workers
|
c["workers"] = workers
|
||||||
|
|
||||||
# Builders and Schedulers
|
# Builders and Schedulers
|
||||||
builders, schedulers = pipeline.populate(devops_env_id)
|
builders, schedulers = pipeline.populate(ENVIRONMENT)
|
||||||
c["builders"] = builders
|
c["builders"] = builders
|
||||||
c["schedulers"] = schedulers
|
c["schedulers"] = schedulers
|
||||||
|
|
||||||
|
@ -80,7 +79,7 @@ def setup() -> Dict[str, Any]:
|
||||||
# status of each build will be pushed to these targets. buildbot/reporters/*.py
|
# status of each build will be pushed to these targets. buildbot/reporters/*.py
|
||||||
# has a variety to choose from, like IRC bots.
|
# has a variety to choose from, like IRC bots.
|
||||||
|
|
||||||
gitea_status_service = gitea.blender.setup_service(devops_env_id)
|
gitea_status_service = gitea.blender.setup_service(ENVIRONMENT)
|
||||||
if gitea_status_service:
|
if gitea_status_service:
|
||||||
c["services"] = [gitea_status_service]
|
c["services"] = [gitea_status_service]
|
||||||
else:
|
else:
|
||||||
|
@ -91,42 +90,33 @@ def setup() -> Dict[str, Any]:
|
||||||
# the 'title' string will appear at the top of this buildbot installation's
|
# the 'title' string will appear at the top of this buildbot installation's
|
||||||
# home pages (linked to the 'titleURL').
|
# home pages (linked to the 'titleURL').
|
||||||
|
|
||||||
c["title"] = f"Bot - {devops_env_id}"
|
c["title"] = f"Blender Buildbot - {ENVIRONMENT}"
|
||||||
c["titleURL"] = "https://projects.blender.org"
|
c["titleURL"] = "https://projects.blender.org"
|
||||||
|
|
||||||
# the 'buildbotURL' string should point to the location where the buildbot's
|
# the 'buildbotURL' string should point to the location where the buildbot's
|
||||||
# internal web server is visible. This typically uses the port number set in
|
# internal web server is visible. This typically uses the port number set in
|
||||||
# the 'www' entry below, but with an externally-visible host name which the
|
# the 'www' entry below, but with an externally-visible host name which the
|
||||||
# buildbot cannot figure out without some help.
|
# buildbot cannot figure out without some help.
|
||||||
c["buildbotURL"] = f"http://{devops_host_id}:8010/"
|
c["buildbotURL"] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/")
|
||||||
|
|
||||||
if devops_env_id != "LOCAL":
|
|
||||||
c["buildbotURL"] = f"http://{devops_host_id}:8000/admin/"
|
|
||||||
|
|
||||||
if devops_env_id == "PROD":
|
|
||||||
c["buildbotURL"] = "https://builder.blender.org/admin/"
|
|
||||||
if devops_env_id == "UATEST":
|
|
||||||
c["buildbotURL"] = "https://builder.uatest.blender.org/admin/"
|
|
||||||
|
|
||||||
# Minimalistic config to activate new web UI
|
# Minimalistic config to activate new web UI
|
||||||
c["www"] = dict(
|
c["www"] = dict(
|
||||||
port=8010, plugins=dict(waterfall_view={}, console_view={}, grid_view={})
|
port=os.environ.get("BUILDBOT_WEB_PORT", 8010),
|
||||||
|
plugins=dict(waterfall_view={}, console_view={}, grid_view={}),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Database
|
# Database
|
||||||
if devops_env_id == "LOCAL":
|
c["db"] = {
|
||||||
c["db"] = {"db_url": "sqlite:///state.sqlite"}
|
"db_url": os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ)
|
||||||
else:
|
}
|
||||||
# PostgreSQL database, as recommended for production environment.
|
|
||||||
c["db"] = {"db_url": "postgresql://buildbot@127.0.0.1/buildbot"}
|
|
||||||
|
|
||||||
c["buildbotNetUsageData"] = None
|
c["buildbotNetUsageData"] = None
|
||||||
|
|
||||||
# Authentication
|
# Authentication
|
||||||
c["www"]["auth"] = conf.auth.fetch_authentication(devops_env_id)
|
c["www"]["auth"] = conf.auth.fetch_authentication(ENVIRONMENT)
|
||||||
|
|
||||||
# Authorization
|
# Authorization
|
||||||
c["www"]["authz"] = conf.auth.fetch_authorization(devops_env_id)
|
c["www"]["authz"] = conf.auth.fetch_authorization(ENVIRONMENT)
|
||||||
|
|
||||||
# Disable UI - does not work
|
# Disable UI - does not work
|
||||||
c["www"]["plugins"] = {
|
c["www"]["plugins"] = {
|
||||||
|
@ -160,8 +150,8 @@ def setup() -> Dict[str, Any]:
|
||||||
r"https://projects.blender.org/\1/\2/commit/%s",
|
r"https://projects.blender.org/\1/\2/commit/%s",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Port for workers to connectto
|
# Port for workers to connect to
|
||||||
c["protocols"] = {"pb": {"port": 9989}}
|
c["protocols"] = {"pb": {"port": os.environ.get("BUILDBOT_WORKER_PORT", 9989)}}
|
||||||
|
|
||||||
# Disable collapsing requests
|
# Disable collapsing requests
|
||||||
c["collapseRequests"] = False
|
c["collapseRequests"] = False
|
||||||
|
|
|
@ -1,5 +1,24 @@
|
||||||
services:
|
services:
|
||||||
buildbot-master:
|
buildbot-master:
|
||||||
env_file: .env
|
env_file: .env.local
|
||||||
volumes:
|
volumes:
|
||||||
- ./config:/buildbot/config
|
- ./config:/buildbot/config
|
||||||
|
|
||||||
|
buildbot-worker:
|
||||||
|
image: 'buildbot/buildbot-worker:${BUILDBOT_IMAGE_TAG:-v4.1.0}'
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
- 'BUILDMASTER=${BUILDMASTER:-buildbot-master}'
|
||||||
|
- 'BUILDMASTER_PORT=${BUILDBOT_WORKER_PORT:-9989}'
|
||||||
|
- 'WORKERNAME=${WORKERNAME:-example-worker}'
|
||||||
|
- 'WORKERPASS=${WORKERPASS:-pass}'
|
||||||
|
- 'WORKER_ENVIRONMENT_BLACKLIST=${WORKER_ENVIRONMENT_BLACKLIST:-DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST}'
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
- CMD
|
||||||
|
- curl
|
||||||
|
- '-f'
|
||||||
|
- 'http://$${BUILDMASTER}:$${BUILDMASTER_PORT}'
|
||||||
|
interval: 5s
|
||||||
|
timeout: 20s
|
||||||
|
retries: 10
|
|
@ -18,7 +18,7 @@ services:
|
||||||
- CMD
|
- CMD
|
||||||
- curl
|
- curl
|
||||||
- '-f'
|
- '-f'
|
||||||
- 'http://localhost:$${BUILDBOT_WEB_PORT}'
|
- '$${BUILDBOT_WEB_URL}'
|
||||||
interval: 2s
|
interval: 2s
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
retries: 15
|
retries: 15
|
||||||
|
@ -41,23 +41,5 @@ services:
|
||||||
- 'POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRESQL}'
|
- 'POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRESQL}'
|
||||||
- 'POSTGRES_USER=${SERVICE_USER_POSTGRESQL}'
|
- 'POSTGRES_USER=${SERVICE_USER_POSTGRESQL}'
|
||||||
- 'POSTGRES_DB=${POSTGRES_DB:-buildbot}'
|
- 'POSTGRES_DB=${POSTGRES_DB:-buildbot}'
|
||||||
buildbot-worker:
|
|
||||||
image: 'buildbot/buildbot-worker:${BUILDBOT_IMAGE_TAG:-v4.1.0}'
|
|
||||||
restart: unless-stopped
|
|
||||||
environment:
|
|
||||||
- 'BUILDMASTER=${BUILDMASTER:-buildbot-master}'
|
|
||||||
- 'BUILDMASTER_PORT=${BUILDBOT_WORKER_PORT:-9989}'
|
|
||||||
- 'WORKERNAME=${WORKERNAME:-example-worker}'
|
|
||||||
- 'WORKERPASS=${WORKERPASS:-pass}'
|
|
||||||
- 'WORKER_ENVIRONMENT_BLACKLIST=${WORKER_ENVIRONMENT_BLACKLIST:-DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST}'
|
|
||||||
healthcheck:
|
|
||||||
test:
|
|
||||||
- CMD
|
|
||||||
- curl
|
|
||||||
- '-f'
|
|
||||||
- 'http://$${BUILDMASTER}:$${BUILDMASTER_PORT}'
|
|
||||||
interval: 5s
|
|
||||||
timeout: 20s
|
|
||||||
retries: 10
|
|
||||||
volumes:
|
volumes:
|
||||||
buildbot-db: {}
|
buildbot-db: {}
|
Loading…
Reference in a new issue