commit: f43ba834fa9a6ef83d81f6e5512078ac741a6ac6 Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Mon Dec 28 16:03:12 2020 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Mon Dec 28 16:15:59 2020 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f43ba834
Make upgrade_master work Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org> README.txt | 8 + buildbot_gentoo_ci/__init__.py | 0 buildbot_gentoo_ci/config/__init__.py | 0 buildbot_gentoo_ci/config/builders.py | 9 + buildbot_gentoo_ci/config/buildfactory.py | 8 + buildbot_gentoo_ci/config/config.py | 135 ++++++++ buildbot_gentoo_ci/config/schedulers.py | 56 ++++ buildbot_gentoo_ci/config/workers.py | 8 + buildbot_gentoo_ci/db/__init__.py | 0 buildbot_gentoo_ci/db/connector.py | 98 ++++++ buildbot_gentoo_ci/db/migrate/README | 4 + buildbot_gentoo_ci/db/migrate/migrate.cfg | 20 ++ buildbot_gentoo_ci/db/migrate/versions/__init__.py | 0 buildbot_gentoo_ci/db/model.py | 352 +++++++++++++++++++++ buildbot_gentoo_ci/scripts/update_db.py | 110 +++++++ gentooci.cfg | 17 + licenses/GPL-2 | 339 ++++++++++++++++++++ master.cfg | 80 +++++ 18 files changed, 1244 insertions(+) diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..1e08eb1 --- /dev/null +++ b/README.txt @@ -0,0 +1,8 @@ +============================= +Gentoo Ci Build System +============================= + +This is a POC +We use Buildbot to build and test packages + +https://wiki.gentoo.org/wiki/Project:Tinderbox-cluster diff --git a/buildbot_gentoo_ci/__init__.py b/buildbot_gentoo_ci/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/buildbot_gentoo_ci/config/__init__.py b/buildbot_gentoo_ci/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py new file mode 100644 index 0000000..b663fba --- /dev/null +++ b/buildbot_gentoo_ci/config/builders.py @@ -0,0 +1,9 @@ +# Copyright 2020 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +from buildbot.plugins import util +from buildbot_gentoo_ci.config import buildfactory + +def gentoo_builders(b=[]): + b.append(util.BuilderConfig(name='update_db_packages', workername='updatedb_1', factory=buildfactory.f_update_db_packages())) + return b diff --git a/buildbot_gentoo_ci/config/buildfactory.py b/buildbot_gentoo_ci/config/buildfactory.py new file mode 100644 index 0000000..0943031 --- /dev/null +++ b/buildbot_gentoo_ci/config/buildfactory.py @@ -0,0 +1,8 @@ +# Copyright 2020 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +from buildbot.plugins import util + +def f_update_db_packages(): + f = util.BuildFactory() + return f diff --git a/buildbot_gentoo_ci/config/config.py b/buildbot_gentoo_ci/config/config.py new file mode 100644 index 0000000..3ad8595 --- /dev/null +++ b/buildbot_gentoo_ci/config/config.py @@ -0,0 +1,135 @@ +# This file has parts from Buildbot and is modifyed by Gentoo Authors. +# Buildbot is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members +# Origins: buildbot.config.py +# Modifyed by Gentoo Authors. +# Copyright 2020 Gentoo Authors + +import datetime +import inspect +import os +import re +import sys +import traceback +import warnings +from types import MethodType + +from twisted.python import failure +from twisted.python import log +from twisted.python.compat import execfile +from zope.interface import implementer + +from buildbot import interfaces +from buildbot import locks +from buildbot import util +from buildbot.interfaces import IRenderable +from buildbot.revlinks import default_revlink_matcher +from buildbot.util import ComparableMixin +from buildbot.util import bytes2unicode +from buildbot.util import config as util_config +from buildbot.util import identifiers as util_identifiers +from buildbot.util import safeTranslate +from buildbot.util import service as util_service +from buildbot.warnings import ConfigWarning +from buildbot.warnings import warn_deprecated +from buildbot.config import ConfigErrors, error, loadConfigDict + +_errors = None + +DEFAULT_DB_URL = 'sqlite:///gentoo.sqlite' + +#Use GentooCiConfig.loadFromDict +@implementer(interfaces.IConfigLoader) +class FileLoader(ComparableMixin): + compare_attrs = ['basedir', 'configFileName'] + + def __init__(self, basedir, configFileName): + self.basedir = basedir + self.configFileName = configFileName + + def loadConfig(self): + # from here on out we can batch errors together for the user's + # convenience + global _errors + _errors = errors = ConfigErrors() + + try: + filename, config_dict = loadConfigDict( + self.basedir, self.configFileName) + config = GentooCiConfig.loadFromDict(config_dict, filename) + except ConfigErrors as e: + errors.merge(e) + finally: + _errors = None + + if errors: + raise errors + + return config + +# Modifyed for Gentoo Ci settings +class GentooCiConfig(util.ComparableMixin): + + def __init__(self): + self.db = dict( + db_url=DEFAULT_DB_URL, + ) + + _known_config_keys = set([ + "db_url", + ]) + + compare_attrs = list(_known_config_keys) + + @classmethod + def loadFromDict(cls, config_dict, filename): + # warning, all of this is loaded from a thread + global _errors + _errors = errors = ConfigErrors() + + # check for unknown keys + unknown_keys = set(config_dict.keys()) - cls._known_config_keys + if unknown_keys: + if len(unknown_keys) == 1: + error('Unknown BuildmasterConfig key {}'.format(unknown_keys.pop())) + else: + error('Unknown BuildmasterConfig keys {}'.format(', '.join(sorted(unknown_keys)))) + + # instantiate a new config object, which will apply defaults + # automatically + config = cls() + # and defer the rest to sub-functions, for code clarity + try: + config.load_db(config_dict) + finally: + _errors = None + + if errors: + raise errors + + return config + + @staticmethod + def getDbUrlFromConfig(config_dict, throwErrors=True): + + # we don't attempt to parse db URLs here - the engine strategy will do + # so. + if 'db_url' in config_dict: + return config_dict['db_url'] + + return DEFAULT_DB_URL + + def load_db(self, config_dict): + self.db = dict(db_url=self.getDbUrlFromConfig(config_dict)) diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py new file mode 100644 index 0000000..f5b19da --- /dev/null +++ b/buildbot_gentoo_ci/config/schedulers.py @@ -0,0 +1,56 @@ +# Copyright 2020 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +from buildbot.plugins import schedulers, util + [email protected] +def builderUpdateDbNames(self, props): + builders = set() + for f in props.files: + if f.endswith('.ebuild'): + builders.add('update_db_packages') + return list(builders) + [email protected] +def cpvUpdateDb(props): + cpv_changes = [] + for f in props.files: + if f.endswith('.ebuild'): + cppv = f.split('.eb', 0) + cpv = cppv.split('/', 0) + '/' + cppv.split('/', 2) + if not cpv in cpv_changes: + cpv_changes.append(cpv) + return cpv_changes + +def gentoo_schedulers(): + scheduler_update_db = schedulers.SingleBranchScheduler( + name='scheduler_update_db', + treeStableTimer=60, + properties = { + 'cpv_changes' : cpvUpdateDb, + }, + builderNames = builderUpdateDbNames, + change_filter=util.ChangeFilter(branch='master'), + ) + test_updatedb = schedulers.ForceScheduler( + name="force", + buttonName="pushMe!", + label="My nice Force form", + builderNames=['update_db_packages'], + # A completely customized property list. The name of the + # property is the name of the parameter + properties=[ + util.NestedParameter(name="options", label="Build Options", + layout="vertical", fields=[ + util.StringParameter(name="cpv_changes", + label="Package to check", + default="dev-lang/python-3.8", size=80), + util.StringParameter(name="repository", + label="repo", + default="gentoo", size=80), + ]) + ]) + s = [] + s.append(test_updatedb) + #s.append(scheduler_update_db) + return s diff --git a/buildbot_gentoo_ci/config/workers.py b/buildbot_gentoo_ci/config/workers.py new file mode 100644 index 0000000..50a4751 --- /dev/null +++ b/buildbot_gentoo_ci/config/workers.py @@ -0,0 +1,8 @@ +# Copyright 2020 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +from buildbot.plugins import worker + +def gentoo_workers(w=[]): + w.append(worker.LocalWorker('updatedb_1')) + return w diff --git a/buildbot_gentoo_ci/db/__init__.py b/buildbot_gentoo_ci/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/buildbot_gentoo_ci/db/connector.py b/buildbot_gentoo_ci/db/connector.py new file mode 100644 index 0000000..682e72a --- /dev/null +++ b/buildbot_gentoo_ci/db/connector.py @@ -0,0 +1,98 @@ +# This file has parts from Buildbot and is modifyed by Gentoo Authors. +# Buildbot is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members +# Origins: buildbot.db.connector.py +# Modifyed by Gentoo Authors. +# Copyright 2020 Gentoo Authors + + +import textwrap + +from twisted.application import internet +from twisted.internet import defer +from twisted.python import log + +from buildbot import util +from buildbot.db import enginestrategy +from buildbot.db import exceptions +from buildbot.db import pool +from buildbot.util import service + +from buildbot_gentoo_ci.db import model + +upgrade_message = textwrap.dedent("""\ + + The Buildmaster database needs to be upgraded before this version of + buildbot can run. Use the following command-line + + buildbot upgrade-master {basedir} + + to upgrade the database, and try starting the buildmaster again. You may + want to make a backup of your buildmaster before doing so. + """).strip() + +# Gentoo Ci tables and ConnectorComponent +class DBConnector(service.ReconfigurableServiceMixin, + service.AsyncMultiService): + # The connection between Buildbot and its backend database. This is + # generally accessible as master.db, but is also used during upgrades. + # + # Most of the interesting operations available via the connector are + # implemented in connector components, available as attributes of this + # object, and listed below. + + # Period, in seconds, of the cleanup task. This master will perform + # periodic cleanup actions on this schedule. + CLEANUP_PERIOD = 3600 + + def __init__(self, basedir): + super().__init__() + self.setName('db') + self.basedir = basedir + + # set up components + self._engine = None # set up in reconfigService + self.pool = None # set up in reconfigService + + @defer.inlineCallbacks + def setServiceParent(self, p): + yield super().setServiceParent(p) + self.model = model.Model(self) + + @defer.inlineCallbacks + def setup(self, config, check_version=True, verbose=True): + db_url = config.db['db_url'] + + log.msg("Setting up database with URL %r" + % util.stripUrlPassword(db_url)) + + # set up the engine and pool + self._engine = enginestrategy.create_engine(db_url, + basedir=self.basedir) + self.pool = pool.DBThreadPool( + self._engine, reactor=self.master.reactor, verbose=verbose) + + # make sure the db is up to date, unless specifically asked not to + if check_version: + if db_url == 'sqlite://': + # Using in-memory database. Since it is reset after each process + # restart, `buildbot upgrade-master` cannot be used (data is not + # persistent). Upgrade model here to allow startup to continue. + self.model.upgrade() + current = yield self.model.is_current() + if not current: + for l in upgrade_message.format(basedir=self.basedir).split('\n'): + log.msg(l) + raise exceptions.DatabaseNotReadyError() diff --git a/buildbot_gentoo_ci/db/migrate/README b/buildbot_gentoo_ci/db/migrate/README new file mode 100644 index 0000000..c5f51f2 --- /dev/null +++ b/buildbot_gentoo_ci/db/migrate/README @@ -0,0 +1,4 @@ +This is a database migration repository. + +More information at +https://sqlalchemy-migrate.readthedocs.io/en/latest/ diff --git a/buildbot_gentoo_ci/db/migrate/migrate.cfg b/buildbot_gentoo_ci/db/migrate/migrate.cfg new file mode 100644 index 0000000..8be171d --- /dev/null +++ b/buildbot_gentoo_ci/db/migrate/migrate.cfg @@ -0,0 +1,20 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=GentooCi + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] diff --git a/buildbot_gentoo_ci/db/migrate/versions/__init__.py b/buildbot_gentoo_ci/db/migrate/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py new file mode 100644 index 0000000..8865517 --- /dev/null +++ b/buildbot_gentoo_ci/db/model.py @@ -0,0 +1,352 @@ +# This file has parts from Buildbot and is modifyed by Gentoo Authors. +# Buildbot is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members +# Origins: buildbot.db.model.py +# Modifyed by Gentoo Authors. +# Copyright 2020 Gentoo Authors + +import uuid +import migrate +import migrate.versioning.repository +import sqlalchemy as sa +from migrate import exceptions # pylint: disable=ungrouped-imports + +from twisted.internet import defer +from twisted.python import log +from twisted.python import util + +from buildbot.db import base +from buildbot.db.migrate_utils import test_unicode +from buildbot.util import sautils + +try: + from migrate.versioning.schema import ControlledSchema # pylint: disable=ungrouped-imports +except ImportError: + ControlledSchema = None + + +class Model(base.DBConnectorComponent): + # + # schema + # + + metadata = sa.MetaData() + + # NOTES + + # * server_defaults here are included to match those added by the migration + # scripts, but they should not be depended on - all code accessing these + # tables should supply default values as necessary. The defaults are + # required during migration when adding non-nullable columns to existing + # tables. + # + # * dates are stored as unix timestamps (UTC-ish epoch time) + # + # * sqlalchemy does not handle sa.Boolean very well on MySQL or Postgres; + # use sa.SmallInteger instead + + # Tables related to gentoo-ci-cloud + # ------------------------- + + repositorys = sautils.Table( + "repositorys", metadata, + # unique id per repository + sa.Column('uuid', sa.String(36), primary_key=True, + default=lambda: str(uuid.uuid4()), + ), + # repository's name + sa.Column('name', sa.String(255), nullable=False), + # description of the repository + sa.Column('description', sa.Text, nullable=True), + sa.Column('mirror_url', sa.String(255), nullable=True), + sa.Column('auto', sa.Boolean, default=False), + sa.Column('enabled', sa.Boolean, default=False), + sa.Column('ebuild', sa.Boolean, default=False), + ) + + # Use by GitPoller + repository_gitpuller = sautils.Table( + "repository_gitpuller", metadata, + # unique id per repository + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('repository_uuid', sa.String(36), + sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'), + nullable=False), + sa.Column('project', sa.String(255), nullable=False, default='gentoo'), + sa.Column('url', sa.String(255), nullable=False), + sa.Column('branche', sa.String(255), nullable=False, default='master'), + ) + + projects = sautils.Table( + "projects", metadata, + # unique id per project + sa.Column('uuid', sa.String(36), primary_key=True, + default=lambda: str(uuid.uuid4()), + ), + # project's name + sa.Column('name', sa.String(255), nullable=False), + # description of the project + sa.Column('description', sa.Text, nullable=True), + sa.Column('profile', sa.String(255), nullable=False), + sa.Column('portage_repository_uuid', sa.Integer, + sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'), + nullable=False), + sa.Column('keyword_id', sa.Integer, + sa.ForeignKey('keywords.id', ondelete='CASCADE'), + nullable=False), + sa.Column('unstable', sa.Boolean, default=False), + sa.Column('auto', sa.Boolean, default=False), + sa.Column('enabled', sa.Boolean, default=False), + sa.Column('created_by', sa.Integer, + sa.ForeignKey('users.uid', ondelete='CASCADE'), + nullable=False), + ) + + # What repository's use by projects + projects_repositorys = sautils.Table( + "projects_repositorys", metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('projects_uuid', sa.String(36), + sa.ForeignKey('projects.uuid', ondelete='CASCADE'), + nullable=False), + sa.Column('repository_uuid', sa.String(36), + sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'), + nullable=False), + ) + keywords = sautils.Table( + "keywords", metadata, + # unique id per project + sa.Column('id', sa.Integer, primary_key=True), + # project's name + sa.Column('keyword', sa.String(255), nullable=False), + ) + + categorys = sautils.Table( + "categories", metadata, + sa.Column('uuid', sa.String(36), primary_key=True, + default=lambda: str(uuid.uuid4()) + ), + sa.Column('name', sa.String(255), nullable=False), + ) + + packages = sautils.Table( + "packages", metadata, + sa.Column('uuid', sa.String(36), primary_key=True, + default=lambda: str(uuid.uuid4()), + ), + sa.Column('name', sa.String(255), nullable=False), + sa.Column('category_uuid', sa.String(36), + sa.ForeignKey('categories.uuid', ondelete='CASCADE'), + nullable=False), + sa.Column('repository_uuid', sa.String(36), + sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'), + nullable=False), + sa.Column('deleted', sa.Boolean, default=False), + sa.Column('deleted_at', sa.Integer, nullable=True), + ) + + ebuilds = sautils.Table( + "ebuilds", metadata, + sa.Column('uuid', sa.String(36), primary_key=True, + default=lambda: str(uuid.uuid4()), + ), + sa.Column('name', sa.String(255), nullable=False), + sa.Column('package_uuid', sa.String(36), + sa.ForeignKey('packages.uuid', ondelete='CASCADE'), + nullable=False), + sa.Column('ebuild_hash', sa.String(255), nullable=False), + sa.Column('deleted', sa.Boolean, default=False), + sa.Column('deleted_at', sa.Integer, nullable=True), + ) + + ebuildkeywords = sautils.Table( + "ebuildkeywords", metadata, + # unique id per project + sa.Column('id', sa.Integer, primary_key=True), + # project's name + sa.Column('keyword_id', sa.Integer, + sa.ForeignKey('keywords.id', ondelete='CASCADE')), + sa.Column('ebuild_uuid', sa.String(36), + sa.ForeignKey('ebuilds.uuid', ondelete='CASCADE')), + sa.Column('status', sa.String(255), nullable=False), + ) + + # Tables related to users + # ----------------------- + + # This table identifies individual users, and contains buildbot-specific + # information about those users. + users = sautils.Table( + "users", metadata, + # unique user id number + sa.Column("uid", sa.Integer, primary_key=True), + + # identifier (nickname) for this user; used for display + sa.Column("identifier", sa.String(255), nullable=False), + + # username portion of user credentials for authentication + sa.Column("bb_username", sa.String(128)), + + # password portion of user credentials for authentication + sa.Column("bb_password", sa.String(128)), + ) + + # Indexes + # ------- + + + + # MySQL creates indexes for foreign keys, and these appear in the + # reflection. This is a list of (table, index) names that should be + # expected on this platform + + implied_indexes = [ + ] + + # Migration support + # ----------------- + + # this is a bit more complicated than might be expected because the first + # seven database versions were once implemented using a homespun migration + # system, and we need to support upgrading masters from that system. The + # old system used a 'version' table, where SQLAlchemy-Migrate uses + # 'migrate_version' + + repo_path = util.sibpath(__file__, "migrate") + + @defer.inlineCallbacks + def is_current(self): + if ControlledSchema is None: + # this should have been caught earlier by enginestrategy.py with a + # nicer error message + raise ImportError("SQLAlchemy/SQLAlchemy-Migrate version conflict") + + def thd(engine): + # we don't even have to look at the old version table - if there's + # no migrate_version, then we're not up to date. + repo = migrate.versioning.repository.Repository(self.repo_path) + repo_version = repo.latest + try: + # migrate.api doesn't let us hand in an engine + schema = ControlledSchema(engine, self.repo_path) + db_version = schema.version + except exceptions.DatabaseNotControlledError: + return False + + return db_version == repo_version + ret = yield self.db.pool.do_with_engine(thd) + return ret + + # returns a Deferred that returns None + def create(self): + # this is nice and simple, but used only for tests + def thd(engine): + self.metadata.create_all(bind=engine) + return self.db.pool.do_with_engine(thd) + + @defer.inlineCallbacks + def upgrade(self): + + # here, things are a little tricky. If we have a 'version' table, then + # we need to version_control the database with the proper version + # number, drop 'version', and then upgrade. If we have no 'version' + # table and no 'migrate_version' table, then we need to version_control + # the database. Otherwise, we just need to upgrade it. + + def table_exists(engine, tbl): + try: + r = engine.execute("select * from {} limit 1".format(tbl)) + r.close() + return True + except Exception: + return False + + # http://code.google.com/p/sqlalchemy-migrate/issues/detail?id=100 + # means we cannot use the migrate.versioning.api module. So these + # methods perform similar wrapping functions to what is done by the API + # functions, but without disposing of the engine. + def upgrade(engine): + schema = ControlledSchema(engine, self.repo_path) + changeset = schema.changeset(None) + with sautils.withoutSqliteForeignKeys(engine): + for version, change in changeset: + log.msg('migrating schema version {} -> {}'.format(version, version + 1)) + schema.runchange(version, change, 1) + + def check_sqlalchemy_migrate_version(): + # sqlalchemy-migrate started including a version number in 0.7; we + # support back to 0.6.1, but not 0.6. We'll use some discovered + # differences between 0.6.1 and 0.6 to get that resolution. + version = getattr(migrate, '__version__', 'old') + if version == 'old': + try: + from migrate.versioning import schemadiff + if hasattr(schemadiff, 'ColDiff'): + version = "0.6.1" + else: + version = "0.6" + except Exception: + version = "0.0" + version_tup = tuple(map(int, version.split('-', 1)[0].split('.'))) + log.msg("using SQLAlchemy-Migrate version {}".format(version)) + if version_tup < (0, 6, 1): + raise RuntimeError(("You are using SQLAlchemy-Migrate {}. " + "The minimum version is 0.6.1.").format(version)) + + def version_control(engine, version=None): + ControlledSchema.create(engine, self.repo_path, version) + + # the upgrade process must run in a db thread + def thd(engine): + # if the migrate_version table exists, we can just let migrate + # take care of this process. + if table_exists(engine, 'migrate_version'): + r = engine.execute( + "select version from migrate_version limit 1") + old_version = r.scalar() + if old_version < 40: + raise EightUpgradeError() + try: + upgrade(engine) + except sa.exc.NoSuchTableError as e: # pragma: no cover + if 'migration_tmp' in str(e): + log.err('A serious error has been encountered during the upgrade. The ' + 'previous upgrade has been likely interrupted. The database has ' + 'been damaged and automatic recovery is impossible.') + log.err('If you believe this is an error, please submit a bug to the ' + 'Buildbot project.') + raise + + # if the version table exists, then we can version_control things + # at that version, drop the version table, and let migrate take + # care of the rest. + elif table_exists(engine, 'version'): + raise EightUpgradeError() + + # otherwise, this db is new, so we don't bother using the migration engine + # and just create the tables, and put the version directly to + # latest + else: + # do some tests before getting started + test_unicode(engine) + + log.msg("Initializing empty database") + Model.metadata.create_all(engine) + repo = migrate.versioning.repository.Repository(self.repo_path) + + version_control(engine, repo.latest) + + check_sqlalchemy_migrate_version() + yield self.db.pool.do_with_engine(thd) diff --git a/buildbot_gentoo_ci/scripts/update_db.py b/buildbot_gentoo_ci/scripts/update_db.py new file mode 100644 index 0000000..29e9072 --- /dev/null +++ b/buildbot_gentoo_ci/scripts/update_db.py @@ -0,0 +1,110 @@ +# This file has parts from Buildbot and is modifyed by Gentoo Authors. +# Buildbot is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members +# Origins: buildbot.scripts.base.py +# buildbot.scripts.upgrade_master.py +# Modifyed by Gentoo Authors. +# Copyright 2020 Gentoo Authors + +import os +import signal +import sys +import traceback + +from twisted.internet import defer +from buildbot.master import BuildMaster +from buildbot.util import stripUrlPassword +from buildbot.config import ConfigErrors + +from buildbot_gentoo_ci.db import connector +from buildbot_gentoo_ci.config.config import FileLoader + +# Use FileLoader from Gentoo Ci +def loadConfig(config, configFileName='master.cfg'): + if not config['quiet']: + print("checking {}".format(configFileName)) + + try: + master_cfg = FileLoader( + config['basedir'], configFileName).loadConfig() + except ConfigErrors as e: + print("Errors loading configuration:") + + for msg in e.errors: + print(" " + msg) + return None + except Exception: + print("Errors loading configuration:") + traceback.print_exc(file=sys.stdout) + return None + + return master_cfg + +#Use the db from Gentoo Ci [email protected] +def upgradeDatabase(config, master_cfg): + if not config['quiet']: + print("upgrading database ({})".format(stripUrlPassword(master_cfg.db['db_url']))) + print("Warning: Stopping this process might cause data loss") + + def sighandler(signum, frame): + msg = " ".join(""" + WARNING: ignoring signal {}. + This process should not be interrupted to avoid database corruption. + If you really need to terminate it, use SIGKILL. + """.split()) + print(msg.format(signum)) + + prev_handlers = {} + try: + for signame in ("SIGTERM", "SIGINT", "SIGQUIT", "SIGHUP", + "SIGUSR1", "SIGUSR2", "SIGBREAK"): + if hasattr(signal, signame): + signum = getattr(signal, signame) + prev_handlers[signum] = signal.signal(signum, sighandler) + + master = BuildMaster(config['basedir']) + master.config = master_cfg + master.db.disownServiceParent() + db = connector.DBConnector(basedir=config['basedir']) + yield db.setServiceParent(master) + yield db.setup(master_cfg, check_version=False, verbose=not config['quiet']) + yield db.model.upgrade() + yield db.masters.setAllMastersActiveLongTimeAgo() + + finally: + # restore previous signal handlers + for signum, handler in prev_handlers.items(): + signal.signal(signum, handler) + +# Use gentooci.cfg for config +def upgradeGentooCi(config): + master_cfg = loadConfig(config, 'gentooci.cfg') + if not master_cfg: + return defer.succeed(1) + return _upgradeMaster(config, master_cfg) + +# No changes +def _upgradeMaster(config, master_cfg): + try: + upgradeDatabase(config, master_cfg) + except Exception: + e = traceback.format_exc() + print("problem while upgrading!:\n" + e, file=sys.stderr) + return 1 + else: + if not config['quiet']: + print("upgrade complete") + return 0 diff --git a/gentooci.cfg b/gentooci.cfg new file mode 100644 index 0000000..5036ae9 --- /dev/null +++ b/gentooci.cfg @@ -0,0 +1,17 @@ +# -*- python -*- +# ex: set filetype=python: + +# This is a sample gentoo ci buildmaster config file. It must be installed as +# 'gentooci.cfg' in your buildmaster's base directory. + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +####### DB URL +####### DB URL +# This specifies what database buildbot uses to store its state. +# It's easy to start with sqlite, but it's recommended to switch to a dedicated +# database, such as PostgreSQL or MySQL, for use in production environments. +# http://docs.buildbot.net/current/manual/configuration/global.html#database-specification +c['db_url'] = "mysql://buildbot:[email protected]/gentooci?max_idle=300" diff --git a/licenses/GPL-2 b/licenses/GPL-2 new file mode 100644 index 0000000..0e845b5 --- /dev/null +++ b/licenses/GPL-2 @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/master.cfg b/master.cfg new file mode 100644 index 0000000..996f917 --- /dev/null +++ b/master.cfg @@ -0,0 +1,80 @@ +# -*- python -*- +# ex: set filetype=python: +from buildbot_gentoo_ci.config import schedulers, workers, builders + +# This is a sample buildmaster config file. It must be installed as +# 'master.cfg' in your buildmaster's base directory. + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +####### WORKERS + +# The 'workers' list defines the set of recognized workers. Each element is +# a Worker object, specifying a unique worker name and password. The same +# worker name and password must be configured on the worker. +c['workers'] = workers.gentoo_workers() + +# 'protocols' contains information about protocols which master will use for +# communicating with workers. You must define at least 'port' option that workers +# could connect to your master with this protocol. +# 'port' must match the value configured into the workers (with their +# --master option) +c['protocols'] = {'pb': {'port': 9989}} + +####### CHANGESOURCES + +# the 'change_source' setting tells the buildmaster how it should find out +# about source code changes. Here we point to the buildbot version of a python hello-world project. + +# c['change_source'] = [] + +####### SCHEDULERS + +# Configure the Schedulers, which decide how to react to incoming changes. In this +# case, just kick off a 'runtests' build + +c['schedulers'] = schedulers.gentoo_schedulers() + +####### BUILDERS + +# The 'builders' list defines the Builders, which tell Buildbot how to perform a build: +# what steps, and which workers can execute them. Note that any particular build will +# only take place on one worker. + +c['builders'] = builders.gentoo_builders() + +####### BUILDBOT SERVICES + +# 'services' is a list of BuildbotService items like reporter targets. The +# status of each build will be pushed to these targets. buildbot/reporters/*.py +# has a variety to choose from, like IRC bots. + +#c['services'] = [] + +####### PROJECT IDENTITY + +# the 'title' string will appear at the top of this buildbot installation's +# home pages (linked to the 'titleURL'). + +c['title'] = "Gentoo CI" +c['titleURL'] = "https://gentoo-ci.gentoo.org"; + +# the 'buildbotURL' string should point to the location where the buildbot's +# internal web server is visible. This typically uses the port number set in +# the 'www' entry below, but with an externally-visible host name which the +# buildbot cannot figure out without some help. + +c['buildbotURL'] = "http://localhost:8010/"; + +# minimalistic config to activate new web UI +c['www'] = dict(port=8010, + plugins=dict(waterfall_view={}, console_view={}, grid_view={})) + +####### DB URL +# This specifies what database buildbot uses to store its state. +# It's easy to start with sqlite, but it's recommended to switch to a dedicated +# database, such as PostgreSQL or MySQL, for use in production environments. +# http://docs.buildbot.net/current/manual/configuration/global.html#database-specification +c['db_url'] = "mysql://buildbot:[email protected]/buildbot?max_idle=300"
