From f95dce7cfc33e1d5a0e606c5562cac59c2dc0fdd Mon Sep 17 00:00:00 2001 From: Magnus Granberg Date: Mon, 26 Jun 2023 23:52:10 +0200 Subject: Add run_clean_db_request to clean db Signed-off-by: Magnus Granberg --- buildbot_gentoo_ci/config/builders.py | 29 +++++++++ buildbot_gentoo_ci/config/buildfactorys.py | 9 +++ buildbot_gentoo_ci/config/schedulers.py | 3 + buildbot_gentoo_ci/steps/clean.py | 97 ++++++++++++++++++++++++++++++ buildbot_gentoo_ci/steps/package.py | 45 ++++++++++++++ 5 files changed, 183 insertions(+) create mode 100644 buildbot_gentoo_ci/steps/clean.py diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py index b3ddb16..584dfdf 100644 --- a/buildbot_gentoo_ci/config/builders.py +++ b/buildbot_gentoo_ci/config/builders.py @@ -20,6 +20,24 @@ def getWorkers(worker_type, workers): worker_list.append(worker['uuid']) return worker_list +# check if we have same package_data +@defer.inlineCallbacks +def checkPackageData(master, builder, req1, req2): + canBeCollapsed = yield buildrequest.BuildRequest.canBeCollapsed(master, req1, req2) + selfBuildset , otherBuildset = yield defer.gatherResults([ + master.data.get(('buildsets', req1['buildsetid'])), + master.data.get(('buildsets', req2['buildsetid'])) + ]) + print(f"TestCollapsed") + print(f"canBeCollapsed: {canBeCollapsed}") + print(f"selfBuildset: {selfBuildset}") + print(f"otherBuildset: {otherBuildset}") + if canBeCollapsed and selfBuildset['parent_buildid'] != None and \ + otherBuildset['parent_buildid'] != None: + return True + else: + return False + @defer.inlineCallbacks def CanWorkerBuildProject(builder, wfb, request): gentooci = builder.master.namedServices['services'].namedServices['gentooci'] @@ -118,4 +136,15 @@ def gentoo_builders(worker_data): factory=buildfactorys.run_build_images_request() ) ) + # Use multiplay workers + b.append(util.BuilderConfig( + name='run_clean_db_request', + workernames=getWorkersLocal('local', worker_data['local']), + # look builder so we only do one time + # look= + # if we allready runnin with same package_data properties then skip/collapse + #collapseRequests=checkPackageData, + factory=buildfactorys.run_clean_db_request() + ) + ) return b diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index b47a209..6755308 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -14,6 +14,7 @@ from buildbot_gentoo_ci.steps import portage from buildbot_gentoo_ci.steps import logs from buildbot_gentoo_ci.steps import repos from buildbot_gentoo_ci.steps import nodes +from buildbot_gentoo_ci.steps import clean def update_db_check(): f = util.BuildFactory() @@ -66,6 +67,8 @@ def update_db_cpv(): f.addStep(package.TriggerCheckForV()) # update metadata if needed f.addStep(package.CheckMetadataPackagePath()) + # clean package db if needed (clean up ebuilds in db) + f.addStep(package.TriggerCleanPackageDb()) return f def update_db_v(): @@ -201,3 +204,9 @@ def run_build_images_request(): # set the needed steps for making the image f.addStep(nodes.SetupBuildTypeAndSteps()) return f + +def run_clean_db_request(): + f = util.BuildFactory() + # set needed Propertys + f.addStep(clean.SetupPropertys()) + return f diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py index 7833233..c34d714 100644 --- a/buildbot_gentoo_ci/config/schedulers.py +++ b/buildbot_gentoo_ci/config/schedulers.py @@ -92,6 +92,8 @@ def gentoo_schedulers(): builderNames=["parse_build_log"]) run_build_images_request = schedulers.Triggerable(name="run_build_images_request", builderNames=["run_build_images_request"]) + run_clean_db_request = schedulers.Triggerable(name="run_clean_db_request", + builderNames=["run_clean_db_request"]) s = [] s.append(create_images) s.append(scheduler_update_db) @@ -101,4 +103,5 @@ def gentoo_schedulers(): s.append(build_request_data) s.append(run_build_request) s.append(parse_build_log) + s.append(run_clean_db_request) return s diff --git a/buildbot_gentoo_ci/steps/clean.py b/buildbot_gentoo_ci/steps/clean.py new file mode 100644 index 0000000..73733f1 --- /dev/null +++ b/buildbot_gentoo_ci/steps/clean.py @@ -0,0 +1,97 @@ +# Copyright 2023 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +from twisted.internet import defer +from twisted.python import log + +from buildbot.process.buildstep import BuildStep +from buildbot.process.results import SUCCESS +from buildbot.process.results import FAILURE +from buildbot.process.results import SKIPPED +from buildbot.plugins import steps, util + +class SetupPropertys(BuildStep): + name = 'Setup propertys for clean db' + description = 'Running' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @defer.inlineCallbacks + def run(self): + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + aftersteps_list = [] + # Check on deleted ebuild on package + deleted_ebuilds_data = yield self.gentooci.db.versions.getEbuildsByPackage(self.getProperty("package_data")['uuid'], deleted=True) + # check if older then X days + #FIXME: set days in config and move the check to db side + days = 30 + days_in_s = 86400 * days + now = int(self.master.reactor.seconds()) + for ebuild_data in deleted_ebuilds_data: + no_bug = True + # check if older the days + if ebuild_data['deleted_at'] < (now - days_in_s): + print(f"Ebuild : {ebuild_data['uuid']} will be removed") + # get build_data for deleted_ebuild_data['uuid'] + build_data_list = yield self.gentooci.db.builds.getBuildsByVersionUuid(ebuild_data['uuid']) + print(build_data_list) + for build_data in build_data_list: + if build_data['bug_id'] == 0: + # get buildsets + print(f"Build : {build_data['id']} will be removed") + yield self.build.addStepsAfterCurrentStep([RemoveBuildFromDb(build_data)]) + else: + no_bug = False + if no_bug: + yield self.build. addStepsAfterLastStep([RemoveEbuildFromDb(ebuild_data)]) + return SUCCESS + +class RemoveEbuildFromDb(BuildStep): + name = 'Clean ebuild from db' + description = 'Running' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, ebuild_data, **kwargs): + self.ebuild_data = ebuild_data + super().__init__(**kwargs) + + @defer.inlineCallbacks + def run(self): + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + yield self.gentooci.db.versions.removeVersionKeyword(self.ebuild_data['uuid']) + yield self.gentooci.db.versions.removeVersionMetadata(self.ebuild_data['uuid']) + yield self.gentooci.db.versions.removeVersion(self.ebuild_data['uuid']) + self.descriptionDone = f"Ebuild : {self.ebuild_data['uuid']} will be removed" + return SUCCESS + +class RemoveBuildFromDb(BuildStep): + name = 'Clean builds from db' + description = 'Running' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, build_data, **kwargs): + self.build_data = build_data + super().__init__(**kwargs) + + @defer.inlineCallbacks + def run(self): + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + if self.build_data['buildbot_build_id'] != 0: + self.descriptionDone = f"BuildBot Build : {self.build_data['buildbot_build_id']} will be removed" + bb_build_data = yield self.master.db.builds.getBuild(self.build_data['buildbot_build_id']) + # remove buildbot data on db + # remove steps and logs + yield self.master.db.logs.deleteLogChunks(self.build_data['buildbot_build_id']) + # remove propertys + # get buildsets + # remove build_data + yield self.gentooci.db.builds.removeBuild(self.build_data['id']) + return SUCCESS diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py index 13eafa2..847ba12 100644 --- a/buildbot_gentoo_ci/steps/package.py +++ b/buildbot_gentoo_ci/steps/package.py @@ -12,6 +12,7 @@ from twisted.python import log from buildbot.process.buildstep import BuildStep from buildbot.process.results import SUCCESS from buildbot.process.results import FAILURE +from buildbot.process.results import SKIPPED from buildbot.process import remotecommand from buildbot.plugins import steps @@ -282,3 +283,47 @@ class TriggerCheckForV(BuildStep): ) yield self.build.addStepsAfterCurrentStep(addStepUpdateVData) return SUCCESS + +class TriggerCleanPackageDb(BuildStep): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + name = 'TriggerCleanPackageDb' + description = 'Running' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + @defer.inlineCallbacks + def run(self): + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + aftersteps_list = [] + # Check on deleted ebuild on package + deleted_ebuilds_data = yield self.gentooci.db.versions.getEbuildsByPackage(self.getProperty("package_data")['uuid'], deleted=True) + # check if older then X days + #FIXME: set days in config and move the check to db side + days = 30 + days_in_s = 86400 * days + needcleaning = 0 + now = int(self.master.reactor.seconds()) + for deleted_ebuild_data in deleted_ebuilds_data: + # check if older the days + if deleted_ebuild_data['deleted_at'] < (now - days_in_s): + needcleaning = needcleaning + 1 + print(f"{str(needcleaning)} ebuilds need cleaning in db") + if needcleaning > 0: + aftersteps_list.append( + steps.Trigger( + schedulerNames=['run_clean_db_request'], + waitForFinish=False, + updateSourceStamp=False, + set_properties={ + 'package_data' : self.getProperty("package_data"), + } + ) + ) + yield self.build.addStepsAfterCurrentStep(aftersteps_list) + self.descriptionDone = f"Need DB Cleaning: Yes" + return SUCCESS + self.descriptionDone = f"Need DB Cleaning: No" + return SKIPPED -- cgit v1.2.3-65-gdbad