path: root/buildbot_gentoo_ci/steps/logs.py

# Copyright 2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

import os
import re
import gzip
import io
import hashlib
import json

from portage.versions import catpkgsplit

from twisted.internet import defer
from twisted.python import log

from buildbot.process.buildstep import BuildStep
from buildbot.process.properties import Properties
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import WARNINGS
from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
from buildbot.plugins import util

#from buildbot_gentoo_ci.steps import minio
from buildbot_gentoo_ci.steps import master as master_steps
from buildbot_gentoo_ci.steps import bugs
from buildbot_gentoo_ci.utils.regex import stripQuotesAndMore, finishTitle

def PersOutputOfLogParser(rc, stdout, stderr):
    build_summery_output = {}
    build_summery_output['rc'] = rc
    summary_log_dict = {}
    # split the lines
    for line in stdout.split('\n'):
        #FIXME: check if line start with {"[1-9]": {
        if line.startswith('{'):
            for k, v in json.loads(line).items():
                summary_log_dict[int(k)] = {
                                    'text' : v['text'],
                                    'pattern_infos' : [],
                                    }
                for s in v['pattern_info']:
                    summary_log_dict[int(k)]['pattern_infos'].append({
                                                            'search_type' : s['search_type'],
                                                            'status' : s['status'],
                                                            'line' : s['line'],
                                                            'search_pattern' : s['search_pattern'],
                                                            'phase' : s['phase'],
                                                            'uuid' : s['uuid'],
                                                            'description' : s['description'],
                                                            })
    build_summery_output['summary_log_dict'] = summary_log_dict
    #FIXME: Handling of stderr output
    return {
        'build_summery_output' : build_summery_output
        }

def PersOutputOfEmergeInfo(rc, stdout, stderr):
    #FIXME: line for package info
    emerge_info_output = {}
    emerge_info_output['rc'] = rc
    emerge_info_list = []
    emerge_package_info = []
    for line in stdout.split('\n'):
        if line.startswith('['):
            emerge_package_info.append(line)
        else:
            emerge_info_list.append(line)
    emerge_info_output['emerge_info'] = emerge_info_list
    emerge_info_output['emerge_package_info'] = emerge_package_info
    return {
        'emerge_info_output' : emerge_info_output
        }

class SetupPropertys(BuildStep):
    
    name = 'SetupPropertys'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        # set this in config
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        project_data = yield self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_build_data')['project_uuid'])
        default_project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project']['update_db'])
        version_data = yield self.gentooci.db.versions.getVersionByUuid(self.getProperty('project_build_data')['version_uuid'])
        self.setProperty("project_data", project_data, 'project_data')
        self.setProperty("default_project_data", default_project_data, 'default_project_data')
        self.setProperty("version_data", version_data, 'version_data')
        self.setProperty("status", 'completed', 'status')
        if isinstance(self.getProperty('faild_cpv'), str):
            log_cpv = self.getProperty('faild_cpv')
        else:
            log_cpv = self.getProperty('cpv')
        self.setProperty("log_cpv", log_cpv, 'log_cpv')
        self.setProperty("bgo", False, 'bgo')
        self.descriptionDone = 'Runing log checker on ' + log_cpv
        logsdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']))
        self.setProperty("logsdir", logsdir, 'logsdir')
        return SUCCESS

class SetupParserBuildLoger(BuildStep):

    name = 'SetupParserBuildLoger'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        self.aftersteps_list = []
        log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
        build_log_file_compressed = log_cpv['full_logname'] + '.xz'
        mastersrc_log = yield os.path.join(self.getProperty('logsdir'), build_log_file_compressed)
        log_py = 'log_parser.py'
        mastersrc_py = yield os.path.join(self.master.basedir, log_py)
        repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty("project_data")['settings_repository_uuid'])
        # Git clone settings repo
        self.aftersteps_list.append(steps.GitLab(repourl=repository_data['url'],
            name = 'RunGit',
            descriptionDone=repository_data['name'],
            mode=repository_data['mode'],
            method=repository_data['method'],
            submodules=True,
            alwaysUseLatest=repository_data['alwaysuselatest']
        ))
        # Upload logfile to worker
        self.aftersteps_list.append(steps.FileDownload(
                                                    mastersrc=mastersrc_log,
                                                    workerdest=build_log_file_compressed
                                                    ))
        # Upload log parser py code
        self.aftersteps_list.append(steps.FileDownload(
                                                    mastersrc=mastersrc_py,
                                                    workerdest=log_py
                                                    ))
        #Untar the log
        shell_commad_list = []
        shell_commad_list.append('xz')
        shell_commad_list.append('-dv')
        shell_commad_list.append(build_log_file_compressed)
        self.aftersteps_list.append(
            steps.ShellCommand(
                            name = 'Decompress build log with xz',
                            command = shell_commad_list,
        ))
        # Run the log parser code
        command = []
        command.append('python3')
        command.append(log_py)
        command.append('-f')
        command.append(log_cpv['full_logname'])
        command.append('-u')
        command.append(self.getProperty('project_data')['uuid'])
        command.append('-c')
        #FIXME: set it by images/flavors
        command.append('8')
        #FIXME: debug
        #command.append('-d')
        self.aftersteps_list.append(steps.SetPropertyFromCommand(
                                                            name = 'RunBuildLogParser',
                                                            haltOnFailure = True,
                                                            flunkOnFailure = True,
                                                            command=command,
                                                            strip=False,
                                                            extract_fn=PersOutputOfLogParser,
                                                            timeout=3600
                                                            ))
        yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
        return SUCCESS

class MakeIssue(BuildStep):

    name = 'MakeIssue'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def getNiceErrorLine(self, full_line):
        new_words = []
        for word in full_line.split(' '):
            new_words.append(finishTitle(stripQuotesAndMore(word)))
        return ' '.join(new_words)

    def BuildWordList(self):
        word_list = []
        word_list.append(self.error_dict['cpv'])
        c = catpkgsplit(self.error_dict['cpv'])[0]
        p = catpkgsplit(self.error_dict['cpv'])[1]
        cp = c + '/' + p
        word_list.append(cp)
        for word in self.error_dict['title_phase'].split(' '):
            word_list.append(word)
        for word in self.error_dict['title_issue_nice'].split(' '):
            word_list.append(word)
        return word_list

    def ClassifyIssue(self):
        # get the title for the issue
        text_issue_list = []
        text_phase_list = []
        for k, v in sorted(self.summary_log_dict.items()):
            # get the issue error
            for s in v['pattern_infos']:
                if s['search_type'] == self.error_dict['phase'] and s['status'] == 'error':
                    text_issue_list.append(v['text'])
        # add the issue error
        self.error_dict['cpv'] = self.getProperty('log_cpv')
        if text_issue_list != []:
            title_line = text_issue_list[0].replace('*', '').strip()
            self.error_dict['title_issue'] = title_line
            self.error_dict['title_issue_nice'] = self.getNiceErrorLine(title_line)
            self.error_dict['title_found'] = True
        else:
            self.error_dict['title_issue'] = 'title_issue : None'
            self.error_dict['title_issue_nice'] = 'title_issue_nice : None'
            self.error_dict['title_found'] = False
        self.error_dict['title_phase'] = 'fails to '+ self.error_dict['phase']
        #set the error title
        self.error_dict['title'] = ' '.join([self.error_dict['title_phase'] + ':', self.error_dict['title_issue']])
        self.error_dict['title_nice'] = ' '.join([self.error_dict['title_phase'] + ':', self.error_dict['title_issue_nice']])
        self.error_dict['words'] = self.BuildWordList()

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        self.summary_log_dict = self.getProperty('build_summery_output')['summary_log_dict']
        error = False
        warning = False
        self.summary_log_list = []
        self.error_dict = {}
        self.aftersteps_list = []
        #self.error_dict['hash'] = hashlib.sha256()
        for k, v in sorted(self.summary_log_dict.items()):
            self.summary_log_list.append(v['text'])
            #self.error_dict['hash'].update(v['text'].encode('utf-8'))
            for s in v['pattern_infos']:
                if s['status'] == 'warning':
                    warning = True
            # check if the build did fail
            if v['text'].startswith(' * ERROR:') and v['text'].endswith(' phase):'):
                # get phase error
                phase_error = v['text'].split(' (')[1].split(' phase')[0]
                self.error_dict['phase'] = phase_error
                error = True
        #FIXME: write summary_log_list to a file
        # add issue/bug/pr report
        if error:
            self.ClassifyIssue()
            print(self.error_dict)
            self.setProperty("status", 'failed', 'status')
            self.setProperty("error_dict", self.error_dict, 'error_dict')
            self.aftersteps_list.append(bugs.GetBugs())
        if warning:
            self.setProperty("status", 'warning', 'status')
            self.setProperty("bgo", False, 'bgo')
        self.setProperty("summary_log_list", self.summary_log_list, 'summary_log_list')
        if self.aftersteps_list is not []:
            yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
        return SUCCESS

class Summarylog(BuildStep):

    name = 'Summarylog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        #setup the log
        log = yield self.addLog('summary')
        # add line for line
        for line in self.getProperty('summary_log_list'):
            yield log.addStdout(line + '\n')
        return SUCCESS

class ReadEmergeInfoLog(BuildStep):

    name = 'ReadEmergeInfoLog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    #@defer.inlineCallbacks
    def run(self):
        emerge_info_output = {}
        emerge_info_list = []
        emerge_package_info = []
        # Read the file and add it to a property
        with open(os.path.join(self.getProperty('logsdir'), 'emerge_info.txt'), encoding='utf-8') as source:
            emerge_info = source.read()
        # set emerge_info_output Property
        for line in emerge_info.split('\n'):
            if line.startswith('['):
                emerge_package_info.append(line)
            else:
                emerge_info_list.append(line)
        emerge_info_output['emerge_info'] = emerge_info_list
        emerge_info_output['emerge_package_info'] = emerge_package_info
        self.setProperty("emerge_info_output", emerge_info_output, 'emerge_info_output')
        return SUCCESS

class setEmergeInfoLog(BuildStep):

    name = 'setEmergeInfoLog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        #setup the log
        log = yield self.addLog('emerge_info')
        #FIXME: add emerge info to db
        # add line for line
        for line in self.getProperty('emerge_info_output')['emerge_info']:
            yield log.addStdout(line + '\n')
        return SUCCESS

class setPackageInfoLog(BuildStep):

    name = 'setPackageInfoLog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        #setup the log
        log = yield self.addLog('package_info')
        #FIXME: add package info to db
        # add line for line
        for line in self.getProperty('emerge_info_output')['emerge_package_info']:
            yield log.addStdout(line + '\n')
        return SUCCESS


class Upload(BuildStep):

    name = 'Upload'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
        bucket = self.getProperty('project_data')['uuid'] + '-' + 'logs'
        file_path = yield os.path.join(self.getProperty('logsdir') ,log_cpv['full_logname'])
        aftersteps_list = []
        aftersteps_list.append(minio.putFileToMinio(file_path, log_cpv['full_logname'], bucket))
        yield self.build.addStepsAfterCurrentStep(aftersteps_list)
        return SUCCESS

class ParserPkgCheckLog(BuildStep):

    name = 'ParserPkgCheckLog'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnWarnings = False
    flunkOnFailure = False
    warnOnWarnings = False

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        if self.getProperty("pkg_check_log_data") is None:
            return SKIPPED
        returnstatus = SUCCESS
        error = False
        warning = False
        log = yield self.addLog('Pkgcheck')
        print(self.getProperty("pkg_check_log_data"))
        for a in self.getProperty("pkg_check_log_data"):
            status = ''
            print(a)
            if isinstance(a, dict):
                for k, i in a.items():
                    if k.startswith('_'):
                        if k == '_info':
                            status = 'INFO: '
                        if k == '_error':
                            status = 'ERROR: '
                            error = True
                        if k == '_warning':
                            status = 'WARNING: '
                            warning = True
                        if k == '_style':
                            status = 'STYLE: '
                        if isinstance(i, dict):
                            for b, c in i.items():
                                yield log.addStdout(status + b + c + '\n')
                    else:
                        yield log.addStdout(i + '\n')
        if error:
            returnstatus = FAILURE
        if warning and not error:
            returnstatus = WARNINGS
        return returnstatus

class SetupBugReportSteps(BuildStep):
    name = 'SetupBugReportSteps'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        bug_config = self.gentooci.config.project['bug_config']
        # add bug enable profile and repo
        if bug_config['enable'] and self.getProperty('status') == 'failed' and self.getProperty('error_dict')['title_found'] and not self.getProperty('bgo'):
            aftersteps_list = []
            #Post
            p = Properties()
            p.master = self.master
            separator = ' '
            separator2 = '\n'
            bug_args = {}
            bug_args['user'] = bug_config['user']
            bug_args['passwd'] = yield p.render(util.Secret("bugs_password"))
            bug_params = {}
            title = separator.join([bug_config['extra_summery'], self.getProperty('error_dict')['cpv'], self.getProperty('error_dict')['title_nice']])
            # bug title max 170
            if len(title) >= 170:
                title = title[:167] + '...'
            bug_params['summary'] = title
            # max 16384
            description_list = self.getProperty('summary_log_list')
            del description_list[-6:]
            description_text = separator2.join(description_list)
            bug_params['description'] = description_text + separator
            bug_params['assigned_to'] = bug_config['user']
            #params['cc'] = cc
            bug_params['url'] = f"https://ci.dev.gentoo.org:8010/#/builders/7/builds/{self.getProperty('buildnumber')}"
            print(bug_params)
            # post
            aftersteps_list.append(bugs.Post(bug_args, bug_params))
            # add buildbot info
            bug_params2 = {}
            bug_params2['comment'] = separator2.join(bug_config['buildbot_comment'])
            aftersteps_list.append(bugs.Modify(bug_args, bug_params2))
            # add build info
            comment_list = []
            comment_list.append(f"Was trying to build {self.getProperty('cpv')} Commit: {self.getProperty('revision')} Repository: {self.getProperty('repository')}")
            if self.getProperty('faild_cpv') != self.getProperty('cpv'):
                comment_list.append(f"Depend: {self.getProperty('faild_cpv')} fail to build")
            comment_list.append(f"Package info:")
            for line in self.getProperty('emerge_info_output')['emerge_package_info']:
                comment_list.append(f"{line}")
            comment_list.append(f"Emerge info:")
            for line in self.getProperty('emerge_info_output')['emerge_info']:
                comment_list.append(f"{line}")
            bug_params3 = {}
            bug_params3['comment'] = separator2.join(comment_list)
            aftersteps_list.append(bugs.Modify(bug_args, bug_params3))
            # Attach build log
            file_list = []
            with os.scandir(self.getProperty('logsdir')) as entries:
                for entry in entries:
                    file_list.append(entry.name)
            print(file_list)
            for filename in file_list:
                bug_params4 = {}
                bug_params4['filename'] = yield os.path.join(self.getProperty('logsdir'), filename)
                #bug_params4['comment'] = filename
                aftersteps_list.append(bugs.Attach(bug_args, bug_params4))
            yield self.build.addStepsAfterCurrentStep(aftersteps_list)
            return SUCCESS
        return SKIPPED

class setBuildStatus(BuildStep):

    name = 'setBuildStatus'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = False
    flunkOnFailure = True
    warnOnWarnings = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        project_build_data = self.getProperty('project_build_data')
        bugid = 0
        if self.getProperty('bgo'):
            bugid = self.getProperty('bgo')['id']
        yield self.gentooci.db.builds.setBugIdBuilds(
                                                    project_build_data['id'],
                                                    bugid
                                                    )
        yield self.gentooci.db.builds.setStatusBuilds(
                                                    project_build_data['id'],
                                                    self.getProperty('status')
                                                    )
        if self.getProperty('status') == 'failed':
            return FAILURE
        if self.getProperty('status') == 'warning':
            return WARNINGS
        return SUCCESS