diff options
| -rw-r--r-- | buildbot_gentoo_ci/steps/builders.py | 33 | ||||
| -rw-r--r-- | py/log_parser.py | 11 |
2 files changed, 29 insertions, 15 deletions
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py index a5638b5..117fb70 100644 --- a/buildbot_gentoo_ci/steps/builders.py +++ b/buildbot_gentoo_ci/steps/builders.py @@ -74,7 +74,7 @@ def PersOutputOfEmerge(rc, stdout, stderr): if line.startswith('>>>'): if line.startswith('>>> Failed to'): emerge_output['failed'] = line.split(' ')[4][:-1] - if line.endswith('.log.gz') and emerge_output['failed']: + if line.endswith('.log') and emerge_output['failed']: log_path_list.append(line.split(' ')[2]) #FIXME: Handling of >>> output pass @@ -83,7 +83,7 @@ def PersOutputOfEmerge(rc, stdout, stderr): if line.startswith('!!! existing preserved libs'): pass if line.startswith(' * '): - if line.endswith('.log.gz'): + if line.endswith('.log'): log_path_list.append(line.split(' ')[4]) #FIXME: Handling of depclean output dict of packages that get removed or saved emerge_output['packages'] = package_dict @@ -122,7 +122,7 @@ def PersOutputOfEmerge(rc, stdout, stderr): change_use[cpv_split[0]] = change_use_list emerge_output['change_use'] = change_use if line.startswith(' * '): - if line.endswith('.log.gz'): + if line.endswith('.log'): log_path = line.split(' ')[3] if log_path not in inlog_path_list: log_path_list.append(log_path) @@ -681,13 +681,13 @@ class CheckBuildWorkDirs(BuildStep): def run(self): cpv = self.getProperty('faild_cpv') cpv_build_dir = yield os.path.join('/', 'var', 'tmp', 'portage', self.getProperty('cpv_build_dir')) - compressed_log_file = cpv.replace('/', '_') + '.' + str(self.getProperty("buildnumber")) + '.logs.tar.bz2' + compressed_log_file = cpv.replace('/', '_') + '.' + str(self.getProperty("buildnumber")) + '.logs.tar.xz' masterdest_file = yield os.path.join(self.getProperty('masterdest'), compressed_log_file) # cpv_build_work_dir = yield os.path.join(cpv_build_dir, 'work') if self.getProperty('build_workdir_find_output')['build_workdir_find'] != []: shell_commad_list = [] shell_commad_list.append('tar') - shell_commad_list.append('-cjpf') + shell_commad_list.append('-cJvf') shell_commad_list.append(compressed_log_file) for find_line in sorted(self.getProperty('build_workdir_find_output')['build_workdir_find']): print(find_line) @@ -695,7 +695,7 @@ class CheckBuildWorkDirs(BuildStep): shell_commad_list.append(filename) self.aftersteps_list.append( steps.ShellCommand( - name = 'Tar logs', + name = 'Tar/xz logs', command = shell_commad_list, workdir = cpv_build_dir )) @@ -761,12 +761,23 @@ class CheckEmergeLogs(BuildStep): @defer.inlineCallbacks def getLogFile(self, cpv, log_dict): - file = log_dict[cpv]['full_logname'] - destfile = yield os.path.join(self.getProperty('masterdest'), file) - sourcefile = log_dict[cpv]['log_path'] + compressed_log_file = log_dict[cpv]['full_logname'] + '.xz' + destfile = yield os.path.join(self.getProperty('masterdest'), compressed_log_file) + sourcefile = log_dict[cpv]['log_path'] + '.xz' name = 'Upload build log' - url = '/'.join([hosturl, self.getProperty('workername'), str(self.getProperty("buildnumber")), file]) - urlText = file + url = '/'.join([hosturl, self.getProperty('workername'), str(self.getProperty("buildnumber")), compressed_log_file]) + urlText = compressed_log_file + shell_commad_list = [] + shell_commad_list.append('xz') + shell_commad_list.append('-zv') + shell_commad_list.append(log_dict[cpv]['full_logname']) + self.aftersteps_list.append( + steps.ShellCommand( + name = 'Compress build log with xz', + descriptionDone = log_dict[cpv]['full_logname'], + command = shell_commad_list, + workdir = os.path.dirname(log_dict[cpv]['log_path']) + '/' + )) self.addFileUploade(sourcefile, destfile, name, url, urlText) @defer.inlineCallbacks diff --git a/py/log_parser.py b/py/log_parser.py index f5c4eb5..eb081a9 100644 --- a/py/log_parser.py +++ b/py/log_parser.py @@ -5,7 +5,6 @@ import sys from multiprocessing import Pool, cpu_count import re import io -import gzip import json import os from sqlalchemy.ext.declarative import declarative_base @@ -138,9 +137,13 @@ def runLogParser(args): log_search_pattern = get_log_search_pattern(Session, args.uuid, config['default_uuid']) Session.close() # read the log file to dict - for text_line in io.TextIOWrapper(io.BufferedReader(gzip.open(args.file)), encoding='utf8', errors='ignore'): - logfile_text_dict[index] = text_line.strip('\n') - index = index + 1 + with open(args.file, encoding='utf8', errors='ignore') as f: + for text_line in f: + logfile_text_dict[index] = text_line.strip('\n') + index = index + 1 + #for text_line in io.TextIOWrapper(io.BufferedReader(open(args.file)), encoding='utf8', errors='ignore'): + # logfile_text_dict[index] = text_line.strip('\n') + # index = index + 1 # run the search parse pattern on the text lines #params = [(log_search_pattern, text, line_index,) for line_index, text in logfile_text_dict.items()] with getMultiprocessingPool(config) as pool: |