path: root/toolbin/tests/run_nightly.py

#!/usr/bin/python
# -*- Mode: python -*-

# Copyright (C) 2001-2020 Artifex Software, Inc.
# All Rights Reserved.
#
# This software is provided AS-IS with no warranty, either express or
# implied.
#
# This software is distributed under license and may not be copied,
# modified or distributed except as expressly authorized under the terms
# of the license contained in the file LICENSE in this distribution.
#
# Refer to licensing information at http://www.artifex.com or contact
# Artifex Software, Inc.,  1305 Grant Avenue - Suite 200, Novato,
# CA 94945, U.S.A., +1(415)492-9861, for further information.
#


# overview
# 	update revision (default is HEAD)
# 	run all testfiles with all parameter sets using the gs executable - calculate each checksum
# 	compare checksums with 1) previous results 2) baseline checksums
# 	report baseline changes
# 	email results
# 
# temporary output:
#	the output images from gs execution are not saved
#
# permanent output:
# 	checksum database for the day (an "anydbm" database [testfile-device,checksum]
# 
# input:
# 	testing.cfg - configuration many significant variables (in /home/regression/regression, or .)
# 	command line - control details of execution
# 	files in compare file directory (ususally /home/regression/comparefiles)
# 	The following are "anydbm" databases as described above
#	 	database of checksums of the raster files in /home/regression/raster
#	 	databases of past nightly runs in /home/regression/regression/daily (only the most recent is used)
# 	baseline.log - log of updates to baseline raster files (in /home/regression/regression/baseline) [text file]
# 	
# Note that the raster files themselves are not used in this process.
# 
# Hierarchy of python scripts
# 	run_nightly.py
#		gsconf.py - read testing.cfg and create an object defining the environment for all other scripts
# 		run_regression.py - execute gs, calculate checksum, insert into database
#			gs_checkall.py
#				gs_checkraster.py
#				gs_checkpdfwrite.py
#					gsparamset.py - define the "parameter set" - the set of output devices to be tested for each file
#					gstestgs.py, gstestutil.py - run gs (used the built-in Python unit test harness)
#					gssum.py - checksum calculation and database addition
#			
# 		testdiff.py - compare three checksum databases
# 		get_baseline_log.py - extract recent changes to the baseline.log 
#
#
#
#



import os
import sys, shutil
import re
import time
import optparse, myoptparse
import string
import build_revision
import gsconf, gsparamsets
import anydbm
from popen2 import Popen4

# configuration variables

myself="unknown"
myversion="2.01"

def logMessage(message,file,revision,printMessage=True):
    message_time=time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime())
    message=myself+" "+revision+" "+message_time+" "+message
    if printMessage:
        print message
    message+="\n"
    if file:
        file.write(message)
        file.flush()

def die(msg):
    sendmail(gsconf.report_from, gsconf.report_to, "error running regression", msg)
    sys.exit(0)

def read_all_lines(f):
    file = open(f, 'r')
    lines = file.readlines()
    file.close()
    return string.join(lines, '')

def get_count_in_db(dbname):
    try:
        db = anydbm.open(dbname, 'r')
        if not db:
            count=0
        else:
            count=len(db)
    except:
        count=0
    print dbname,count

def get_revision(dir=None):
    if dir:
        cwd=os.getcwd()
        os.chdir(dir)

    command="svn update --non-recursive"
    if os.system(command) != 0:
        message = "run_nightly.py:get_revision.py: cannot svn update"
        logMessage(message,None,"unknown")
        return "unknown"

    command="svn info"
    p = os.popen(command)
    for line in p:
        if "Revision:" in line:
            revision=line.strip('Revision: ')
            revision=revision.strip('\n')
	    break
    else:
        revision = None
    if dir:
        os.chdir(cwd)
    return revision

def change_gsproduct(file):
    tmpfile = "%s.tmp" % (file,)

    startre = re.compile("^#ifndef\ GS_PRODUCT$")
    changere = re.compile("^.*?\"[A-Za-z -]+\".*?$")
    endre = re.compile("^$")

    old = open(file, "r")
    new = open(tmpfile, "w")

    state = 0
    for line in old.readlines():
        print "checking:", line
        if state == 0:
            m = startre.search(line)
            if m:
                state = 1
                
            new.write(line)
        elif state == 1:
            m = changere.search(line)
            if m:
                state = 2
                new.write("\t\"GPL Ghostscript\"\n")
		print "rewriting '%s' to GPL Ghostscript" % line.strip()
            else:
                new.write(line)
        elif state == 2:
            m = endre.search(line)
            if m:
                state = 0

            new.write(line)

    old.close()
    new.close()

    os.unlink(file)
    os.rename(tmpfile, file)

def sendmail(frm, to, subject, text):
    import smtplib

    keyword = gsconf.mailkeyword
    if keyword:
        msg = 'From: %s\r\nTo: %s\r\nSubject: %s\r\nX-Fnord: %s\r\n\r\n%s' % (frm, to, subject, keyword, text)
    else:
        msg = 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s' % (frm, to, subject, text)

#   server = smtplib.SMTP(gsconf.mail_server)
#   server.sendmail(frm, to, msg)
#   server.quit()
    SENDMAIL = "/usr/sbin/sendmail" # sendmail location
    p = os.popen("%s -t" % SENDMAIL, "w")
    p.write(msg)
    p.close()

def run_nightly(options,arguments):
    global myself

    myself=options.myself
    # what do we work on?
    revision      = options.revision
    release       = options.release
    
    # what do we build
    update           = not options.noupdate
    
    options.svn       = not options.nosvn
    options.configure = not options.noconfigure
    options.make      = not options.nomake
    options.makeclean = not options.nomakeclean
    
    options.remove   = False  # never remove the gs executable, we need it for regression testing
    
    # what do we run?
    runRegression    = not options.noregression
    testRegression   = not options.notestregression
    baselineChanges  = not options.nobaselinechanges
    printRegression  = options.printregression
    
    # where do stdout and stderr go?
    options.capture  = not options.nocapture
    
    p = os.popen('hostname -s')
    if p:
        hostname = string.strip(p.readline())
        p.close()
    else:
        hostname = 'unknown host'
        
    p = os.popen('uname -m -s')
    if p:
        hostuname=string.strip(p.readline())
        p.close()
    else:
        hostuname=""

    hostname=hostname + " (" + hostuname + ')'

    if release:
        print "NOT SUPPORTED: need to build a release",release
        sys.exit(1)

    if not revision or revision == "HEAD":
        gsroot=gsconf.gsroot
        revision_value=get_revision(gsroot)
        gsinstall=gsconf.installtree
        gspath=gsconf.installtree+"bin/gs"
        revision_full="HEAD:"+revision_value
    else:
        gsroot=gsconf.root+"gs."+revision+"/"
        gsconf.checksumdb = revision
        revision_value=revision
        gsrevision_root=gsconf.root+"gs."+revision+"/"
        gspath=gsrevision_root+"bin/gs"
        gsinstall=None
        revision_full=revision_value

    logdir=gsconf.logdir
    if not os.path.exists(logdir):
        os.mkdir(logdir)

    now=time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime())
    mail_message = now+" "+revision_value+"\n"

    prefix=logdir+revision_value+"."

    update_stdout=prefix+gsconf.update_stdout
    update_stderr=prefix+gsconf.update_stderr
    make_stdout=prefix+gsconf.make_stdout
    make_stderr=prefix+gsconf.make_stderr
    config_stdout=prefix+gsconf.config_stdout
    config_stderr=prefix+gsconf.config_stderr
    install_stdout=prefix+gsconf.install_stdout
    install_stderr=prefix+gsconf.install_stderr
    
    cumulative_name=prefix+gsconf.cumulative
    
    history_name=logdir+gsconf.history        # permanent file, accumulates history across days
    
    if not os.path.exists(history_name):
        messageHistory="creating history file "+history_name
    else:
        messageHistory="opening history file "+history_name
    history_file = open(history_name, "w")    

    cumulative_file = open(cumulative_name, "w")
    message="cumulative"
    logMessage(message,cumulative_file,revision_full,printMessage=False)
    logMessage(messageHistory,cumulative_file,revision_full,printMessage=False)

    message="running on " + hostname +" my version "+myversion
    logMessage(message,cumulative_file,revision_full)
    logMessage(message,history_file,revision_full,printMessage=False)
    if options.version:
        sys.exit(1)

    if options.removeonly:
        path=gsroot
        message="removing "+path
        logMessage(message,cumulative_file,revision_full)
        shutil.rmtree(path,ignore_errors=True)
        if gsinstall:
            path=gsinstall
            message="removing "+path
            logMessage(message,cumulative_file,revision_full)
            shutil.rmtree(path,ignore_errors=True)
        sys.exit(1)

    mail_message+=message+"\n"

    if not os.path.exists(gsconf.comparefiledir + "/.svn") or \
	    os.system("svn update " + gsconf.comparefiledir) != 0:
	message = "run_nightly.py: cannot svn update " + gsconf.comparefiledir
	logMessage(message,None,"n/a")

    if not os.path.exists(gsconf.comparefiledir + "/../tests/eps" + "/.svn") or \
	    os.system("svn update " + gsconf.comparefiledir + "/../tests/eps") != 0:
	message = "run_nightly.py: cannot svn update " + gsconf.comparefiledir + "/../tests/eps"
	logMessage(message,None,"n/a")
    if os.path.exists(gsconf.comparefiledir + "/../tests/eps"):
        os.system("ln -f -s " + gsconf.comparefiledir + "/../tests/eps/* " + gsconf.comparefiledir)
    if not os.path.exists(gsconf.comparefiledir + "/../tests/ps" + "/.svn") or \
	    os.system("svn update " + gsconf.comparefiledir + "/../tests/ps") != 0:
	message = "run_nightly.py: cannot svn update " + gsconf.comparefiledir + "/../tests/ps"
	logMessage(message,None,"n/a")
    if os.path.exists(gsconf.comparefiledir + "/../tests/ps"):
        os.system("ln -f -s " + gsconf.comparefiledir + "/../tests/ps/* " + gsconf.comparefiledir)
    if not os.path.exists(gsconf.comparefiledir + "/../tests/pdf" + "/.svn") or \
	    os.system("svn update " + gsconf.comparefiledir + "/../tests/pdf") != 0:
	message = "run_nightly.py: cannot svn update " + gsconf.comparefiledir + "/../tests/pdf"
	logMessage(message,None,"n/a")
    if os.path.exists(gsconf.comparefiledir + "/../tests/pdf"):
        os.system("ln -f -s " + gsconf.comparefiledir + "/../tests/pdf/* " + gsconf.comparefiledir)
    comparefiles = os.listdir(gsconf.comparefiledir)
    count=len(comparefiles)

    message="comparefiles from "+gsconf.comparefiledir+" number of files "+str(count)
    logMessage(message,cumulative_file,revision_full)
    logMessage(message,history_file,revision_full,printMessage=False)
    mail_message+=message+"\n"

    # use the smallest that is requested
    if options.oneset:    
        gsparamsets.testparamsets = gsparamsets.testparamsets_one
        gsparamsets.pdftestparamsets = gsparamsets.pdftestparamsets_one
        gsparamsets.testparamsets_name = "one"
    elif options.minset:
        gsparamsets.testparamsets = gsparamsets.testparamsets_minimum
        gsparamsets.pdftestparamsets = gsparamsets.pdftestparamsets_minimum
        gsparamsets.testparamsets_name = "minimum"
    elif options.maxset:    
        gsparamsets.testparamsets = gsparamsets.testparamsets_maximum
        gsparamsets.pdftestparamsets = gsparamsets.pdftestparamsets_full
        gsparamsets.testparamsets_name = "maximum"
    elif options.fullset:    
        gsparamsets.testparamsets = gsparamsets.testparamsets_full
        gsparamsets.pdftestparamsets = gsparamsets.pdftestparamsets_full
        gsparamsets.testparamsets_name = "full"
    else:
        pass # use default in gsparamsets

    gsparamsets.testparamsets = gsparamsets.testparamsets_one
    gsparamsets.pdftestparamsets = gsparamsets.pdftestparamsets_full
    gsparamsets.testparamsets_name = "one+pdf"


    message="test parameter set name: "+gsparamsets.testparamsets_name
    logMessage(message,cumulative_file,revision_full)
    logMessage(message,history_file,revision_full,printMessage=False)
    mail_message+=message+"\n"

    message="logging to "+prefix+"*"
    logMessage(message,cumulative_file,revision_full)
    mail_message+=message+"\n"

    message="sending mail to "+gsconf.report_to
    logMessage(message,cumulative_file,revision_full)
    logMessage(message,history_file,revision_full,printMessage=False)

    if update:
        message="starting update_ghostscript"
        logMessage(message,cumulative_file,revision_full)
    
        (err,message) = build_revision.update_ghostscript(revision,release,
                                                          gsroot,
                                                          options,
                                                          update_stdout,update_stderr,
                                                          config_stdout,config_stderr,
                                                          make_stdout,make_stderr,
                                                          install_stdout,install_stderr,
                                                          cumulative_file
                                                          )
        if err != 0:
            die(message)
    else:
        message="skipping update of ghostscript tree and executable"
        logMessage(message,cumulative_file,revision_full)

    message = "running makemissing"
    logMessage(message,cumulative_file,revision_full)
    os.system("./makemissing 2>logs/makemissing.stderr >logs/makemissing.stdout")


    prefix=logdir+revision_value+"."

    baselines_name=prefix+gsconf.baselines
    differences_name=prefix+gsconf.differences
    regression_name=prefix+gsconf.regression
    email_name=prefix+gsconf.email

    if not os.path.exists(gspath):
        message=myself+" FATAL "+"the gs executable does not exist "+gspath
        logMessage(message,cumulative_file,revision_full)
        die(message)

    regression_file = open(regression_name, "w")
    message="regression"
    logMessage(message,regression_file,revision_full,printMessage=False)

    message="revision of executable "+revision_full
    logMessage(message,regression_file,revision_full,printMessage=False)
    logMessage(message,history_file,revision_full,printMessage=False)
    mail_message+=message+"\n"

    revision_name=prefix+revision
    revision_file = open(revision_name, "w")
    revision_file.close()

    # create string "regression_results" for regression results
    regression_results = '\n'
    if runRegression:
        print

        message="starting regression testing on "+revision
        logMessage(message,regression_file,revision_full,printMessage=False)
        logMessage(message,cumulative_file,revision_full)

        timearg=" --time="+"\""+now+"\""
        trackarg=" --track"

        # revision can be HEAD
        if revision != "HEAD":
            revisionarg=" --revision="+revision
        else: revisionarg=""
   
        # revision_value always has the revision number, also for HEAD
        revision_valuearg=" --revision_value="+revision_value

        if release:
            releasearg=" --release="+release
        else: releasearg=""
    
        command=gsconf.run_regression_script + releasearg + revisionarg + revision_valuearg + timearg + trackarg

        if options.capture:
            capture=" 2>&1"
            command+=capture

        message=command
        logMessage(message,regression_file,revision_full,printMessage=False)
        logMessage(message,cumulative_file,revision_full)

        regression_results += '\nThe complete list of regressions for today:\n'
        regression_results += command +"\n"

        p = os.popen(command)
        if p == None:
            message="cannot open and run the regression script."
            logMessage(message,cumulative_file,revision_full)
            die(message)

        pattern = 'ok$'
        for line in p.readlines():
            if printRegression: print line
            if not re.search(pattern, line):
                regression_results += line
        p.close()

        # count number of entries in daily db
        if gsconf.__dict__.has_key("checksumdb") and gsconf.checksumdb:
            newchecksum=gsconf.checksumdb
        else:
            newchecksum=time.strftime("%Y%m%d", time.localtime())

    else:
        message="skipping regression testing"
        logMessage(message,regression_file,revision_full,printMessage=False)
        logMessage(message,cumulative_file,revision_full)

        regression_results+="gs regression processing was skipped"
    # end if runRegression

    regression_results += '\n'

    # create string "difference_results" for regression results
    if testRegression:
        print

        if gsconf.__dict__.has_key("checksumdb") and gsconf.checksumdb:
            newchecksum=gsconf.checksumdb
        else:
            newchecksum=time.strftime("%Y%m%d", time.localtime())

#        message="verify existence of the checksum database "+newchecksum
#        logMessage(message,cumulative_file,revision_full,printMessage=False)
#        logMessage(message,regression_file,revision_full)

        message="starting difference_results"
        logMessage(message,cumulative_file,revision_full,printMessage=False)
        logMessage(message,regression_file,revision_full)

        if options.oldrevision:
            oldchecksum=options.oldrevision
        else:
            # find the most recent checksum file in the gsconf.dailydir
            oneday=24*60*60
            testtime=time.time()
            for count in range(1,30):
                testtime-=oneday
                oldchecksum=time.strftime("%Y%m%d", time.localtime(testtime))
                datafile=gsconf.dailydir+oldchecksum+".db"
                if os.path.exists(datafile):
                    try:
                        db = anydbm.open(datafile)
                        if db:
                            db_hasdata=True
                            db.close()
                            if db_hasdata:
                                message="old checksum database used "+datafile
                                logMessage(message,cumulative_file,revision_full,printMessage=True)
                                break
                            else:
                                message="old checksum database empty "+datafile
                                logMessage(message,cumulative_file,revision_full,printMessage=True)
                    except:
                        message="old checksum database could not be opened "+datafile
                        logMessage(message,cumulative_file,revision_full,printMessage=True)
                else:
                    message="old checksum database does not exist "+datafile
                    logMessage(message,cumulative_file,revision_full,printMessage=True)
            else:
                message="cannot find any recent checksum database file",
                logMessage(message,cumulative_file,revision_full)
                message="FATAL cannot find any recent checksum database file"
                logMessage(message,cumulative_file,revision_full)
                die(message)

        oldchecksumpath = gsconf.dailydir + oldchecksum + ".db"
        if not os.path.exists(oldchecksumpath):
            message=myself+" FATAL "+"the old checksum database does not exist "+oldchecksumpath
            logMessage(message,cumulative_file,revision_full)
            die(message)

        recent_ctime = time.ctime(testtime)

        message="found recent checksum database "+oldchecksum
        logMessage(message,cumulative_file,revision_full,printMessage=False)
        logMessage(message,regression_file,revision_full)
        logMessage(message,history_file,revision_full,printMessage=False)

        difference_results = 'The following regression changes happened since recent report: '+oldchecksum
        difference_results += '\n'

        command=gsconf.testdiff_script + " " + oldchecksum + " " + newchecksum
        if options.capture:
            capture=" 2>&1"
            command+=capture

        logMessage(command,regression_file,revision_full,printMessage=False)
        logMessage(command,cumulative_file,revision_full)
        p = os.popen(command)
        if p:
            for line in p.readlines():
                if printRegression: print line
                difference_results += line
            p.close()
        else:
            difference_results = difference_results + 'no results from difference script\n'

        message="get baseline updates"
        logMessage(message,regression_file,revision_full,printMessage=False)
        logMessage(message,cumulative_file,revision_full)


        message="marcos: about to call get_baseline_log_script"
        logMessage(message,cumulative_file,revision_full)

        # Get updated baseline_changes
        baseline_changes = 'The following files had their baselines updated:\n'

        command=gsconf.get_baseline_log_script + " '" + recent_ctime + "'"
        baseline_changes += command+"\n"

        logMessage(command,cumulative_file,revision_full)
        p = Popen4(command,1)
        if p:
            for line in p.fromchild.readlines():
                baseline_changes += line
                p.wait()
        else:
            baseline_changes = baseline_changes + 'no results from baseline script\n'
            baseline_changes += "\n"

        message="marcos: done with get_baseline_log_script"
        logMessage(message,cumulative_file,revision_full)

        logMessage(baseline_changes,regression_file,revision_full)
        logMessage(difference_results,regression_file,revision_full)
    else:
        difference_results="difference testing was skipped"
    difference_results += '\n'
    # end if testRegression
    
    message="marcos: writing to differences file"
    logMessage(message,cumulative_file,revision_full)

    differences_file = open(differences_name, "w")
    message="differences"
    differences_file.write(message)
    logMessage(message,differences_file,revision_full,printMessage=False)


    if baselineChanges:
        print

        baselines_file = open(baselines_name, "w")
        message="baseline changes"
        baselines_file.write(message)
        logMessage(baseline_changes,baselines_file,revision_full,printMessage=False)

    else:
        baseline_changes = "difference testing was skipped"
        
    message="marcos: done writing to differences file"
    logMessage(message,cumulative_file,revision_full)

    # end if baselineChanges

    mail_message+=difference_results+"\n"
    mail_message+=regression_results+"\n"
    mail_message+=baseline_changes+"\n"

    email_file = open(email_name, "w")
    subject='gs regression report - %s - revision: %s' % (now,revision_full)

    message="subject "+subject
    logMessage(message,cumulative_file,revision_full)
    logMessage(message,email_file,revision_full,printMessage=False)

    message="send mail to "+gsconf.report_to
    logMessage(message,cumulative_file,revision_full)
    logMessage(message,email_file,revision_full,printMessage=False)

    logMessage(mail_message,email_file,revision_full,printMessage=False)
    email_file.close()

    if options.printmailmessage:
        print
        print "mail_message start"
        print mail_message
        print "mail_message end"    
        print

    while True:
        try:
            print "sendmail"
            sendmail(gsconf.report_from, gsconf.report_to, subject, mail_message)
            break
            time.sleep(5)
        except:
            print "sendmail fail"
            time.sleep(15)
            pass

    message="done"
    logMessage(message,cumulative_file,revision_full)

    regression_file.close()
    cumulative_file.close()
    history_file.close()

    print regression_name
    print cumulative_name
    print regression_name

    return 0

#########################################################

if __name__ == "__main__":

    os.umask(0002)

    optionsParser=optparse.OptionParser()

    optionsParser.add_option('--version',action='store_true',help="get my version")

    optionsParser.add_option('--removeonly',action='store_true',help="remove the build directory - do nothing else")

    optionsParser.add_option('--noupdate',action='store_true',help="do not update the gs executable")
    # "update" includes svn, configure, makeclean, clean, install

    optionsParser.add_option('--nosvn',action='store_true',help="do not update the source from the svn repository")
    optionsParser.add_option('--noconfigure',action='store_true',help="do not run auto configure")
    optionsParser.add_option('--nomakeclean',action='store_true',help="do not make clean before make")
    optionsParser.add_option('--nomake',action='store_true',help="do not make")

    optionsParser.add_option('--nocapture',action='store_true',help="do not capture stdout and stderr from commands")


    optionsParser.add_option('--noregression',action='store_true',help="do not run regression")
    optionsParser.add_option('--notestregression',action='store_true',help="do not test regression")
    optionsParser.add_option('--nobaselinechanges',action='store_true',help="do not analyze baselinechanges")
    optionsParser.add_option('--printregression',action='store_true',help="print regression output")
    optionsParser.add_option('--printmailmessage',action='store_true',help="print mail message")

    optionsParser.add_option('--fullset',action='store_true',help="full gsparamsets")
    optionsParser.add_option('--minset',action='store_true',help="minimum gsparamsets")
    optionsParser.add_option('--maxset',action='store_true',help="maximum gsparamsets")
    optionsParser.add_option('--oneset',action='store_true',help="one gsparamsets")    

    optionsParser.add_option('--release',action='store_true',help="under contruction")
    optionsParser.add_option('--oldrevision',action='store',help="old revision for checksum comparison",default=None)

    (options,arguments)=myoptparse.parseCommandLine(optionsParser,deviceSkip=True,testfileSkip=True,listfileSkip=True)

    if options.version:
        print options.myself,"version",myversion
        sys.exit(1)

    result = run_nightly(options,arguments)

    sys.exit(result)