from __future__ import print_function

import codecs
import inspect
import io
import os
import shutil
import subprocess
import sys
import time
import traceback
import threading


def place( frame_record):
    '''
    Useful debugging function - returns representation of source position of
    caller.
    '''
    filename    = frame_record.filename
    line        = frame_record.lineno
    function    = frame_record.function
    ret = os.path.split( filename)[1] + ':' + str( line) + ':' + function + ':'
    if 0:
        tid = str(threading.currentThread())
        ret = '[' + tid + '] ' + ret
    return ret


def expand_nv( text, caller):
    '''
    Returns <text> with special handling of {<expression>} items.

    text:
        String containing {<expression>} items.
    caller:
        If an int, the number of frames to step up when looking for file:line
        information or evaluating expressions.

        Otherwise should be a frame record as returned by inspect.stack()[].

    <expression> is evaluated in <caller>'s context using eval(), and expanded
    to <expression> or <expression>=<value>.

    If <expression> ends with '=', this character is removed and we prefix the
    result with <expression>=.

    E.g.:
        x = 45
        y = 'hello'
        expand_nv( 'foo {x} {y=}')
    returns:
        foo 45 y=hello

    <expression> can also use ':' and '!' to control formatting, like
    str.format().
    '''
    if isinstance( caller, int):
        frame_record = inspect.stack()[ caller]
    else:
        frame_record = caller
    frame = frame_record.frame
    try:
        def get_items():
            '''
            Yields (pre, item), where <item> is contents of next {...} or None,
            and <pre> is preceding text.
            '''
            pos = 0
            pre = ''
            while 1:
                if pos == len( text):
                    yield pre, None
                    break
                rest = text[ pos:]
                if rest.startswith( '{{') or rest.startswith( '}}'):
                    pre += rest[0]
                    pos += 2
                elif text[ pos] == '{':
                    close = text.find( '}', pos)
                    if close < 0:
                        raise Exception( 'After "{" at offset %s, cannot find closing "}". text is: %r' % (
                                pos, text))
                    yield pre, text[ pos+1 : close]
                    pre = ''
                    pos = close + 1
                else:
                    pre += text[ pos]
                    pos += 1

        ret = ''
        for pre, item in get_items():
            ret += pre
            nv = False
            if item:
                if item.endswith( '='):
                    nv = True
                    item = item[:-1]
                expression, tail = split_first_of( item, '!:')
                try:
                    value = eval( expression, frame.f_globals, frame.f_locals)
                    value_text = ('{0%s}' % tail).format( value)
                except Exception as e:
                    value_text = '{??Failed to evaluate %r in context %s:%s because: %s??}' % (
                            expression,
                            frame_record.filename,
                            frame_record.lineno,
                            e,
                            )
                if nv:
                    ret += '%s=' % expression
                ret += value_text

        return ret

    finally:
        del frame


class LogPrefixTime:
    def __init__( self, date=False, time_=True, elapsed=False):
        self.date = date
        self.time = time_
        self.elapsed = elapsed
        self.t0 = time.time()
    def __call__( self):
        ret = ''
        if self.date:
            ret += time.strftime( ' %F')
        if self.time:
            ret += time.strftime( ' %T')
        if self.elapsed:
            ret += ' (+%s)' % time_duration( time.time() - self.t0, s_format='%.1f')
        if ret:
            ret = ret.strip() + ': '
        return ret

class LogPrefixFileLine:
    def __call__( self, caller):
        if isinstance( caller, int):
            caller = inspect.stack()[ caller]
        return place( caller) + ' '

class LogPrefixScopes:
    '''
    Internal use only.
    '''
    def __init__( self):
        self.items = []
    def __call__( self):
        ret = ''
        for item in self.items:
            if callable( item):
                item = item()
            ret += item
        return ret


class LogPrefixScope:
    '''
    Can be used to insert scoped prefix to log output.
    '''
    def __init__( self, prefix):
        g_log_prefixe_scopes.items.append( prefix)
    def __enter__( self):
        pass
    def __exit__( self, exc_type, exc_value, traceback):
        global g_log_prefix
        g_log_prefixe_scopes.items.pop()


g_log_delta = 0

class LogDeltaScope:
    '''
    Can be used to temporarily change verbose level of logging.

    E.g to temporarily increase logging:

        with jlib.LogDeltaScope(-1):
            ...
    '''
    def __init__( self, delta):
        self.delta = delta
        global g_log_delta
        g_log_delta += self.delta
    def __enter__( self):
        pass
    def __exit__( self, exc_type, exc_value, traceback):
        global g_log_delta
        g_log_delta -= self.delta

# Special item that can be inserted into <g_log_prefixes> to enable
# temporary addition of text into log prefixes.
#
g_log_prefixe_scopes = LogPrefixScopes()

# List of items that form prefix for all output from log().
#
g_log_prefixes = []


def log_text( text=None, caller=1, nv=True):
    '''
    Returns log text, prepending all lines with text from g_log_prefixes.

    text:
        The text to output. Each line is prepended with prefix text.
    caller:
        If an int, the number of frames to step up when looking for file:line
        information or evaluating expressions.

        Otherwise should be a frame record as returned by inspect.stack()[].
    nv:
        If true, we expand {...} in <text> using expand_nv().
    '''
    if isinstance( caller, int):
        caller += 1
    prefix = ''
    for p in g_log_prefixes:
        if callable( p):
            if isinstance( p, LogPrefixFileLine):
                p = p(caller)
            else:
                p = p()
        prefix += p

    if text is None:
        return prefix

    if nv:
        text = expand_nv( text, caller)

    if text.endswith( '\n'):
        text = text[:-1]
    lines = text.split( '\n')

    text = ''
    for line in lines:
        text += prefix + line + '\n'
    return text



s_log_levels_cache = dict()
s_log_levels_items = []

def log_levels_find( caller):
    if not s_log_levels_items:
        return 0

    tb = traceback.extract_stack( None, 1+caller)
    if len(tb) == 0:
        return 0
    filename, line, function, text = tb[0]

    key = function, filename, line,
    delta = s_log_levels_cache.get( key)

    if delta is None:
        # Calculate and populate cache.
        delta = 0
        for item_function, item_filename, item_delta in s_log_levels_items:
            if item_function and not function.startswith( item_function):
                continue
            if item_filename and not filename.startswith( item_filename):
                continue
            delta = item_delta
            break

        s_log_levels_cache[ key] = delta

    return delta


def log_levels_add( delta, filename_prefix, function_prefix):
    '''
    log() calls from locations with filenames starting with <filename_prefix>
    and/or function names starting with <function_prefix> will have <delta>
    added to their level.

    Use -ve delta to increase verbosity from particular filename or function
    prefixes.
    '''
    log( 'adding level: {filename_prefix=!r} {function_prefix=!r}')

    # Sort in reverse order so that long functions and filename specs come
    # first.
    #
    s_log_levels_items.append( (function_prefix, filename_prefix, delta))
    s_log_levels_items.sort( reverse=True)


def log( text, level=0, caller=1, nv=True, out=None):
    '''
    Writes log text, with special handling of {<expression>} items in <text>
    similar to python3's f-strings.

    text:
        The text to output.
    caller:
        How many frames to step up to get caller's context when evaluating
        file:line information and/or expressions. Or frame record as returned
        by inspect.stack()[].
    nv:
        If true, we expand {...} in <text> using expand_nv().
    out:
        Where to send output. If None we use sys.stdout.

    <expression> is evaluated in our caller's context (<n> stack frames up)
    using eval(), and expanded to <expression> or <expression>=<value>.

    If <expression> ends with '=', this character is removed and we prefix the
    result with <expression>=.

    E.g.:
        x = 45
        y = 'hello'
        expand_nv( 'foo {x} {y=}')
    returns:
        foo 45 y=hello

    <expression> can also use ':' and '!' to control formatting, like
    str.format().
    '''
    if out is None:
        out = sys.stdout
    level += g_log_delta
    if isinstance( caller, int):
        caller += 1
    level += log_levels_find( caller)
    if level <= 0:
        text = log_text( text, caller, nv=nv)
        out.write( text)
        out.flush()


def log0( text, caller=1, nv=True, out=None):
    '''
    Most verbose log. Same as log().
    '''
    log( text, level=0, caller=caller+1, nv=nv, out=out)

def log1( text, caller=1, nv=True, out=None):
    log( text, level=1, caller=caller+1, nv=nv, out=out)

def log2( text, caller=1, nv=True, out=None):
    log( text, level=2, caller=caller+1, nv=nv, out=out)

def log3( text, caller=1, nv=True, out=None):
    log( text, level=3, caller=caller+1, nv=nv, out=out)

def log4( text, caller=1, nv=True, out=None):
    log( text, level=4, caller=caller+1, nv=nv, out=out)

def log5( text, caller=1, nv=True, out=None):
    '''
    Least verbose log.
    '''
    log( text, level=5, caller=caller+1, nv=nv, out=out)

def logx( text, caller=1, nv=True, out=None):
    '''
    Does nothing, useful when commenting out a log().
    '''
    pass


def log_levels_add_env( name='JLIB_log_levels'):
    '''
    Added log levels encoded in an environmental variable.
    '''
    t = os.environ.get( name)
    if t:
        for ffll in t.split( ','):
            ffl, delta = ffll.split( '=', 1)
            delta = int( delta)
            ffl = ffl.split( ':')
            if 0:
                pass
            elif len( ffl) == 1:
                filename = ffl
                function = None
            elif len( ffl) == 2:
                filename, function = ffl
            else:
                assert 0
            log_levels_add( delta, filename, function)


def strpbrk( text, substrings):
    '''
    Finds first occurrence of any item in <substrings> in <text>.

    Returns (pos, substring) or (len(text), None) if not found.
    '''
    ret_pos = len( text)
    ret_substring = None
    for substring in substrings:
        pos = text.find( substring)
        if pos >= 0 and pos < ret_pos:
            ret_pos = pos
            ret_substring = substring
    return ret_pos, ret_substring


def split_first_of( text, substrings):
    '''
    Returns (pre, post), where <pre> doesn't contain any item in <substrings>
    and <post> is empty or starts with an item in <substrings>.
    '''
    pos, _ = strpbrk( text, substrings)
    return text[ :pos], text[ pos:]



log_levels_add_env()


def force_line_buffering():
    '''
    Ensure sys.stdout and sys.stderr are line-buffered. E.g. makes things work
    better if output is piped to a file via 'tee'.

    Returns original out,err streams.
    '''
    stdout0 = sys.stdout
    stderr0 = sys.stderr
    sys.stdout = os.fdopen( os.dup( sys.stdout.fileno()), 'w', 1)
    sys.stderr = os.fdopen( os.dup( sys.stderr.fileno()), 'w', 1)
    return stdout0, stderr0


def exception_info( exception=None, limit=None, out=None, prefix='', oneline=False):
    '''
    General replacement for traceback.* functions that print/return information
    about exceptions. This function provides a simple way of getting the
    functionality provided by these traceback functions:

        traceback.format_exc()
        traceback.format_exception()
        traceback.print_exc()
        traceback.print_exception()

    Returns:
        A string containing description of specified exception and backtrace.

    Inclusion of outer frames:
        We improve upon traceback.* in that we also include stack frames above
        the point at which an exception was caught - frames from the top-level
        <module> or thread creation fn to the try..catch block, which makes
        backtraces much more useful.

        Google 'sys.exc_info backtrace incomplete' for more details.

        We deliberately leave a slightly curious pair of items in the backtrace
        - the point in the try: block that ended up raising an exception, and
        the point in the associated except: block from which we were called.

        For clarity, we insert an empty frame in-between these two items, so
        that one can easily distinguish the two parts of the backtrace.

        So the backtrace looks like this:

            root (e.g. <module> or /usr/lib/python2.7/threading.py:778:__bootstrap():
            ...
            file:line in the except: block where the exception was caught.
            ::(): marker
            file:line in the try: block.
            ...
            file:line where the exception was raised.

        The items after the ::(): marker are the usual items that traceback.*
        shows for an exception.

    Also the backtraces that are generated are more concise than those provided
    by traceback.* - just one line per frame instead of two - and filenames are
    output relative to the current directory if applicatble. And one can easily
    prefix all lines with a specified string, e.g. to indent the text.

    Returns a string containing backtrace and exception information, and sends
    returned string to <out> if specified.

    exception:
        None, or a (type, value, traceback) tuple, e.g. from sys.exc_info(). If
        None, we call sys.exc_info() and use its return value.
    limit:
        None or maximum number of stackframes to output.
    out:
        None or callable taking single <text> parameter or object with a
        'write' member that takes a single <text> parameter.
    prefix:
        Used to prefix all lines of text.
    '''
    if exception is None:
        exception = sys.exc_info()
    etype, value, tb = exception

    if sys.version_info[0] == 2:
        out2 = io.BytesIO()
    else:
        out2 = io.StringIO()
    try:

        frames = []

        # Get frames above point at which exception was caught - frames
        # starting at top-level <module> or thread creation fn, and ending
        # at the point in the catch: block from which we were called.
        #
        # These frames are not included explicitly in sys.exc_info()[2] and are
        # also omitted by traceback.* functions, which makes for incomplete
        # backtraces that miss much useful information.
        #
        for f in reversed(inspect.getouterframes(tb.tb_frame)):
            ff = f[1], f[2], f[3], f[4][0].strip()
            frames.append(ff)

        if 1:
            # It's useful to see boundary between upper and lower frames.
            frames.append( None)

        # Append frames from point in the try: block that caused the exception
        # to be raised, to the point at which the exception was thrown.
        #
        # [One can get similar information using traceback.extract_tb(tb):
        #   for f in traceback.extract_tb(tb):
        #       frames.append(f)
        # ]
        for f in inspect.getinnerframes(tb):
            ff = f[1], f[2], f[3], f[4][0].strip()
            frames.append(ff)

        cwd = os.getcwd() + os.sep
        if oneline:
            if etype and value:
                # The 'exception_text' variable below will usually be assigned
                # something like '<ExceptionType>: <ExceptionValue>', unless
                # there was no explanatory text provided (e.g. "raise Exception()").
                # In this case, str(value) will evaluate to ''.
                exception_text = traceback.format_exception_only(etype, value)[0].strip()
                filename, line, fnname, text = frames[-1]
                if filename.startswith(cwd):
                    filename = filename[len(cwd):]
                if not str(value):
                    # The exception doesn't have any useful explanatory text
                    # (for example, maybe it was raised by an expression like
                    # "assert <expression>" without a subsequent comma).  In
                    # the absence of anything more helpful, print the code that
                    # raised the exception.
                    exception_text += ' (%s)' % text
                line = '%s%s at %s:%s:%s()' % (prefix, exception_text, filename, line, fnname)
                out2.write(line)
        else:
            out2.write( '%sBacktrace:\n' % prefix)
            for frame in frames:
                if frame is None:
                    out2.write( '%s    ^except raise:\n' % prefix)
                    continue
                filename, line, fnname, text = frame
                if filename.startswith( cwd):
                    filename = filename[ len(cwd):]
                if filename.startswith( './'):
                    filename = filename[ 2:]
                out2.write( '%s    %s:%s:%s(): %s\n' % (
                        prefix, filename, line, fnname, text))

            if etype and value:
                out2.write( '%sException:\n' % prefix)
                lines = traceback.format_exception_only( etype, value)
                for line in lines:
                    out2.write( '%s    %s' % ( prefix, line))

        text = out2.getvalue()

        # Write text to <out> if specified.
        out = getattr( out, 'write', out)
        if callable( out):
            out( text)
        return text

    finally:
        # clear things to avoid cycles.
        exception = None
        etype = None
        value = None
        tb = None
        frames = None


def number_sep( s):
    '''
    Simple number formatter, adds commas in-between thousands. <s> can
    be a number or a string. Returns a string.
    '''
    if not isinstance( s, str):
        s = str( s)
    c = s.find( '.')
    if c==-1:   c = len(s)
    end = s.find('e')
    if end == -1:   end = s.find('E')
    if end == -1:   end = len(s)
    ret = ''
    for i in range( end):
        ret += s[i]
        if i<c-1 and (c-i-1)%3==0:
            ret += ','
        elif i>c and i<end-1 and (i-c)%3==0:
            ret += ','
    ret += s[end:]
    return ret

assert number_sep(1)=='1'
assert number_sep(12)=='12'
assert number_sep(123)=='123'
assert number_sep(1234)=='1,234'
assert number_sep(12345)=='12,345'
assert number_sep(123456)=='123,456'
assert number_sep(1234567)=='1,234,567'


class Stream:
    '''
    Base layering abstraction for streams - abstraction for things like
    sys.stdout to allow prefixing of all output, e.g. with a timestamp.
    '''
    def __init__( self, stream):
        self.stream = stream
    def write( self, text):
        self.stream.write( text)

class StreamPrefix:
    '''
    Prefixes output with a prefix, which can be a string or a callable that
    takes no parameters and return a string.
    '''
    def __init__( self, stream, prefix):
        self.stream = stream
        self.at_start = True
        if callable(prefix):
            self.prefix = prefix
        else:
            self.prefix = lambda : prefix

    def write( self, text):
        if self.at_start:
            text = self.prefix() + text
            self.at_start = False
        append_newline = False
        if text.endswith( '\n'):
            text = text[:-1]
            self.at_start = True
            append_newline = True
        text = text.replace( '\n', '\n%s' % self.prefix())
        if append_newline:
            text += '\n'
        self.stream.write( text)

    def flush( self):
        self.stream.flush()


def debug( text):
    if callable(text):
        text = text()
    print( text)

debug_periodic_t0 = [0]
def debug_periodic( text, override=0):
    interval = 10
    t = time.time()
    if t - debug_periodic_t0[0] > interval or override:
        debug_periodic_t0[0] = t
        debug(text)


def time_duration( seconds, verbose=False, s_format='%i'):
    '''
    Returns string expressing an interval.

    seconds:
        The duration in seconds
    verbose:
        If true, return like '4 days 1 hour 2 mins 23 secs', otherwise as
        '4d3h2m23s'.
    s_format:
        If specified, use as printf-style format string for seconds.
    '''
    x = abs(seconds)
    ret = ''
    i = 0
    for div, text in [
            ( 60, 'sec'),
            ( 60, 'min'),
            ( 24, 'hour'),
            ( None, 'day'),
            ]:
        force = ( x == 0 and i == 0)
        if div:
            remainder = x % div
            x = int( x/div)
        else:
            remainder = x
        if not verbose:
            text = text[0]
        if remainder or force:
            if verbose and remainder > 1:
                # plural.
                text += 's'
            if verbose:
                text = ' %s ' % text
            if i == 0:
                remainder = s_format % remainder
            ret = '%s%s%s' % ( remainder, text, ret)
        i += 1
    ret = ret.strip()
    if ret == '':
        ret = '0s'
    if seconds < 0:
        ret = '-%s' % ret
    return ret

assert time_duration( 303333) == '3d12h15m33s'
assert time_duration( 303333.33, s_format='%.1f') == '3d12h15m33.3s'
assert time_duration( 303333, verbose=True) == '3 days 12 hours 15 mins 33 secs'
assert time_duration( 303333.33, verbose=True, s_format='%.1f') == '3 days 12 hours 15 mins 33.3 secs'

assert time_duration( 0) == '0s'
assert time_duration( 0, verbose=True) == '0 sec'


def date_time( t=None):
    if t is None:
        t = time.time()
    return time.strftime( "%F-%T", time.gmtime( t))

def stream_prefix_time( stream):
    '''
    Returns StreamPrefix that prefixes lines with time and elapsed time.
    '''
    t_start = time.time()
    def prefix_time():
        return '%s (+%s): ' % (
                time.strftime( '%T'),
                time_duration( time.time() - t_start, s_format='0.1f'),
                )
    return StreamPrefix( stream, prefix_time)

def stdout_prefix_time():
    '''
    Changes sys.stdout to prefix time and elapsed time; returns original
    sys.stdout.
    '''
    ret = sys.stdout
    sys.stdout = stream_prefix_time( sys.stdout)
    return ret


def make_stream( out):
    '''
    If <out> already has a .write() member, returns <out>.

    Otherwise a stream-like object with a .write() method that writes to <out>.

    out:
        Where output is sent.
        If None, output is lost.
        Otherwise if an integer, we do: os.write( out, text)
        Otherwise if callable, we do: out( text)
        Otherwise we assume <out> is python stream or similar already.
    '''
    if getattr( out, 'write', None):
        return out
    class Ret:
        def flush():
            pass
    ret = Ret()
    if out is None:
        ret.write = lambda text: None
    elif isinstance( out, int):
        ret.write = lambda text: os.write( out, text)
    elif callable( out):
        ret.write = out
    else:
        ret.write = lambda text: out.write( text)
    return ret


def system_raw(
        command,
        out=None,
        shell=True,
        encoding='latin_1',
        errors='strict',
        buffer_len=-1,
        ):
    '''
    Runs command, writing output to <out> which can be an int fd, a python
    stream or a Stream object.

    Args:
        command:
            The command to run.
        out:
            Where output is sent.
            If None, output is lost.
            If -1, output is sent to stdout and stderr.
            Otherwise if an integer, we do: os.write( out, text)
            Otherwise if callable, we do: out( text)
            Otherwise we assume <out> is python stream or similar, and do: out.write(text)
        shell:
            Whether to run command inside a shell (see subprocess.Popen).
        encoding:
            Sepecify the encoding used to translate the command's output
            to characters.

            Note that if <encoding> is None and we are being run by python3,
            <out> will be passed bytes, not a string.

            Note that latin_1 will never raise a UnicodeDecodeError.
        errors:
            How to handle encoding errors; see docs for codecs module for
            details.
        buffer_len:
            The number of bytes we attempt to read at a time. If -1 we read
            output one line at a time.

    Returns:
        subprocess's <returncode>, i.e. -N means killed by signal N, otherwise
        the exit value (e.g. 12 if command terminated with exit(12)).
    '''
    if out == -1:
        stdin = 0
        stdout = 1
        stderr = 2
    else:
        stdin = None
        stdout = subprocess.PIPE
        stderr = subprocess.STDOUT
    child = subprocess.Popen(
            command,
            shell=shell,
            stdin=stdin,
            stdout=stdout,
            stderr=stderr,
            close_fds=True,
            #encoding=encoding - only python-3.6+.
            )

    child_out = child.stdout
    if encoding:
        child_out = codecs.getreader( encoding)( child_out, errors)

    out = make_stream( out)

    if stdout == subprocess.PIPE:
        if buffer_len == -1:
            for line in child_out:
                out.write( line)
        else:
            while 1:
                text = child_out.read( buffer_len)
                if not text:
                    break
                out.write( text)
    #decode( lambda : os.read( child_out.fileno(), 100), outfn, encoding)

    return child.wait()

if __name__ == '__main__':

    if os.getenv( 'jtest_py_system_raw_test') == '1':
        out = io.StringIO()
        system_raw(
                'jtest_py_system_raw_test=2 python jlib.py',
                sys.stdout,
                encoding='utf-8',
                #'latin_1',
                errors='replace',
                )
        print( repr( out.getvalue()))

    elif os.getenv( 'jtest_py_system_raw_test') == '2':
        for i in range(256):
            sys.stdout.write( chr(i))


def system(
        command,
        verbose=None,
        raise_errors=True,
        out=None,
        prefix=None,
        rusage=False,
        shell=True,
        encoding=None,
        errors='replace',
        buffer_len=-1,
        ):
    '''
    Runs a command like os.system() or subprocess.*, but with more flexibility.

    We give control over where the command's output is sent, whether to return
    the output and/or exit code, and whether to raise an exception if the
    command fails.

    We also support the use of /usr/bin/time to gather rusage information.

        command:
            The command to run.
        verbose:
            If true, we output information about the command that we run, and
            its result.

            If callable or something with a .write() method, information is
            sent to <verbose> itself. Otherwise it is sent to <out> (without
            applying <prefix>).
        raise_errors:
            If true, we raise an exception if the command fails, otherwise we
            return the failing error code or zero.
        out:
            Python stream, fd, callable or Stream instance to which output is
            sent.

            If <out> is 'return', we buffer the output and return (e,
            <output>). Note that if raise_errors is true, we only return if <e>
            is zero.

            If -1, output is sent to stdout and stderr.
        prefix:
            If not None, should be prefix string or callable used to prefix
            all output. [This is for convenience to avoid the need to do
            out=StreamPrefix(...).]
        rusage:
            If true, we run via /usr/bin/time and return rusage string
            containing information on execution. <raise_errors> and
            out='return' are ignored.
        shell:
            Passed to underlying subprocess.Popen() call.
        encoding:
            Sepecify the encoding used to translate the command's output
            to characters. Defaults to utf-8.
        errors:
            How to handle encoding errors; see docs for codecs module
            for details. Defaults to 'replace' so we never raise a
            UnicodeDecodeError.
        buffer_len:
            The number of bytes we attempt to read at a time. If -1 we read
            output one line at a time.

    Returns:
        If <rusage> is true, we return the rusage text.

        Else if raise_errors is true:
            If the command failed, we raise an exception.
            Else if <out> is 'return' we return the text output from the command.
            Else we return None

        Else if <out> is 'return', we return (e, text) where <e> is the
        command's exit code and <text> is the output from the command.

        Else we return <e>, the command's exit code.
    '''
    if encoding is None:
        if sys.version_info[0] == 2:
            # python-2 doesn't seem to implement 'replace' properly.
            encoding = None
            errors = None
        else:
            encoding = 'utf-8'
            errors = 'replace'

    out_original = out
    if out is None:
        out = sys.stdout
    elif out == 'return':
        # Store the output ourselves so we can return it.
        out = io.StringIO()
    else:
        out = make_stream( out)

    if verbose:
        if getattr( verbose, 'write', None):
            pass
        elif callable( verbose):
            verbose = make_stream( verbose)
        else:
            verbose = out

    if prefix:
        out = StreamPrefix( out, prefix)

    if verbose:
        verbose.write( 'running: %s\n' % command)

    if rusage:
        command2 = ''
        command2 += '/usr/bin/time -o ubt-out -f "D=%D E=%D F=%F I=%I K=%K M=%M O=%O P=%P R=%r S=%S U=%U W=%W X=%X Z=%Z c=%c e=%e k=%k p=%p r=%r s=%s t=%t w=%w x=%x C=%C"'
        command2 += ' '
        command2 += command
        system_raw( command2, out, shell, encoding, errors, buffer_len=buffer_len)
        with open('ubt-out') as f:
            rusage_text = f.read()
        #print 'have read rusage output: %r' % rusage_text
        if rusage_text.startswith( 'Command '):
            # Annoyingly, /usr/bin/time appears to write 'Command
            # exited with ...' or 'Command terminated by ...' to the
            # output file before the rusage info if command doesn't
            # exit 0.
            nl = rusage_text.find('\n')
            rusage_text = rusage_text[ nl+1:]
        return rusage_text
    else:
        e = system_raw( command, out, shell, encoding, errors, buffer_len=buffer_len)

        if verbose:
            verbose.write( '[returned e=%s]\n' % e)

        if raise_errors:
            if e:
                raise Exception( 'command failed: %s' % command)
            if out_original == 'return':
                return out.getvalue()
            return

        if out_original == 'return':
            return e, out.getvalue()
        return e

def get_gitfiles( directory, submodules=False):
    '''
    Returns list of all files known to git in <directory>; <directory> must be
    somewhere within a git checkout.

    Returned names are all relative to <directory>.

    If .git directory, we also create <directory>/jtest-git-files. Otherwise we
    assume a this file already exists.
    '''
    if os.path.isdir( '%s/.git' % directory):
        command = 'cd ' + directory + ' && git ls-files'
        if submodules:
            command += ' --recurse-submodules'
        command += ' > jtest-git-files'
        system( command, verbose=sys.stdout)

    with open( '%s/jtest-git-files' % directory, 'r') as f:
        text = f.read()
    ret = text.split( '\n')
    return ret

def get_git_id_raw( directory):
    if not os.path.isdir( '%s/.git' % directory):
        return
    text = system(
            f'cd {directory} && (PAGER= git show --pretty=oneline|head -n 1 && git diff)',
            out='return',
            )
    return text

def get_git_id( directory, allow_none=False):
    '''
    Returns text where first line is '<git-sha> <commit summary>' and remaining
    lines contain output from 'git diff' in <directory>.

    directory:
        Root of git checkout.
    allow_none:
        If true, we return None if <directory> is not a git checkout and
        jtest-git-id file does not exist.
    '''
    filename = f'{directory}/jtest-git-id'
    text = get_git_id_raw( directory)
    if text:
        with open( filename, 'w') as f:
            f.write( text)
    elif os.path.isfile( filename):
        with open( filename) as f:
            text = f.read()
    else:
        if not allow_none:
            raise Exception( f'Not in git checkout, and no file {filename}.')
        text = None
    return text

class Args:
    '''
    Iterates over argv items. Does getopt-style splitting of args starting with
    single '-' character.
    '''
    def __init__( self, argv):
        self.argv = argv
        self.pos = 0
        self.pos_sub = None
    def next( self):
        while 1:
            if self.pos >= len(self.argv):
                raise StopIteration()
            arg = self.argv[self.pos]
            if (not self.pos_sub
                    and arg.startswith('-')
                    and not arg.startswith('--')
                    ):
                # Start splitting current arg.
                self.pos_sub = 1
            if self.pos_sub and self.pos_sub >= len(arg):
                # End of '-' sub-arg.
                self.pos += 1
                self.pos_sub = None
                continue
            if self.pos_sub:
                # Return '-' sub-arg.
                ret = arg[self.pos_sub]
                self.pos_sub += 1
                return f'-{ret}'
            # Return normal arg.
            self.pos += 1
            return arg

def update_file( text, filename):
    '''
    Writes <text> to <filename>. Does nothing if contents of <filename> are
    already <text>.
    '''
    try:
        with open( filename) as f:
            text0 = f.read()
    except OSError:
        text0 = None
    if text == text0:
        log( 'Unchanged: ' + filename)
    else:
        log( 'Updating:  ' + filename)
        # Write to temp file and rename, to ensure we are atomic.
        filename_temp = f'{filename}-jlib-temp'
        with open( filename_temp, 'w') as f:
            f.write( text)
        os.rename( filename_temp, filename)


def mtime( filename, default=0):
    '''
    Returns mtime of file, or <default> if error - e.g. doesn't exist.
    '''
    try:
        return os.path.getmtime( filename)
    except OSError:
        return default

def get_filenames( paths):
    '''
    Yields each file in <paths>, walking any directories.
    '''
    if isinstance( paths, str):
        paths = (paths,)
    for name in paths:
        if os.path.isdir( name):
            for dirpath, dirnames, filenames in os.walk( name):
                for filename in filenames:
                    path = os.path.join( dirpath, filename)
                    yield path
        else:
            yield name

def remove( path):
    '''
    Removes file or directory, without raising exception if it doesn't exist.

    We assert-fail if the path still exists when we return, in case of
    permission problems etc.
    '''
    try:
        os.remove( path)
    except Exception:
        pass
    shutil.rmtree( path, ignore_errors=1)
    assert not os.path.exists( path)


# Things for figuring out whether files need updating, using mtimes.
#
def newest( names):
    '''
    Returns mtime of newest file in <filenames>. Returns 0 if no file exists.
    '''
    assert isinstance( names, (list, tuple))
    assert names
    ret_t = 0
    ret_name = None
    for filename in get_filenames( names):
        t = mtime( filename)
        if t > ret_t:
            ret_t = t
            ret_name = filename
    return ret_t, ret_name

def oldest( names):
    '''
    Returns mtime of oldest file in <filenames> or 0 if no file exists.
    '''
    assert isinstance( names, (list, tuple))
    assert names
    ret_t = None
    ret_name = None
    for filename in get_filenames( names):
        t = mtime( filename)
        if ret_t is None or t < ret_t:
            ret_t = t
            ret_name = filename
    if ret_t is None:
        ret_t = 0
    return ret_t, ret_name

def update_needed( infiles, outfiles):
    '''
    If any file in <infiles> is newer than any file in <outfiles>, returns
    string description. Otherwise returns None.
    '''
    in_tmax, in_tmax_name = newest( infiles)
    out_tmin, out_tmin_name = oldest( outfiles)
    if in_tmax > out_tmin:
        text = f'{in_tmax_name} is newer than {out_tmin_name}'
        return text

def build(
        infiles,
        outfiles,
        command,
        force_rebuild=False,
        out=None,
        all_reasons=False,
        verbose=True,
        prefix=None,
        ):
    '''
    Ensures that <outfiles> are up to date using enhanced makefile-like
    determinism of dependencies.

    Rebuilds <outfiles> by running <command> if we determine that any of them
    are out of date.

    infiles:
        Names of files that are read by <command>. Can be a single filename. If
        an item is a directory, we expand to all filenames in the directory's
        tree.
    outfiles:
        Names of files that are written by <command>. Can also be a single
        filename.
    command:
        Command to run.
    force_rebuild:
        If true, we always re-run the command.
    out:
        A callable, passed to jlib.system(). If None, we use jlib.log() with
        our caller's stack record.
    all_reasons:
        If true we check all ways for a build being needed, even if we already
        know a build is needed; this only affects the diagnostic that we
        output.
    verbose:
        Passed to jlib.system().
    prefix:
        Passed to jlib.system().

    We compare mtimes of <infiles> and <outfiles>, and we also detect changes
    to the command itself.

    If any of infiles are newer than any of outfiles, or <command> is
    different to contents of commandfile '<outfile[0]>.cmd, then truncates
    commandfile and runs <command>. If <command> succeeds we writes <command>
    to commandfile.
    '''
    if isinstance( infiles, str):
        infiles = (infiles,)
    if isinstance( outfiles, str):
        infiles = (outfiles,)

    if not out:
        out_frame_record = inspect.stack()[1]
        out = lambda text: log( text, nv=0, caller=out_frame_record)

    command_filename = f'{outfiles[0]}.cmd'

    reasons = []

    if not reasons or all_reasons:
        if force_rebuild:
            reasons.append( 'force_rebuild was specified')

    if not reasons or all_reasons:
        try:
            with open( command_filename) as f:
                command0 = f.read()
        except Exception:
            command0 = None
        if command != command0:
            if command0:
                reasons.append( 'command has changed')
            else:
                reasons.append( 'no previous command')

    if not reasons or all_reasons:
        reason = update_needed( infiles, outfiles)
        if reason:
            reasons.append( reason)

    if not reasons:
        out( 'Already up to date: ' + ' '.join(outfiles))
        return

    if out:
        out( 'Rebuilding because %s: %s' % (
                ', and '.join( reasons),
                ' '.join(outfiles),
                ))

    # Empty <command_filename) while we run the command so that if command
    # fails but still creates target(s), then next time we will know target(s)
    # are not up to date.
    #
    with open( command_filename, 'w') as f:
        pass

    system( command, out=out, verbose=verbose, prefix=prefix)

    with open( command_filename, 'w') as f:
        f.write( command)


def link_l_flags( sos):
    '''
    Returns flags needed to link with items in <sos>. For each unique item we
    use -L with parent directory, and -l with embedded name (without leading
    'lib' or trailing '.co').
    '''
    dirs = set()
    names = []
    if isinstance( sos, str):
        sos = (sos,)
    for so in sos:
        dir_ = os.path.dirname( so)
        name = os.path.basename( so)
        assert name.startswith( 'lib')
        assert name.endswith ( '.so')
        name = name[3:-3]
        dirs.add( dir_)
        names.append( name)
    ret = ''
    # Important to use sorted() here, otherwise ordering from set() is
    # arbitrary causing occasional spurious rebuilds by jlib.build().
    for dir_ in sorted(dirs):
        ret += f' -L {dir_}'
    for name in names:
        ret += f' -l {name}'
    return ret