ci_check

#! /usr/bin/env python3
################################################################################
#
# Copyright 2020 OpenHW Group
#
# Licensed under the Solderpad Hardware Licence, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://solderpad.org/licenses/
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier:Apache-2.0 WITH SHL-2.0
#
################################################################################
#
# ci_check: python script to run a sanity regression.  Intended to be used
#           to check updates prior to a pull-request. Uses the same
#           .metrics.json control script as the Metrics CI tool-chain.
#           Compiles and executes whatever is listed in the "regressions"
#           list-of-dictionaries.
#
# Author: Mike Thompson
#  email: mike@openhwgroup.org
#
# Written with Python 3.6.9 on Ubuntu 18.04.  Your python mileage may vary.
#
# Restriction:
#     - Blindly uses .metrics.json with no ability for user over-ride.
#
# TODO:
#      1. Check results using the "isPass" key.
#      2. Handle Verilator a little more elegantly.
#      3. Terminate if compile fails.
################################################################################

import json
import sys
import os
import argparse
import subprocess
import pprint
import yaml
import re
import shutil

if (sys.version_info < (3,0,0)):
    print ('Requires python 3')
    exit(1)

################################################################################
# Globals....
name_of_ci_check_regression = '<core>_ci_check_dev' # must match the name of one regression list in .metrics.json
core_tests                  = ['misalign', 'illegal', 'dhrystone', 'fibonacci', 'riscv_ebreak_test_0']
try:
    default_core            = os.environ['CV_CORE']
except KeyError:
    default_core            = 'None'

# This script is run from the "bin" directory, but the paths used by simulator
# commands assume we are at the root of the repo.
topdir = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
print('ci_check: topdir  : {}'.format(topdir))

################################################################################
# Command-line arguments

parser = argparse.ArgumentParser()
parser.add_argument("-s", "--simulator",     help="SystemVerilog simulator",                          choices=['dsim', 'xrun', 'vsim', 'vcs', 'riviera'])
parser.add_argument("--core",                help="Set the core to test (default: {})".format(default_core), default=default_core)
parser.add_argument("-d", "--debug",         help="Display debug messages",                           action="store_true")
parser.add_argument("-p", "--print_command", help="Print commands to stdout, do not run",             action="store_true")
parser.add_argument("-c", "--check_only",    help="Check previosu results (do not run)",              action="store_true")
parser.add_argument("-k", "--keep",          help="Keep previous cloned or generated files",          action="store_true")
parser.add_argument("-v", "--verilator",     help="Run Verilator on the CORE testbench",              action="store_true")
parser.add_argument("-u", "--no_uvm",        help="DO NOT run CI regression on the UVM testbench",    action="store_true")
parser.add_argument('--iss',                 help='Force USE_ISS flag to each test run (use 0 or 1), default: Enabled')
parser.add_argument("--repo",                help="Use this repo for the RTL, not one in Makefile",   type=str)
parser.add_argument("--branch",              help="Use this branch for the RTL, not one in Makefile", type=str)
parser.add_argument("--hash",                help="Use this hash for the RTL, not one in Makefile",   type=str)

args   = parser.parse_args()
debug  = 0       # warning, too much info!
prcmd  = 0       # prints cmds to stdout
veril  = 0       # run Verilator on CORE when set
uvm    = 1       # Run UVM CI regression by default
################################################################################

# Set correct simulator output path
if 'CV_RESULTS' in os.environ:
    uvm_results_dir = os.path.abspath(os.getenv('CV_RESULTS'))
else:
    uvm_results_dir = os.path.abspath(os.path.join(topdir, args.core.lower(), 'sim/uvmt'))

sim_results_dir = os.path.abspath(os.path.join(uvm_results_dir, '{}_results'.format(args.simulator)))

################################################################################
# Methods....

# Pretty (or at least, not so ugly) pass/fail summary
def pr_result(line='/really/long/ugly/path/test.log: SIMULATION PASSED'):
    rline = line.decode('utf-8').rstrip()     # remove trailing whitespace
    if not re.search('SIMULATION', rline):
      print('!!! Unexpected test result: {}'.format(rline))
      exit(1)
    plist = rline.split('/')                  # build a list from path
    res   = plist[-1]                         # result we want is in last entry of list
    iii   = res.find(':')                     # ":" is delimiter between file name and pass-or-fail
    rlist = [ res[0:iii], ':', res[-7:] ]     # result list (works becaused " PASSED", " FAILED" and "ABORTED" all have 7 chars)
    print('{: >40} {} {: >7}'.format(*rlist)) # pretty printing: produces "test.log:  PASSED" using default value of arg


# Check results and print something useful
def check_uvm_results(check_only=0):
    fail_count = 0
    expct_fail = 0
    pass_count = 0

    if os.path.exists(sim_results_dir):
        fails = subprocess.Popen('grep "SIMULATION FAILED" `find {} -name "*.log" -print`'.format(sim_results_dir),
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT,
                                  shell="TRUE")
        passes = subprocess.Popen('grep "SIMULATION PASSED" `find {} -name "*.log" -print`'.format(sim_results_dir),
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT,
                                  shell="TRUE")

        print ('\n\nCI Check results:')
        for line in fails.stdout.readlines():
            pr_result(line)
            fail_count += 1
            # TODO: make this a list of known failures (hopefully there won't be that many...)
            if (
                (re.search('riscv_compliance', line.decode('utf-8')))
                # or (re.search('riscv_ebreak', line.decode('utf-8')))
               ):
                expct_fail += 1

        for line in passes.stdout.readlines():
            pr_result(line)
            pass_count += 1

        if (pass_count == 0):
            print ('\nNo UVM logfiles found.\n')
        elif ((not check_only) and ((pass_count+fail_count) != num_tests)):
            print ('CI Check FAILED: Expected '+str(num_tests)+' tests to run but found only '+str(fail_count+pass_count)+' PASSED or FAILED messages');
        elif ((fail_count == 0) and (pass_count >=3)):
            print ('\nCI Check PASSED with no failures.')
            print ('OK to issue a pull-request.\n')
        elif (fail_count == expct_fail):
            print ('\nCI Check PASSED with KNOWN failure(s).')
            print ('OK to issue a pull-request.\n')
        else:
            print ('\nCI Check FAILED with unknown failures.')
            print ('Please fix before issuing a pull-request.\n')
    else:
        print ('\nCI Check FAILED with non-existent sim directory: {}'.format(sim_results_dir))
        print ('Please fix before issuing a pull-request.\n')


def check_core_results(run_count):
    core_runs = subprocess.Popen('grep "EXIT SUCCESS" -R -I ../{}/sim/core/simulation_results'.format(args.core.lower()),
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT,
                                 shell="TRUE")

    if (run_count == 0):
        print ('\nNo CORE logfiles found.\n')
    else:
        for line in core_runs.stdout.readlines():
            #core_runs = line.decode('utf-8').rstrip()
            print (line.decode('utf-8').rstrip())
            run_count -= 1

        if (run_count == 0):
            print ('\nCI Check of CORE testbench PASSED.\nYou must also run the UVM CI regression before submitting a PR.')
        else:
            print ('\nCI Check of CORE testbench FAILED.')
            print ('Please fix before issuing a pull-request.\n')


# This script may do some unexpected things, so give the user an escape hatch.
def ask_user():
    txt = input("Is this what you want [Y/N]? ")
    if not ((txt == 'Y') or (txt == 'y')):
        exit(1)

# Load regression YAML
def load_regress_yaml(regression):
    '''Load the regression yaml and return the dictionary'''
    full_regression = os.path.join(topdir, '{}/regress'.format(args.core.lower()), regression + '.yaml')
    fh = open(full_regression, 'r')
    # Newer PyYAMLs must specify explicit loader (policy) or will issue warnings
    # Older PyYAMLs will not support the Loader argument
    # So try the new way first, then catch to the old way
    try:
        dict = yaml.load(fh, Loader=yaml.FullLoader)
    except AttributeError:
        dict = yaml.load(fh)
    fh.close()

    return dict

################################################################################
# Handle Command-line arguments

if (args.debug):
    debug = 1

if (args.print_command):
    prcmd = 1

if (args.core == 'None'):
    print('No default core specified in CV_CORE and no --core defined')
    exit(1)

if (not os.path.exists(os.path.join(topdir, args.core.lower(), 'sim/uvmt/Makefile'))):
    print('Core directory for {} not found'.format(args.core))
    exit(1)

if (not args.verilator and args.no_uvm):
    print ('Specifying --no_uvm without --verilator means I do nothing...  Type `ci_check -h` for usage.')
    exit(1)

if (args.check_only):
    if (args.verilator):
        check_core_results(len(core_tests)+1) # +1 because 'make' runs hello-world
    if not (args.no_uvm):
        check_uvm_results(args.check_only)
    exit(0)

if (args.no_uvm):
    uvm = 0

if (args.verilator):
    veril = 1
elif (args.simulator == None):
    print ('Must specify a simulator.  Type `ci_check -h` to see how')
    exit(0)
elif (not(shutil.which(args.simulator))):
    print ('ERROR: simulator='+args.simulator+' but executable not found')
    exit(0)
else:
    svtool = args.simulator

name_of_ci_check_regression = name_of_ci_check_regression.replace('<core>', args.core.lower())
print('ci_check: core : {}'.format(args.core))
print('ci_check: name_of_ci_check_regression : {}'.format(name_of_ci_check_regression))
os.environ['CV_CORE'] = args.core.upper()

# --print_command is set: do not actually _do_ anything
if not (prcmd):
    if (args.keep):
        print ('Keeping previously cloned version of the RTL plus any previously generated files')
        ask_user()
    else:
        print ('This will delete your previously cloned RTL repo plus all previously generated files')
        ask_user()
        os.chdir(os.path.abspath(os.path.join(topdir, args.core.lower(), 'sim/uvmt')))
        os.system('make clean_all')
        os.chdir(os.path.join(topdir, '{}/sim/core'.format(args.core.lower())))
        os.system('make clean_all')
        os.chdir(os.path.join(topdir, 'bin'))

################################################################################
# script starts here




# This script is run from the "ci" directory, but the paths used by simulator
# commands assume we are at the root of the repo.
os.chdir(topdir)

#
# Step 1: Run the User Regression for the UVM verification environment.
#
# .metrics.json is the CI regression config used by Metrics CI tools.
if (uvm):
    with open(os.path.join(topdir, '.metrics.json')) as f:
      metrics_dict = json.load(f)

    # Get the build command
    for key in metrics_dict:
        if (key == 'builds'):
            builds_dict = metrics_dict['builds']
            if (debug):
                print (json.dumps(builds_dict, indent=2, sort_keys=True))
            for key in builds_dict:
                if (key == 'list'):
                   list_dict = builds_dict['list']
                   if (debug):
                       print (json.dumps(list_dict, indent=2, sort_keys=True))
            for key in list_dict:
                if (key['name'] != 'uvmt_{}'.format(args.core.lower())):
                    continue
                build_cmd_list = (key['cmd']).split()
                build_cmd = ' '.join(build_cmd_list[0:-1])
                build_cmd = build_cmd.replace(' DSIM_WORK=/mux-flow/build/repo/dsim_work', '')
                build_cmd = build_cmd.replace(' SIM_RESULTS=/mux-flow/build/results', '')
                if (build_cmd != ''):
                    build_cmd = build_cmd.replace('dsim', svtool)
                    if (args.repo):
                        build_cmd = build_cmd + ' CV_CORE_REPO=' + args.repo
                    if (args.branch):
                        build_cmd = build_cmd + ' CV_CORE_BRANCH=' + args.branch
                    if (args.hash):
                        build_cmd = build_cmd + ' CV_CORE_HASH=' + args.hash
                    if (prcmd or debug):
                        print(build_cmd)
                    else:
                        os.system(build_cmd)
                        os.chdir(topdir)      # cmd in .metrics.json assumes all cmds start from here
                else:
                    print ('ERROR: cannot find build command in .metrics.json')
                    exit(0)

    # Get the simulation command(s)
    for key in metrics_dict:
        if (key == 'regressions'):
            regressions_dict = metrics_dict['regressions']
            if (debug):
                print (json.dumps(regressions_dict, indent=2))
            for item in regressions_dict:
                if (item['name'] != name_of_ci_check_regression):
                    continue
                else:
                    if (debug):
                        print('Running', name_of_ci_check_regression)

                # Load it in here!
                m = re.search('\-\-file=(\w+)', item['tests']['listCmd'])
                if not m:
                    print('Cannot parse listCmd: {}'.format(item['tests']['listCmd']))
                    exit(1)
                lists_dict = load_regress_yaml(m.group(1))['tests']

                if (debug):
                    pprint.pprint(lists_dict)

                num_tests = 0
                for key in lists_dict.values():
                    run_cmd = key['cmd']

                    if run_cmd == '':
                        print ('ERROR: cannot find run command in .metrics.json')
                        exit(1)

                    try:
                        if key['num'] > 1:
                            num = key['num']
                    except KeyError:
                        num = 1

                    # The iss command-line switch takes precedence,
                    # Otherwise use what is in the regression yaml if defined
                    # Otherwise set to 1
                    if args.iss != None:
                        iss = int(args.iss)
                    else:
                        try:
                            iss = key['iss']
                        except KeyError:
                            iss = 1

                    for n in range(num):
                        # Add directory
                        full_run_cmd = 'cd {}; {}'.format(key['dir'], run_cmd)
                        # Add SIMULATOR
                        full_run_cmd += ' SIMULATOR={}'.format(svtool)
                        # Add indices
                        full_run_cmd += ' GEN_START_INDEX={} RUN_INDEX={}'.format(n, n)
                        # Add ISS
                        full_run_cmd += ' USE_ISS={}'.format('YES' if iss else 'NO')

                        # Only DSIM can run corev-dv (riscv-dv) at present
                        if ( svtool == 'dsim' or svtool == 'xrun' or svtool == 'vsim' or svtool == 'riviera' or svtool == 'vcs'):
                            num_tests+=1
                            if (prcmd or debug):
                                print(full_run_cmd)
                            else:
                                os.system(full_run_cmd)
                                os.chdir(topdir)      # cmd in .metrics.json assumes all cmds start from here
                        else:
                            num_tests+=1
                            if not ( re.search('corev_', full_run_cmd) ):
                                if (prcmd or debug):
                                    print(full_run_cmd)
                                else:
                                    os.system(full_run_cmd)
                                    os.chdir(topdir)      # cmd in .metrics.json assumes all cmds start from here

else:  # if(uvm):
    print ('Not running UVM CI regression')

#
# Step 2: Optionally run a few testcases on the CORE testbench using verilator.
# TODO: this is pretty brain-dead...
#
if  (veril):
    print ('Running Verilator tests on CORE testbench...')
    if (debug):
        print('{}/sim/core'.format(args.core.lower()))
    os.chdir(os.path.join(topdir,'{}/sim/core'.format(args.core.lower())))

    if (prcmd or debug):
        print('make')
        for core_test in core_tests:
            print('make veri-test TEST=' + core_test)
    if not (prcmd):
        os.system('make')
        for core_test in core_tests:
            os.system('make veri-test TEST=' + core_test)

os.chdir(topdir)      # cmd in .metrics.json assumes all cmds start from here

# Unless this is just a run to dump simulation commands (--print_commands),
# grep results out of the logfiles and print to stdout
os.chdir('bin')
if not (prcmd):
    if (veril):
        check_core_results(len(core_tests)+1) # +1 because 'make' runs hello-world
    if (uvm):
        check_uvm_results()
    else:
        print ('UVM CI Regression not run. (Please do so before issuing a pull-request.)')

## end ##