Mercurial > hg > CbC > CbC_llvm
view clang/utils/analyzer/SATest.py @ 266:00f31e85ec16 default tip
Added tag current for changeset 31d058e83c98
author | Shinji KONO <kono@ie.u-ryukyu.ac.jp> |
---|---|
date | Sat, 14 Oct 2023 10:13:55 +0900 |
parents | 1f2b6ac9f198 |
children |
line wrap: on
line source
#!/usr/bin/env python import argparse import sys import os from subprocess import call SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__)) PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects") DEFAULT_LLVM_DIR = os.path.realpath( os.path.join(SCRIPTS_DIR, os.path.pardir, os.path.pardir, os.path.pardir) ) def add(parser, args): import SATestAdd from ProjectMap import ProjectInfo if args.source == "git" and (args.origin == "" or args.commit == ""): parser.error("Please provide both --origin and --commit if source is 'git'") if args.source != "git" and (args.origin != "" or args.commit != ""): parser.error( "Options --origin and --commit don't make sense when " "source is not 'git'" ) project = ProjectInfo( args.name[0], args.mode, args.source, args.origin, args.commit ) SATestAdd.add_new_project(project) def build(parser, args): import SATestBuild SATestBuild.VERBOSE = args.verbose projects = get_projects(parser, args) tester = SATestBuild.RegressionTester( args.jobs, projects, args.override_compiler, args.extra_analyzer_config, args.extra_checkers, args.regenerate, args.strictness, ) tests_passed = tester.test_all() if not tests_passed: sys.stderr.write("ERROR: Tests failed.\n") sys.exit(42) def compare(parser, args): import CmpRuns choices = [ CmpRuns.HistogramType.RELATIVE.value, CmpRuns.HistogramType.LOG_RELATIVE.value, CmpRuns.HistogramType.ABSOLUTE.value, ] if args.histogram is not None and args.histogram not in choices: parser.error( "Incorrect histogram type, available choices are {}".format(choices) ) dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old) dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new) CmpRuns.dump_scan_build_results_diff( dir_old, dir_new, show_stats=args.show_stats, stats_only=args.stats_only, histogram=args.histogram, verbose_log=args.verbose_log, ) def update(parser, args): import SATestUpdateDiffs from ProjectMap import ProjectMap project_map = ProjectMap() for project in project_map.projects: SATestUpdateDiffs.update_reference_results(project, args.git) def benchmark(parser, args): from SATestBenchmark import Benchmark projects = get_projects(parser, args) benchmark = Benchmark(projects, args.iterations, args.output) benchmark.run() def benchmark_compare(parser, args): import SATestBenchmark SATestBenchmark.compare(args.old, args.new, args.output) def get_projects(parser, args): from ProjectMap import ProjectMap, Size project_map = ProjectMap() projects = project_map.projects def filter_projects(projects, predicate, force=False): return [ project.with_fields( enabled=(force or project.enabled) and predicate(project) ) for project in projects ] if args.projects: projects_arg = args.projects.split(",") available_projects = [project.name for project in projects] # validate that given projects are present in the project map file for manual_project in projects_arg: if manual_project not in available_projects: parser.error( "Project '{project}' is not found in " "the project map file. Available projects are " "{all}.".format(project=manual_project, all=available_projects) ) projects = filter_projects( projects, lambda project: project.name in projects_arg, force=True ) try: max_size = Size.from_str(args.max_size) except ValueError as e: parser.error("{}".format(e)) projects = filter_projects(projects, lambda project: project.size <= max_size) return projects def docker(parser, args): if len(args.rest) > 0: if args.rest[0] != "--": parser.error("REST arguments should start with '--'") args.rest = args.rest[1:] if args.build_image: docker_build_image() elif args.shell: docker_shell(args) else: sys.exit(docker_run(args, " ".join(args.rest))) def docker_build_image(): sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR), shell=True)) def docker_shell(args): try: # First we need to start the docker container in a waiting mode, # so it doesn't do anything, but most importantly keeps working # while the shell session is in progress. docker_run(args, "--wait", "--detach") # Since the docker container is running, we can actually connect to it call("docker exec -it satest bash", shell=True) except KeyboardInterrupt: pass finally: docker_cleanup() def docker_run(args, command, docker_args=""): try: return call( "docker run --rm --name satest " "-v {llvm}:/llvm-project " "-v {build}:/build " "-v {clang}:/analyzer " "-v {scripts}:/scripts " "-v {projects}:/projects " "{docker_args} " "satest-image:latest {command}".format( llvm=args.llvm_project_dir, build=args.build_dir, clang=args.clang_dir, scripts=SCRIPTS_DIR, projects=PROJECTS_DIR, docker_args=docker_args, command=command, ), shell=True, ) except KeyboardInterrupt: docker_cleanup() def docker_cleanup(): print("Please wait for docker to clean up") call("docker stop satest", shell=True) def main(): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers() # add subcommand add_parser = subparsers.add_parser( "add", help="Add a new project for the analyzer testing." ) # TODO: Add an option not to build. # TODO: Set the path to the Repository directory. add_parser.add_argument("name", nargs=1, help="Name of the new project") add_parser.add_argument( "--mode", action="store", default=1, type=int, choices=[0, 1, 2], help="Build mode: 0 for single file project, " "1 for scan_build, " "2 for single file c++11 project", ) add_parser.add_argument( "--source", action="store", default="script", choices=["script", "git", "zip"], help="Source type of the new project: " "'git' for getting from git " "(please provide --origin and --commit), " "'zip' for unpacking source from a zip file, " "'script' for downloading source by running " "a custom script", ) add_parser.add_argument( "--origin", action="store", default="", help="Origin link for a git repository" ) add_parser.add_argument( "--commit", action="store", default="", help="Git hash for a commit to checkout" ) add_parser.set_defaults(func=add) # build subcommand build_parser = subparsers.add_parser( "build", help="Build projects from the project map and compare results with " "the reference.", ) build_parser.add_argument( "--strictness", dest="strictness", type=int, default=0, help="0 to fail on runtime errors, 1 to fail " "when the number of found bugs are different " "from the reference, 2 to fail on any " "difference from the reference. Default is 0.", ) build_parser.add_argument( "-r", dest="regenerate", action="store_true", default=False, help="Regenerate reference output.", ) build_parser.add_argument( "--override-compiler", action="store_true", default=False, help="Call scan-build with " "--override-compiler option.", ) build_parser.add_argument( "-j", "--jobs", dest="jobs", type=int, default=0, help="Number of projects to test concurrently", ) build_parser.add_argument( "--extra-analyzer-config", dest="extra_analyzer_config", type=str, default="", help="Arguments passed to to -analyzer-config", ) build_parser.add_argument( "--extra-checkers", dest="extra_checkers", type=str, default="", help="Extra checkers to enable", ) build_parser.add_argument( "--projects", action="store", default="", help="Comma-separated list of projects to test", ) build_parser.add_argument( "--max-size", action="store", default=None, help="Maximum size for the projects to test", ) build_parser.add_argument("-v", "--verbose", action="count", default=0) build_parser.set_defaults(func=build) # compare subcommand cmp_parser = subparsers.add_parser( "compare", help="Comparing two static analyzer runs in terms of " "reported warnings and execution time statistics.", ) cmp_parser.add_argument( "--root-old", dest="root_old", help="Prefix to ignore on source files for " "OLD directory", action="store", type=str, default="", ) cmp_parser.add_argument( "--root-new", dest="root_new", help="Prefix to ignore on source files for " "NEW directory", action="store", type=str, default="", ) cmp_parser.add_argument( "--verbose-log", dest="verbose_log", help="Write additional information to LOG " "[default=None]", action="store", type=str, default=None, metavar="LOG", ) cmp_parser.add_argument( "--stats-only", action="store_true", dest="stats_only", default=False, help="Only show statistics on reports", ) cmp_parser.add_argument( "--show-stats", action="store_true", dest="show_stats", default=False, help="Show change in statistics", ) cmp_parser.add_argument( "--histogram", action="store", default=None, help="Show histogram of paths differences. " "Requires matplotlib", ) cmp_parser.add_argument("old", nargs=1, help="Directory with old results") cmp_parser.add_argument("new", nargs=1, help="Directory with new results") cmp_parser.set_defaults(func=compare) # update subcommand upd_parser = subparsers.add_parser( "update", help="Update static analyzer reference results based on the previous " "run of SATest build. Assumes that SATest build was just run.", ) upd_parser.add_argument( "--git", action="store_true", help="Stage updated results using git." ) upd_parser.set_defaults(func=update) # docker subcommand dock_parser = subparsers.add_parser( "docker", help="Run regression system in the docker." ) dock_parser.add_argument( "--build-image", action="store_true", help="Build docker image for running tests.", ) dock_parser.add_argument( "--shell", action="store_true", help="Start a shell on docker." ) dock_parser.add_argument( "--llvm-project-dir", action="store", default=DEFAULT_LLVM_DIR, help="Path to LLVM source code. Defaults " "to the repo where this script is located. ", ) dock_parser.add_argument( "--build-dir", action="store", default="", help="Path to a directory where docker should " "build LLVM code.", ) dock_parser.add_argument( "--clang-dir", action="store", default="", help="Path to find/install LLVM installation.", ) dock_parser.add_argument( "rest", nargs=argparse.REMAINDER, default=[], help="Additional args that will be forwarded " "to the docker's entrypoint.", ) dock_parser.set_defaults(func=docker) # benchmark subcommand bench_parser = subparsers.add_parser( "benchmark", help="Run benchmarks by building a set of projects multiple times." ) bench_parser.add_argument( "-i", "--iterations", action="store", type=int, default=20, help="Number of iterations for building each " "project.", ) bench_parser.add_argument( "-o", "--output", action="store", default="benchmark.csv", help="Output csv file for the benchmark results", ) bench_parser.add_argument( "--projects", action="store", default="", help="Comma-separated list of projects to test", ) bench_parser.add_argument( "--max-size", action="store", default=None, help="Maximum size for the projects to test", ) bench_parser.set_defaults(func=benchmark) bench_subparsers = bench_parser.add_subparsers() bench_compare_parser = bench_subparsers.add_parser( "compare", help="Compare benchmark runs." ) bench_compare_parser.add_argument( "--old", action="store", required=True, help="Benchmark reference results to " "compare agains.", ) bench_compare_parser.add_argument( "--new", action="store", required=True, help="New benchmark results to check." ) bench_compare_parser.add_argument( "-o", "--output", action="store", required=True, help="Output file for plots." ) bench_compare_parser.set_defaults(func=benchmark_compare) args = parser.parse_args() args.func(parser, args) if __name__ == "__main__": main()