|
207
|
1 #!/usr/bin/env python
|
|
|
2
|
|
|
3 import argparse
|
|
|
4 import sys
|
|
|
5 import os
|
|
|
6
|
|
|
7 from subprocess import call
|
|
|
8
|
|
|
9 SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__))
|
|
|
10 PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects")
|
|
|
11 DEFAULT_LLVM_DIR = os.path.realpath(os.path.join(SCRIPTS_DIR,
|
|
|
12 os.path.pardir,
|
|
|
13 os.path.pardir,
|
|
|
14 os.path.pardir))
|
|
|
15
|
|
|
16
|
|
|
17 def add(parser, args):
|
|
|
18 import SATestAdd
|
|
|
19 from ProjectMap import ProjectInfo
|
|
|
20
|
|
|
21 if args.source == "git" and (args.origin == "" or args.commit == ""):
|
|
|
22 parser.error(
|
|
|
23 "Please provide both --origin and --commit if source is 'git'")
|
|
|
24
|
|
|
25 if args.source != "git" and (args.origin != "" or args.commit != ""):
|
|
|
26 parser.error("Options --origin and --commit don't make sense when "
|
|
|
27 "source is not 'git'")
|
|
|
28
|
|
|
29 project = ProjectInfo(args.name[0], args.mode, args.source, args.origin,
|
|
|
30 args.commit)
|
|
|
31
|
|
|
32 SATestAdd.add_new_project(project)
|
|
|
33
|
|
|
34
|
|
|
35 def build(parser, args):
|
|
|
36 import SATestBuild
|
|
|
37
|
|
|
38 SATestBuild.VERBOSE = args.verbose
|
|
|
39
|
|
|
40 projects = get_projects(parser, args)
|
|
|
41 tester = SATestBuild.RegressionTester(args.jobs,
|
|
|
42 projects,
|
|
|
43 args.override_compiler,
|
|
|
44 args.extra_analyzer_config,
|
|
|
45 args.regenerate,
|
|
|
46 args.strictness)
|
|
|
47 tests_passed = tester.test_all()
|
|
|
48
|
|
|
49 if not tests_passed:
|
|
|
50 sys.stderr.write("ERROR: Tests failed.\n")
|
|
|
51 sys.exit(42)
|
|
|
52
|
|
|
53
|
|
|
54 def compare(parser, args):
|
|
|
55 import CmpRuns
|
|
|
56
|
|
|
57 choices = [CmpRuns.HistogramType.RELATIVE.value,
|
|
|
58 CmpRuns.HistogramType.LOG_RELATIVE.value,
|
|
|
59 CmpRuns.HistogramType.ABSOLUTE.value]
|
|
|
60
|
|
|
61 if args.histogram is not None and args.histogram not in choices:
|
|
|
62 parser.error("Incorrect histogram type, available choices are {}"
|
|
|
63 .format(choices))
|
|
|
64
|
|
|
65 dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
|
|
|
66 dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)
|
|
|
67
|
|
|
68 CmpRuns.dump_scan_build_results_diff(dir_old, dir_new,
|
|
|
69 show_stats=args.show_stats,
|
|
|
70 stats_only=args.stats_only,
|
|
|
71 histogram=args.histogram,
|
|
|
72 verbose_log=args.verbose_log)
|
|
|
73
|
|
|
74
|
|
|
75 def update(parser, args):
|
|
|
76 import SATestUpdateDiffs
|
|
|
77 from ProjectMap import ProjectMap
|
|
|
78
|
|
|
79 project_map = ProjectMap()
|
|
|
80 for project in project_map.projects:
|
|
|
81 SATestUpdateDiffs.update_reference_results(project, args.git)
|
|
|
82
|
|
|
83
|
|
|
84 def benchmark(parser, args):
|
|
|
85 from SATestBenchmark import Benchmark
|
|
|
86
|
|
|
87 projects = get_projects(parser, args)
|
|
|
88 benchmark = Benchmark(projects, args.iterations, args.output)
|
|
|
89 benchmark.run()
|
|
|
90
|
|
|
91
|
|
|
92 def benchmark_compare(parser, args):
|
|
|
93 import SATestBenchmark
|
|
|
94 SATestBenchmark.compare(args.old, args.new, args.output)
|
|
|
95
|
|
|
96
|
|
|
97 def get_projects(parser, args):
|
|
|
98 from ProjectMap import ProjectMap, Size
|
|
|
99
|
|
|
100 project_map = ProjectMap()
|
|
|
101 projects = project_map.projects
|
|
|
102
|
|
|
103 def filter_projects(projects, predicate, force=False):
|
|
|
104 return [project.with_fields(enabled=(force or project.enabled) and
|
|
|
105 predicate(project))
|
|
|
106 for project in projects]
|
|
|
107
|
|
|
108 if args.projects:
|
|
|
109 projects_arg = args.projects.split(",")
|
|
|
110 available_projects = [project.name
|
|
|
111 for project in projects]
|
|
|
112
|
|
|
113 # validate that given projects are present in the project map file
|
|
|
114 for manual_project in projects_arg:
|
|
|
115 if manual_project not in available_projects:
|
|
|
116 parser.error("Project '{project}' is not found in "
|
|
|
117 "the project map file. Available projects are "
|
|
|
118 "{all}.".format(project=manual_project,
|
|
|
119 all=available_projects))
|
|
|
120
|
|
|
121 projects = filter_projects(projects, lambda project:
|
|
|
122 project.name in projects_arg,
|
|
|
123 force=True)
|
|
|
124
|
|
|
125 try:
|
|
|
126 max_size = Size.from_str(args.max_size)
|
|
|
127 except ValueError as e:
|
|
|
128 parser.error("{}".format(e))
|
|
|
129
|
|
|
130 projects = filter_projects(projects, lambda project:
|
|
|
131 project.size <= max_size)
|
|
|
132
|
|
|
133 return projects
|
|
|
134
|
|
|
135
|
|
|
136 def docker(parser, args):
|
|
|
137 if len(args.rest) > 0:
|
|
|
138 if args.rest[0] != "--":
|
|
|
139 parser.error("REST arguments should start with '--'")
|
|
|
140 args.rest = args.rest[1:]
|
|
|
141
|
|
|
142 if args.build_image:
|
|
|
143 docker_build_image()
|
|
|
144 elif args.shell:
|
|
|
145 docker_shell(args)
|
|
|
146 else:
|
|
|
147 sys.exit(docker_run(args, ' '.join(args.rest)))
|
|
|
148
|
|
|
149
|
|
|
150 def docker_build_image():
|
|
|
151 sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR),
|
|
|
152 shell=True))
|
|
|
153
|
|
|
154
|
|
|
155 def docker_shell(args):
|
|
|
156 try:
|
|
|
157 # First we need to start the docker container in a waiting mode,
|
|
|
158 # so it doesn't do anything, but most importantly keeps working
|
|
|
159 # while the shell session is in progress.
|
|
|
160 docker_run(args, "--wait", "--detach")
|
|
|
161 # Since the docker container is running, we can actually connect to it
|
|
|
162 call("docker exec -it satest bash", shell=True)
|
|
|
163
|
|
|
164 except KeyboardInterrupt:
|
|
|
165 pass
|
|
|
166
|
|
|
167 finally:
|
|
|
168 docker_cleanup()
|
|
|
169
|
|
|
170
|
|
|
171 def docker_run(args, command, docker_args=""):
|
|
|
172 try:
|
|
|
173 return call("docker run --rm --name satest "
|
|
|
174 "-v {llvm}:/llvm-project "
|
|
|
175 "-v {build}:/build "
|
|
|
176 "-v {clang}:/analyzer "
|
|
|
177 "-v {scripts}:/scripts "
|
|
|
178 "-v {projects}:/projects "
|
|
|
179 "{docker_args} "
|
|
|
180 "satest-image:latest {command}"
|
|
|
181 .format(llvm=args.llvm_project_dir,
|
|
|
182 build=args.build_dir,
|
|
|
183 clang=args.clang_dir,
|
|
|
184 scripts=SCRIPTS_DIR,
|
|
|
185 projects=PROJECTS_DIR,
|
|
|
186 docker_args=docker_args,
|
|
|
187 command=command),
|
|
|
188 shell=True)
|
|
|
189
|
|
|
190 except KeyboardInterrupt:
|
|
|
191 docker_cleanup()
|
|
|
192
|
|
|
193
|
|
|
194 def docker_cleanup():
|
|
|
195 print("Please wait for docker to clean up")
|
|
|
196 call("docker stop satest", shell=True)
|
|
|
197
|
|
|
198
|
|
|
199 def main():
|
|
|
200 parser = argparse.ArgumentParser()
|
|
|
201 subparsers = parser.add_subparsers()
|
|
|
202
|
|
|
203 # add subcommand
|
|
|
204 add_parser = subparsers.add_parser(
|
|
|
205 "add",
|
|
|
206 help="Add a new project for the analyzer testing.")
|
|
|
207 # TODO: Add an option not to build.
|
|
|
208 # TODO: Set the path to the Repository directory.
|
|
|
209 add_parser.add_argument("name", nargs=1, help="Name of the new project")
|
|
|
210 add_parser.add_argument("--mode", action="store", default=1, type=int,
|
|
|
211 choices=[0, 1, 2],
|
|
|
212 help="Build mode: 0 for single file project, "
|
|
|
213 "1 for scan_build, "
|
|
|
214 "2 for single file c++11 project")
|
|
|
215 add_parser.add_argument("--source", action="store", default="script",
|
|
|
216 choices=["script", "git", "zip"],
|
|
|
217 help="Source type of the new project: "
|
|
|
218 "'git' for getting from git "
|
|
|
219 "(please provide --origin and --commit), "
|
|
|
220 "'zip' for unpacking source from a zip file, "
|
|
|
221 "'script' for downloading source by running "
|
|
|
222 "a custom script")
|
|
|
223 add_parser.add_argument("--origin", action="store", default="",
|
|
|
224 help="Origin link for a git repository")
|
|
|
225 add_parser.add_argument("--commit", action="store", default="",
|
|
|
226 help="Git hash for a commit to checkout")
|
|
|
227 add_parser.set_defaults(func=add)
|
|
|
228
|
|
|
229 # build subcommand
|
|
|
230 build_parser = subparsers.add_parser(
|
|
|
231 "build",
|
|
|
232 help="Build projects from the project map and compare results with "
|
|
|
233 "the reference.")
|
|
|
234 build_parser.add_argument("--strictness", dest="strictness",
|
|
|
235 type=int, default=0,
|
|
|
236 help="0 to fail on runtime errors, 1 to fail "
|
|
|
237 "when the number of found bugs are different "
|
|
|
238 "from the reference, 2 to fail on any "
|
|
|
239 "difference from the reference. Default is 0.")
|
|
|
240 build_parser.add_argument("-r", dest="regenerate", action="store_true",
|
|
|
241 default=False,
|
|
|
242 help="Regenerate reference output.")
|
|
|
243 build_parser.add_argument("--override-compiler", action="store_true",
|
|
|
244 default=False, help="Call scan-build with "
|
|
|
245 "--override-compiler option.")
|
|
|
246 build_parser.add_argument("-j", "--jobs", dest="jobs",
|
|
|
247 type=int, default=0,
|
|
|
248 help="Number of projects to test concurrently")
|
|
|
249 build_parser.add_argument("--extra-analyzer-config",
|
|
|
250 dest="extra_analyzer_config", type=str,
|
|
|
251 default="",
|
|
|
252 help="Arguments passed to to -analyzer-config")
|
|
|
253 build_parser.add_argument("--projects", action="store", default="",
|
|
|
254 help="Comma-separated list of projects to test")
|
|
|
255 build_parser.add_argument("--max-size", action="store", default=None,
|
|
|
256 help="Maximum size for the projects to test")
|
|
|
257 build_parser.add_argument("-v", "--verbose", action="count", default=0)
|
|
|
258 build_parser.set_defaults(func=build)
|
|
|
259
|
|
|
260 # compare subcommand
|
|
|
261 cmp_parser = subparsers.add_parser(
|
|
|
262 "compare",
|
|
|
263 help="Comparing two static analyzer runs in terms of "
|
|
|
264 "reported warnings and execution time statistics.")
|
|
|
265 cmp_parser.add_argument("--root-old", dest="root_old",
|
|
|
266 help="Prefix to ignore on source files for "
|
|
|
267 "OLD directory",
|
|
|
268 action="store", type=str, default="")
|
|
|
269 cmp_parser.add_argument("--root-new", dest="root_new",
|
|
|
270 help="Prefix to ignore on source files for "
|
|
|
271 "NEW directory",
|
|
|
272 action="store", type=str, default="")
|
|
|
273 cmp_parser.add_argument("--verbose-log", dest="verbose_log",
|
|
|
274 help="Write additional information to LOG "
|
|
|
275 "[default=None]",
|
|
|
276 action="store", type=str, default=None,
|
|
|
277 metavar="LOG")
|
|
|
278 cmp_parser.add_argument("--stats-only", action="store_true",
|
|
|
279 dest="stats_only", default=False,
|
|
|
280 help="Only show statistics on reports")
|
|
|
281 cmp_parser.add_argument("--show-stats", action="store_true",
|
|
|
282 dest="show_stats", default=False,
|
|
|
283 help="Show change in statistics")
|
|
|
284 cmp_parser.add_argument("--histogram", action="store", default=None,
|
|
|
285 help="Show histogram of paths differences. "
|
|
|
286 "Requires matplotlib")
|
|
|
287 cmp_parser.add_argument("old", nargs=1, help="Directory with old results")
|
|
|
288 cmp_parser.add_argument("new", nargs=1, help="Directory with new results")
|
|
|
289 cmp_parser.set_defaults(func=compare)
|
|
|
290
|
|
|
291 # update subcommand
|
|
|
292 upd_parser = subparsers.add_parser(
|
|
|
293 "update",
|
|
|
294 help="Update static analyzer reference results based on the previous "
|
|
|
295 "run of SATest build. Assumes that SATest build was just run.")
|
|
|
296 upd_parser.add_argument("--git", action="store_true",
|
|
|
297 help="Stage updated results using git.")
|
|
|
298 upd_parser.set_defaults(func=update)
|
|
|
299
|
|
|
300 # docker subcommand
|
|
|
301 dock_parser = subparsers.add_parser(
|
|
|
302 "docker",
|
|
|
303 help="Run regression system in the docker.")
|
|
|
304
|
|
|
305 dock_parser.add_argument("--build-image", action="store_true",
|
|
|
306 help="Build docker image for running tests.")
|
|
|
307 dock_parser.add_argument("--shell", action="store_true",
|
|
|
308 help="Start a shell on docker.")
|
|
|
309 dock_parser.add_argument("--llvm-project-dir", action="store",
|
|
|
310 default=DEFAULT_LLVM_DIR,
|
|
|
311 help="Path to LLVM source code. Defaults "
|
|
|
312 "to the repo where this script is located. ")
|
|
|
313 dock_parser.add_argument("--build-dir", action="store", default="",
|
|
|
314 help="Path to a directory where docker should "
|
|
|
315 "build LLVM code.")
|
|
|
316 dock_parser.add_argument("--clang-dir", action="store", default="",
|
|
|
317 help="Path to find/install LLVM installation.")
|
|
|
318 dock_parser.add_argument("rest", nargs=argparse.REMAINDER, default=[],
|
|
|
319 help="Additionall args that will be forwarded "
|
|
|
320 "to the docker's entrypoint.")
|
|
|
321 dock_parser.set_defaults(func=docker)
|
|
|
322
|
|
|
323 # benchmark subcommand
|
|
|
324 bench_parser = subparsers.add_parser(
|
|
|
325 "benchmark",
|
|
|
326 help="Run benchmarks by building a set of projects multiple times.")
|
|
|
327
|
|
|
328 bench_parser.add_argument("-i", "--iterations", action="store",
|
|
|
329 type=int, default=20,
|
|
|
330 help="Number of iterations for building each "
|
|
|
331 "project.")
|
|
|
332 bench_parser.add_argument("-o", "--output", action="store",
|
|
|
333 default="benchmark.csv",
|
|
|
334 help="Output csv file for the benchmark results")
|
|
|
335 bench_parser.add_argument("--projects", action="store", default="",
|
|
|
336 help="Comma-separated list of projects to test")
|
|
|
337 bench_parser.add_argument("--max-size", action="store", default=None,
|
|
|
338 help="Maximum size for the projects to test")
|
|
|
339 bench_parser.set_defaults(func=benchmark)
|
|
|
340
|
|
|
341 bench_subparsers = bench_parser.add_subparsers()
|
|
|
342 bench_compare_parser = bench_subparsers.add_parser(
|
|
|
343 "compare",
|
|
|
344 help="Compare benchmark runs.")
|
|
|
345 bench_compare_parser.add_argument("--old", action="store", required=True,
|
|
|
346 help="Benchmark reference results to "
|
|
|
347 "compare agains.")
|
|
|
348 bench_compare_parser.add_argument("--new", action="store", required=True,
|
|
|
349 help="New benchmark results to check.")
|
|
|
350 bench_compare_parser.add_argument("-o", "--output",
|
|
|
351 action="store", required=True,
|
|
|
352 help="Output file for plots.")
|
|
|
353 bench_compare_parser.set_defaults(func=benchmark_compare)
|
|
|
354
|
|
|
355 args = parser.parse_args()
|
|
|
356 args.func(parser, args)
|
|
|
357
|
|
|
358
|
|
|
359 if __name__ == "__main__":
|
|
|
360 main()
|
