From 339949be25863ac15e24659c2ab4b01185e1234a Mon Sep 17 00:00:00 2001 From: Stephen Smalley Date: Thu, 6 Aug 2020 14:34:18 -0400 Subject: scripts/selinux,selinux: update mdp to enable policy capabilities Presently mdp does not enable any SELinux policy capabilities in the dummy policy it generates. Thus, policies derived from it will by default lack various features commonly used in modern policies such as open permission, extended socket classes, network peer controls, etc. Split the policy capability definitions out into their own headers so that we can include them into mdp without pulling in other kernel headers and extend mdp generate policycap statements for the policy capabilities known to the kernel. Policy authors may wish to selectively remove some of these from the generated policy. Signed-off-by: Stephen Smalley Signed-off-by: Paul Moore --- scripts/selinux/mdp/mdp.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'scripts') diff --git a/scripts/selinux/mdp/mdp.c b/scripts/selinux/mdp/mdp.c index 6ceb88eb9b59..105c1c31a316 100644 --- a/scripts/selinux/mdp/mdp.c +++ b/scripts/selinux/mdp/mdp.c @@ -35,6 +35,9 @@ struct security_class_mapping { #include "classmap.h" #include "initial_sid_to_string.h" +#include "policycap_names.h" + +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) int main(int argc, char *argv[]) { @@ -115,6 +118,10 @@ int main(int argc, char *argv[]) } } + /* enable all policy capabilities */ + for (i = 0; i < ARRAY_SIZE(selinux_policycap_names); i++) + fprintf(fout, "policycap %s;\n", selinux_policycap_names[i]); + /* types, roles, and allows */ fprintf(fout, "type base_t;\n"); fprintf(fout, "role base_r;\n"); -- cgit v1.2.3 From 4036707c7c61a0fec7774aec44bbb7350153e8c2 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 19 Aug 2020 14:47:09 +0200 Subject: scripts/dtc: dtx_diff - make help text formatting consistent None of the help texts use capitalization, except the one for the -T option. Drop the capitalization for consistency. Split the single long line that doesn't fit in 80 characters. Reviewed-by: Frank Rowand Signed-off-by: Geert Uytterhoeven Signed-off-by: Rob Herring Link: https://lore.kernel.org/r/20200819124709.20401-1-geert+renesas@glider.be --- scripts/dtc/dtx_diff | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/dtc/dtx_diff b/scripts/dtc/dtx_diff index 541c432e7d19..d3422ee15e30 100755 --- a/scripts/dtc/dtx_diff +++ b/scripts/dtc/dtx_diff @@ -29,7 +29,8 @@ Usage: -s SRCTREE linux kernel source tree is at path SRCTREE (default is current directory) -S linux kernel source tree is at root of current git repo - -T Annotate output .dts with input source file and line (-T -T for more details) + -T annotate output .dts with input source file and line + (-T -T for more details) -u unsorted, do not sort DTx -- cgit v1.2.3 From b8a49399fb7abd4ec402bea1fec5a974053591b6 Mon Sep 17 00:00:00 2001 From: Andrei Ziureaev Date: Thu, 13 Aug 2020 14:26:11 +0100 Subject: dt-bindings: Use json for processed-schema* Change the format of processed-schema* from yaml to json to speed up validation. With json output, using xargs and appending the output won't work since json has explicit list begin and end characters. Instead, we pass the schema files as a list in a temp file. The parsing time for the processed schema goes down from ~2sec to 70ms. Also, 'make dtbs_check' becomes 33% faster. Some error messages are affected by this change. For example, "True was expected" becomes "... is not of type 'boolean'". The order of messages is also changed. Signed-off-by: Andrei Ziureaev Signed-off-by: Rob Herring --- scripts/Makefile.lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 3d599716940c..94133708889d 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -328,7 +328,7 @@ $(obj)/%.dtb: $(src)/%.dts $(DTC) FORCE DT_CHECKER ?= dt-validate DT_BINDING_DIR := Documentation/devicetree/bindings # DT_TMP_SCHEMA may be overridden from Documentation/devicetree/bindings/Makefile -DT_TMP_SCHEMA ?= $(objtree)/$(DT_BINDING_DIR)/processed-schema.yaml +DT_TMP_SCHEMA ?= $(objtree)/$(DT_BINDING_DIR)/processed-schema.json quiet_cmd_dtb_check = CHECK $@ cmd_dtb_check = $(DT_CHECKER) -u $(srctree)/$(DT_BINDING_DIR) -p $(DT_TMP_SCHEMA) $@ -- cgit v1.2.3 From 4b2bd20c350a66d4f8a7c2da48b115fd4e06d15e Mon Sep 17 00:00:00 2001 From: Sumera Priyadarsini Date: Wed, 5 Aug 2020 14:27:40 +0530 Subject: scripts: coccicheck: Add chain mode to list of modes This patch adds chain mode to the list of available modes in coccicheck. Signed-off-by: Sumera Priyadarsini Signed-off-by: Julia Lawall --- scripts/coccicheck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/coccicheck b/scripts/coccicheck index e04d328210ac..6e37cf36caae 100755 --- a/scripts/coccicheck +++ b/scripts/coccicheck @@ -99,7 +99,7 @@ fi if [ "$MODE" = "" ] ; then if [ "$ONLINE" = "0" ] ; then echo 'You have not explicitly specified the mode to use. Using default "report" mode.' - echo 'Available modes are the following: patch, report, context, org' + echo 'Available modes are the following: patch, report, context, org, chain' echo 'You can specify the mode with "make coccicheck MODE="' echo 'Note however that some modes are not implemented by some semantic patches.' fi -- cgit v1.2.3 From 7a2624e6de03050c9f15af21c216818d5508b5e9 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Fri, 21 Aug 2020 00:55:57 +0100 Subject: coccinelle: add patch rule for dma_alloc_coherent Commit dfd32cad146e ("dma-mapping: remove dma_zalloc_coherent()") removed the definition of dma_zalloc_coherent() and also removed the corresponding patch rule for replacing instances of dma_alloc_coherent + memset in zalloc-simple.cocci (though left the report rule). Add a new patch rule to remove unnecessary calls to memset after allocating with dma_alloc_coherent. While we're at it, fix a couple of typos. Fixes: dfd32cad146e ("dma-mapping: remove dma_zalloc_coherent()") Signed-off-by: Alex Dewar Signed-off-by: Julia Lawall --- scripts/coccinelle/api/alloc/zalloc-simple.cocci | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/coccinelle/api/alloc/zalloc-simple.cocci b/scripts/coccinelle/api/alloc/zalloc-simple.cocci index 26cda3f48f01..b3d0c3c230c1 100644 --- a/scripts/coccinelle/api/alloc/zalloc-simple.cocci +++ b/scripts/coccinelle/api/alloc/zalloc-simple.cocci @@ -127,6 +127,16 @@ statement S; if ((x==NULL) || ...) S - memset((T2)x,0,E1); +@depends on patch@ +type T, T2; +expression x; +expression E1,E2,E3,E4; +statement S; +@@ + x = (T)dma_alloc_coherent(E1, E2, E3, E4); + if ((x==NULL) || ...) S +- memset((T2)x, 0, E2); + //---------------------------------------------------------- // For org mode //---------------------------------------------------------- @@ -199,9 +209,9 @@ statement S; position p; @@ - x = (T)dma_alloc_coherent@p(E2,E1,E3,E4); + x = (T)dma_alloc_coherent@p(E1,E2,E3,E4); if ((x==NULL) || ...) S - memset((T2)x,0,E1); + memset((T2)x,0,E2); @script:python depends on org@ p << r2.p; @@ -217,7 +227,7 @@ p << r2.p; x << r2.x; @@ -msg="WARNING: dma_alloc_coherent use in %s already zeroes out memory, so memset is not needed" % (x) +msg="WARNING: dma_alloc_coherent used in %s already zeroes out memory, so memset is not needed" % (x) coccilib.report.print_report(p[0], msg) //----------------------------------------------------------------- -- cgit v1.2.3 From a2fc3718bc22e85378085568ecc5765fb28cabce Mon Sep 17 00:00:00 2001 From: Denis Efremov Date: Fri, 21 Aug 2020 23:11:37 +0300 Subject: coccinelle: api: add kobj_to_dev.cocci script Use kobj_to_dev() instead of container_of(). Signed-off-by: Denis Efremov Signed-off-by: Julia Lawall --- scripts/coccinelle/api/kobj_to_dev.cocci | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 scripts/coccinelle/api/kobj_to_dev.cocci (limited to 'scripts') diff --git a/scripts/coccinelle/api/kobj_to_dev.cocci b/scripts/coccinelle/api/kobj_to_dev.cocci new file mode 100644 index 000000000000..cd5d31c6fe76 --- /dev/null +++ b/scripts/coccinelle/api/kobj_to_dev.cocci @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0-only +/// +/// Use kobj_to_dev() instead of container_of() +/// +// Confidence: High +// Copyright: (C) 2020 Denis Efremov ISPRAS +// Options: --no-includes --include-headers +// +// Keywords: kobj_to_dev, container_of +// + +virtual context +virtual report +virtual org +virtual patch + + +@r depends on !patch@ +expression ptr; +symbol kobj; +position p; +@@ + +* container_of(ptr, struct device, kobj)@p + + +@depends on patch@ +expression ptr; +@@ + +- container_of(ptr, struct device, kobj) ++ kobj_to_dev(ptr) + + +@script:python depends on report@ +p << r.p; +@@ + +coccilib.report.print_report(p[0], "WARNING opportunity for kobj_to_dev()") + +@script:python depends on org@ +p << r.p; +@@ + +coccilib.org.print_todo(p[0], "WARNING opportunity for kobj_to_dev()") -- cgit v1.2.3 From 14e2ac8de0f91f12122a49f09897b0cd05256460 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 24 Jul 2020 09:00:01 +0200 Subject: kcsan: Support compounded read-write instrumentation Add support for compounded read-write instrumentation if supported by the compiler. Adds the necessary instrumentation functions, and a new type which is used to generate a more descriptive report. Furthermore, such compounded memory access instrumentation is excluded from the "assume aligned writes up to word size are atomic" rule, because we cannot assume that the compiler emits code that is atomic for compound ops. LLVM/Clang added support for the feature in: https://github.com/llvm/llvm-project/commit/785d41a261d136b64ab6c15c5d35f2adc5ad53e3 The new instrumentation is emitted for sets of memory accesses in the same basic block to the same address with at least one read appearing before a write. These typically result from compound operations such as ++, --, +=, -=, |=, &=, etc. but also equivalent forms such as "var = var + 1". Where the compiler determines that it is equivalent to emit a call to a single __tsan_read_write instead of separate __tsan_read and __tsan_write, we can then benefit from improved performance and better reporting for such access patterns. The new reports now show that the ops are both reads and writes, for example: read-write to 0xffffffff90548a38 of 8 bytes by task 143 on cpu 3: test_kernel_rmw_array+0x45/0xa0 access_thread+0x71/0xb0 kthread+0x21e/0x240 ret_from_fork+0x22/0x30 read-write to 0xffffffff90548a38 of 8 bytes by task 144 on cpu 2: test_kernel_rmw_array+0x45/0xa0 access_thread+0x71/0xb0 kthread+0x21e/0x240 ret_from_fork+0x22/0x30 Acked-by: Peter Zijlstra (Intel) Signed-off-by: Marco Elver Signed-off-by: Paul E. McKenney --- scripts/Makefile.kcsan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/Makefile.kcsan b/scripts/Makefile.kcsan index c50f27b3ac56..c37f9518d5d9 100644 --- a/scripts/Makefile.kcsan +++ b/scripts/Makefile.kcsan @@ -11,5 +11,5 @@ endif # of some options does not break KCSAN nor causes false positive reports. CFLAGS_KCSAN := -fsanitize=thread \ $(call cc-option,$(call cc-param,tsan-instrument-func-entry-exit=0) -fno-optimize-sibling-calls) \ - $(call cc-option,$(call cc-param,tsan-instrument-read-before-write=1)) \ + $(call cc-option,$(call cc-param,tsan-compound-read-before-write=1),$(call cc-option,$(call cc-param,tsan-instrument-read-before-write=1))) \ $(call cc-param,tsan-distinguish-volatile=1) -- cgit v1.2.3 From 3570a1bcf45e9a7ddf9ba0e8d6d57cc67675cfef Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 24 Jul 2020 09:00:08 +0200 Subject: locking/atomics: Use read-write instrumentation for atomic RMWs Use instrument_atomic_read_write() for atomic RMW ops. Cc: Will Deacon Cc: Boqun Feng Cc: Arnd Bergmann Cc: Acked-by: Peter Zijlstra (Intel) Signed-off-by: Marco Elver Signed-off-by: Paul E. McKenney --- scripts/atomic/gen-atomic-instrumented.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/atomic/gen-atomic-instrumented.sh b/scripts/atomic/gen-atomic-instrumented.sh index 6afadf73da17..2b7fec7e6abc 100755 --- a/scripts/atomic/gen-atomic-instrumented.sh +++ b/scripts/atomic/gen-atomic-instrumented.sh @@ -5,9 +5,10 @@ ATOMICDIR=$(dirname $0) . ${ATOMICDIR}/atomic-tbl.sh -#gen_param_check(arg) +#gen_param_check(meta, arg) gen_param_check() { + local meta="$1"; shift local arg="$1"; shift local type="${arg%%:*}" local name="$(gen_param_name "${arg}")" @@ -17,17 +18,25 @@ gen_param_check() i) return;; esac - # We don't write to constant parameters - [ ${type#c} != ${type} ] && rw="read" + if [ ${type#c} != ${type} ]; then + # We don't write to constant parameters. + rw="read" + elif [ "${meta}" != "s" ]; then + # An atomic RMW: if this parameter is not a constant, and this atomic is + # not just a 's'tore, this parameter is both read from and written to. + rw="read_write" + fi printf "\tinstrument_atomic_${rw}(${name}, sizeof(*${name}));\n" } -#gen_param_check(arg...) +#gen_params_checks(meta, arg...) gen_params_checks() { + local meta="$1"; shift + while [ "$#" -gt 0 ]; do - gen_param_check "$1" + gen_param_check "$meta" "$1" shift; done } @@ -77,7 +86,7 @@ gen_proto_order_variant() local ret="$(gen_ret_type "${meta}" "${int}")" local params="$(gen_params "${int}" "${atomic}" "$@")" - local checks="$(gen_params_checks "$@")" + local checks="$(gen_params_checks "${meta}" "$@")" local args="$(gen_args "$@")" local retstmt="$(gen_ret_stmt "${meta}")" -- cgit v1.2.3 From 6e22ab9da79343532cd3cde39df25e5a5478c692 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Tue, 25 Aug 2020 21:21:20 +0200 Subject: bpf: Add d_path helper Adding d_path helper function that returns full path for given 'struct path' object, which needs to be the kernel BTF 'path' object. The path is returned in buffer provided 'buf' of size 'sz' and is zero terminated. bpf_d_path(&file->f_path, buf, size); The helper calls directly d_path function, so there's only limited set of function it can be called from. Adding just very modest set for the start. Updating also bpf.h tools uapi header and adding 'path' to bpf_helpers_doc.py script. Signed-off-by: Jiri Olsa Signed-off-by: Alexei Starovoitov Acked-by: Andrii Nakryiko Acked-by: KP Singh Link: https://lore.kernel.org/bpf/20200825192124.710397-11-jolsa@kernel.org --- scripts/bpf_helpers_doc.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/bpf_helpers_doc.py b/scripts/bpf_helpers_doc.py index 5bfa448b4704..08388173973f 100755 --- a/scripts/bpf_helpers_doc.py +++ b/scripts/bpf_helpers_doc.py @@ -432,6 +432,7 @@ class PrinterHelpers(Printer): 'struct __sk_buff', 'struct sk_msg_md', 'struct xdp_md', + 'struct path', ] known_types = { '...', @@ -472,6 +473,7 @@ class PrinterHelpers(Printer): 'struct tcp_request_sock', 'struct udp6_sock', 'struct task_struct', + 'struct path', } mapped_types = { 'u8': '__u8', -- cgit v1.2.3 From 6eb6d05958f3208b3dd2b5311f1a6e68abdbe5d5 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 29 Jul 2020 20:12:32 +0200 Subject: seqlock,tags: Add support for SEQCOUNT_LOCKTYPE() Such that we might easily find seqcount_LOCKTYPE_t and seqcount_LOCKTYPE_init(). Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200729161938.GB2678@hirez.programming.kicks-ass.net --- scripts/tags.sh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/tags.sh b/scripts/tags.sh index 32d3f53af10b..fc5b53259a16 100755 --- a/scripts/tags.sh +++ b/scripts/tags.sh @@ -201,6 +201,8 @@ regex_c=( '/\ Date: Fri, 21 Aug 2020 11:43:58 +0900 Subject: kbuild: hide commands to run Kconfig, and show short log for syncconfig Some targets (localyesconfig, localmodconfig, defconfig) hide the command running, but the others do not. Users know which Kconfig flavor they are running, so it is OK to hide the command. Add $(Q) to all commands consistently. If you want to see the full command running, pass V=1 from the command line. syncconfig is the exceptional case, which occurs without explicit command invocation by the user. Display the Kbuild-style log for it. The ugly bare log will go away. [Before] scripts/kconfig/conf --syncconfig Kconfig [After] SYNC include/config/auto.conf Signed-off-by: Masahiro Yamada --- scripts/kconfig/Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/kconfig/Makefile b/scripts/kconfig/Makefile index 52b59bf9efe4..e46df0a2d4f9 100644 --- a/scripts/kconfig/Makefile +++ b/scripts/kconfig/Makefile @@ -20,19 +20,19 @@ endif unexport CONFIG_ xconfig: $(obj)/qconf - $< $(silent) $(Kconfig) + $(Q)$< $(silent) $(Kconfig) gconfig: $(obj)/gconf - $< $(silent) $(Kconfig) + $(Q)$< $(silent) $(Kconfig) menuconfig: $(obj)/mconf - $< $(silent) $(Kconfig) + $(Q)$< $(silent) $(Kconfig) config: $(obj)/conf - $< $(silent) --oldaskconfig $(Kconfig) + $(Q)$< $(silent) --oldaskconfig $(Kconfig) nconfig: $(obj)/nconf - $< $(silent) $(Kconfig) + $(Q)$< $(silent) $(Kconfig) build_menuconfig: $(obj)/mconf @@ -68,12 +68,12 @@ simple-targets := oldconfig allnoconfig allyesconfig allmodconfig \ PHONY += $(simple-targets) $(simple-targets): $(obj)/conf - $< $(silent) --$@ $(Kconfig) + $(Q)$< $(silent) --$@ $(Kconfig) PHONY += savedefconfig defconfig savedefconfig: $(obj)/conf - $< $(silent) --$@=defconfig $(Kconfig) + $(Q)$< $(silent) --$@=defconfig $(Kconfig) defconfig: $(obj)/conf ifneq ($(wildcard $(srctree)/arch/$(SRCARCH)/configs/$(KBUILD_DEFCONFIG)),) @@ -111,7 +111,7 @@ tinyconfig: # CHECK: -o cache_dir= working? PHONY += testconfig testconfig: $(obj)/conf - $(PYTHON3) -B -m pytest $(srctree)/$(src)/tests \ + $(Q)$(PYTHON3) -B -m pytest $(srctree)/$(src)/tests \ -o cache_dir=$(abspath $(obj)/tests/.cache) \ $(if $(findstring 1,$(KBUILD_VERBOSE)),--capture=no) clean-files += tests/.cache -- cgit v1.2.3 From 8a685db32f2b3495c91efab6e9462c2fd934778f Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:09 +0900 Subject: gen_compile_commands: parse only the first line of .*.cmd files After the allmodconfig build, this script takes about 5 sec on my machine. Most of the run-time is consumed for needless regex matching. We know the format of .*.cmd file; the first line is the build command. There is no need to parse the rest. With this optimization, now it runs 4 times faster. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers Tested-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index c458696ef3a7..1bcf33a93cb9 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -125,11 +125,8 @@ def main(): filepath = os.path.join(dirpath, filename) with open(filepath, 'rt') as f: - for line in f: - result = line_matcher.match(line) - if not result: - continue - + result = line_matcher.match(f.readline()) + if result: try: entry = process_line(directory, dirpath, result.group(1), result.group(2)) -- cgit v1.2.3 From ea6cedc5b8a4e666da41f1173ea19970fa60683f Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:10 +0900 Subject: gen_compile_commands: use choices for --log_levels option Use 'choices' to check if the given parameter is valid. I also simplified the help message because, with 'choices', --help shows the list of valid parameters: --log_level {DEBUG,INFO,WARNING,ERROR,CRITICAL} I started the help message with a lower case, "the level of log ..." in order to be consistent with the -h option: -h, --help show this help message and exit The message "show this help ..." comes from the ArgumentParser library code, and I do not know how to change it. So, I changed our code. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 1bcf33a93cb9..535248cf2d7e 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -45,24 +45,18 @@ def parse_arguments(): 'compile_commands.json in the search directory)') parser.add_argument('-o', '--output', type=str, help=output_help) - log_level_help = ('The level of log messages to produce (one of ' + - ', '.join(_VALID_LOG_LEVELS) + '; defaults to ' + + log_level_help = ('the level of log messages to produce (defaults to ' + _DEFAULT_LOG_LEVEL + ')') - parser.add_argument( - '--log_level', type=str, default=_DEFAULT_LOG_LEVEL, - help=log_level_help) + parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS, + default=_DEFAULT_LOG_LEVEL, help=log_level_help) args = parser.parse_args() - log_level = args.log_level - if log_level not in _VALID_LOG_LEVELS: - raise ValueError('%s is not a valid log level' % log_level) - directory = args.directory or os.getcwd() output = args.output or os.path.join(directory, _DEFAULT_OUTPUT) directory = os.path.abspath(directory) - return log_level, directory, output + return args.log_level, directory, output def process_line(root_directory, file_directory, command_prefix, relative_path): -- cgit v1.2.3 From 6ca4c6d25949117dc5b4845612e290b6d89e70a8 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:11 +0900 Subject: gen_compile_commands: do not support .cmd files under tools/ directory The tools/ directory uses a different build system, and the format of .cmd files is different because the tools builds run in a different work directory. Supporting two formats compilicates the script. The only loss by this change is objtool. Also, rename the confusing variable 'relative_path' because it is not necessarily a relative path. When the output directory is not the direct child of the source tree (e.g. O=foo/bar), it is an absolute path. Rename it to 'file_path'. os.path.join(root_directory, file_path) works whether the file_path is relative or not. If file_path is already absolute, it returns it as-is. I used os.path.abspath() to normalize file paths. If you run this script against the kernel built with O=foo option, the file_path contains '../' patterns. os.path.abspath() fixes up 'foo/bar/../baz' into 'foo/baz', and produces a cleaner commands_database.json. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 535248cf2d7e..49fff0b0b385 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -59,23 +59,21 @@ def parse_arguments(): return args.log_level, directory, output -def process_line(root_directory, file_directory, command_prefix, relative_path): +def process_line(root_directory, command_prefix, file_path): """Extracts information from a .cmd line and creates an entry from it. Args: root_directory: The directory that was searched for .cmd files. Usually used directly in the "directory" entry in compile_commands.json. - file_directory: The path to the directory the .cmd file was found in. command_prefix: The extracted command line, up to the last element. - relative_path: The .c file from the end of the extracted command. - Usually relative to root_directory, but sometimes relative to - file_directory and sometimes neither. + file_path: The .c file from the end of the extracted command. + Usually relative to root_directory, but sometimes absolute. Returns: An entry to append to compile_commands. Raises: - ValueError: Could not find the extracted file based on relative_path and + ValueError: Could not find the extracted file based on file_path and root_directory or file_directory. """ # The .cmd files are intended to be included directly by Make, so they @@ -84,20 +82,14 @@ def process_line(root_directory, file_directory, command_prefix, relative_path): # by Make, so this code replaces the escaped version with '#'. prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#') - cur_dir = root_directory - expected_path = os.path.join(cur_dir, relative_path) - if not os.path.exists(expected_path): - # Try using file_directory instead. Some of the tools have a different - # style of .cmd file than the kernel. - cur_dir = file_directory - expected_path = os.path.join(cur_dir, relative_path) - if not os.path.exists(expected_path): - raise ValueError('File %s not in %s or %s' % - (relative_path, root_directory, file_directory)) + # Use os.path.abspath() to normalize the path resolving '.' and '..' . + abs_path = os.path.abspath(os.path.join(root_directory, file_path)) + if not os.path.exists(abs_path): + raise ValueError('File %s not found' % abs_path) return { - 'directory': cur_dir, - 'file': relative_path, - 'command': prefix + relative_path, + 'directory': root_directory, + 'file': abs_path, + 'command': prefix + file_path, } @@ -122,7 +114,7 @@ def main(): result = line_matcher.match(f.readline()) if result: try: - entry = process_line(directory, dirpath, + entry = process_line(directory, result.group(1), result.group(2)) compile_commands.append(entry) except ValueError as err: -- cgit v1.2.3 From 0a7d376d04a3c84b503f1efecde8f9d9a7430269 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:12 +0900 Subject: gen_compile_commands: reword the help message of -d option I think the help message of the -d option is somewhat misleading. Path to the kernel source directory to search (defaults to the working directory) The part "kernel source directory" is the source of the confusion. Some people misunderstand as if this script did not support separate output directories. Actually, this script also works for out-of-tree builds. You can use the -d option to point to the object output directory, not to the source directory. It should match to the O= option used in the previous kernel build, and then appears in the "directory" field of compile_commands.json. Reword the help message. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 49fff0b0b385..f37c1dac8db4 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -31,13 +31,13 @@ def parse_arguments(): Returns: log_level: A logging level to filter log output. - directory: The directory to search for .cmd files. + directory: The work directory where the objects were built. output: Where to write the compile-commands JSON file. """ usage = 'Creates a compile_commands.json database from kernel .cmd files' parser = argparse.ArgumentParser(description=usage) - directory_help = ('Path to the kernel source directory to search ' + directory_help = ('specify the output directory used for the kernel build ' '(defaults to the working directory)') parser.add_argument('-d', '--directory', type=str, help=directory_help) -- cgit v1.2.3 From 6fca36f1d82ad5bc385187f0aed93bbaefaa2172 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:13 +0900 Subject: gen_compile_commands: make -o option independent of -d option Change the -o option independent of the -d option, which is I think clearer behavior. Some people may like to use -d to specify a separate output directory, but still output the compile_commands.py in the source directory (unless the source tree is read-only) because it is the default location Clang Tools search for the compilation database. Also, move the default parameter to the default= argument of the .add_argument(). Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index f37c1dac8db4..71a0630ae188 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -39,11 +39,13 @@ def parse_arguments(): directory_help = ('specify the output directory used for the kernel build ' '(defaults to the working directory)') - parser.add_argument('-d', '--directory', type=str, help=directory_help) + parser.add_argument('-d', '--directory', type=str, default='.', + help=directory_help) - output_help = ('The location to write compile_commands.json (defaults to ' - 'compile_commands.json in the search directory)') - parser.add_argument('-o', '--output', type=str, help=output_help) + output_help = ('path to the output command database (defaults to ' + + _DEFAULT_OUTPUT + ')') + parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT, + help=output_help) log_level_help = ('the level of log messages to produce (defaults to ' + _DEFAULT_LOG_LEVEL + ')') @@ -52,11 +54,9 @@ def parse_arguments(): args = parser.parse_args() - directory = args.directory or os.getcwd() - output = args.output or os.path.join(directory, _DEFAULT_OUTPUT) - directory = os.path.abspath(directory) - - return args.log_level, directory, output + return (args.log_level, + os.path.abspath(args.directory), + args.output) def process_line(root_directory, command_prefix, file_path): -- cgit v1.2.3 From fc2cb22ec61c1a537337cd0eba458863564a0e1f Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:14 +0900 Subject: gen_compile_commands: move directory walk to a generator function Currently, this script walks under the specified directory (default to the current directory), then parses all .cmd files found. Split it into a separate helper function because the next commit will add more helpers to pick up .cmd files associated with given file(s). There is no point to build and return a huge list at once. I used a generator so it works in the for-loop with less memory. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 44 ++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 12 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 71a0630ae188..e45f17be8817 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -33,6 +33,7 @@ def parse_arguments(): log_level: A logging level to filter log output. directory: The work directory where the objects were built. output: Where to write the compile-commands JSON file. + paths: The list of directories to handle to find .cmd files. """ usage = 'Creates a compile_commands.json database from kernel .cmd files' parser = argparse.ArgumentParser(description=usage) @@ -56,7 +57,28 @@ def parse_arguments(): return (args.log_level, os.path.abspath(args.directory), - args.output) + args.output, + [args.directory]) + + +def cmdfiles_in_dir(directory): + """Generate the iterator of .cmd files found under the directory. + + Walk under the given directory, and yield every .cmd file found. + + Args: + directory: The directory to search for .cmd files. + + Yields: + The path to a .cmd file. + """ + + filename_matcher = re.compile(_FILENAME_PATTERN) + + for dirpath, _, filenames in os.walk(directory): + for filename in filenames: + if filename_matcher.match(filename): + yield os.path.join(dirpath, filename) def process_line(root_directory, command_prefix, file_path): @@ -95,31 +117,29 @@ def process_line(root_directory, command_prefix, file_path): def main(): """Walks through the directory and finds and parses .cmd files.""" - log_level, directory, output = parse_arguments() + log_level, directory, output, paths = parse_arguments() level = getattr(logging, log_level) logging.basicConfig(format='%(levelname)s: %(message)s', level=level) - filename_matcher = re.compile(_FILENAME_PATTERN) line_matcher = re.compile(_LINE_PATTERN) compile_commands = [] - for dirpath, _, filenames in os.walk(directory): - for filename in filenames: - if not filename_matcher.match(filename): - continue - filepath = os.path.join(dirpath, filename) - with open(filepath, 'rt') as f: + for path in paths: + cmdfiles = cmdfiles_in_dir(path) + + for cmdfile in cmdfiles: + with open(cmdfile, 'rt') as f: result = line_matcher.match(f.readline()) if result: try: - entry = process_line(directory, - result.group(1), result.group(2)) + entry = process_line(directory, result.group(1), + result.group(2)) compile_commands.append(entry) except ValueError as err: logging.info('Could not add line from %s: %s', - filepath, err) + cmdfile, err) with open(output, 'wt') as f: json.dump(compile_commands, f, indent=2, sort_keys=True) -- cgit v1.2.3 From ecca4fea1ede446e9ce3afa0b68b07fd1550f4f5 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:15 +0900 Subject: gen_compile_commands: support *.o, *.a, modules.order in positional argument This script currently searches the specified directory for .cmd files. One drawback is it may contain stale .cmd files after you rebuild the kernel several times without 'make clean'. This commit supports *.o, *.a, and modules.order as positional parameters. If such files are given, they are parsed to collect associated .cmd files. I added a generator helper for each of them. This feature is useful to get the list of active .cmd files from the last build, and will be used by the next commit to wire up the compile_commands.json rule to the Makefile. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 100 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index e45f17be8817..92ea0a2ee59e 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -12,6 +12,7 @@ import json import logging import os import re +import subprocess _DEFAULT_OUTPUT = 'compile_commands.json' _DEFAULT_LOG_LEVEL = 'WARNING' @@ -32,8 +33,9 @@ def parse_arguments(): Returns: log_level: A logging level to filter log output. directory: The work directory where the objects were built. + ar: Command used for parsing .a archives. output: Where to write the compile-commands JSON file. - paths: The list of directories to handle to find .cmd files. + paths: The list of files/directories to handle to find .cmd files. """ usage = 'Creates a compile_commands.json database from kernel .cmd files' parser = argparse.ArgumentParser(description=usage) @@ -53,12 +55,21 @@ def parse_arguments(): parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS, default=_DEFAULT_LOG_LEVEL, help=log_level_help) + ar_help = 'command used for parsing .a archives' + parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help) + + paths_help = ('directories to search or files to parse ' + '(files should be *.o, *.a, or modules.order). ' + 'If nothing is specified, the current directory is searched') + parser.add_argument('paths', type=str, nargs='*', help=paths_help) + args = parser.parse_args() return (args.log_level, os.path.abspath(args.directory), args.output, - [args.directory]) + args.ar, + args.paths if len(args.paths) > 0 else [args.directory]) def cmdfiles_in_dir(directory): @@ -81,6 +92,73 @@ def cmdfiles_in_dir(directory): yield os.path.join(dirpath, filename) +def to_cmdfile(path): + """Return the path of .cmd file used for the given build artifact + + Args: + Path: file path + + Returns: + The path to .cmd file + """ + dir, base = os.path.split(path) + return os.path.join(dir, '.' + base + '.cmd') + + +def cmdfiles_for_o(obj): + """Generate the iterator of .cmd files associated with the object + + Yield the .cmd file used to build the given object + + Args: + obj: The object path + + Yields: + The path to .cmd file + """ + yield to_cmdfile(obj) + + +def cmdfiles_for_a(archive, ar): + """Generate the iterator of .cmd files associated with the archive. + + Parse the given archive, and yield every .cmd file used to build it. + + Args: + archive: The archive to parse + + Yields: + The path to every .cmd file found + """ + for obj in subprocess.check_output([ar, '-t', archive]).decode().split(): + yield to_cmdfile(obj) + + +def cmdfiles_for_modorder(modorder): + """Generate the iterator of .cmd files associated with the modules.order. + + Parse the given modules.order, and yield every .cmd file used to build the + contained modules. + + Args: + modorder: The modules.order file to parse + + Yields: + The path to every .cmd file found + """ + with open(modorder) as f: + for line in f: + ko = line.rstrip() + base, ext = os.path.splitext(ko) + if ext != '.ko': + sys.exit('{}: module path must end with .ko'.format(ko)) + mod = base + '.mod' + # The first line of *.mod lists the objects that compose the module. + with open(mod) as m: + for obj in m.readline().split(): + yield to_cmdfile(obj) + + def process_line(root_directory, command_prefix, file_path): """Extracts information from a .cmd line and creates an entry from it. @@ -117,7 +195,7 @@ def process_line(root_directory, command_prefix, file_path): def main(): """Walks through the directory and finds and parses .cmd files.""" - log_level, directory, output, paths = parse_arguments() + log_level, directory, output, ar, paths = parse_arguments() level = getattr(logging, log_level) logging.basicConfig(format='%(levelname)s: %(message)s', level=level) @@ -127,7 +205,21 @@ def main(): compile_commands = [] for path in paths: - cmdfiles = cmdfiles_in_dir(path) + # If 'path' is a directory, handle all .cmd files under it. + # Otherwise, handle .cmd files associated with the file. + # Most of built-in objects are linked via archives (built-in.a or lib.a) + # but some objects are linked to vmlinux directly. + # Modules are listed in modules.order. + if os.path.isdir(path): + cmdfiles = cmdfiles_in_dir(path) + elif path.endswith('.o'): + cmdfiles = cmdfiles_for_o(path) + elif path.endswith('.a'): + cmdfiles = cmdfiles_for_a(path, ar) + elif path.endswith('modules.order'): + cmdfiles = cmdfiles_for_modorder(path) + else: + sys.exit('{}: unknown file type'.format(path)) for cmdfile in cmdfiles: with open(cmdfile, 'rt') as f: -- cgit v1.2.3 From 8b61f748e2a022f9028c612e44b68ad61caec474 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:17 +0900 Subject: gen_compile_commands: remove the warning about too few .cmd files This warning was useful when users previously needed to manually build the kernel and run this script. Now you can simply do 'make compile_commands.json', which updates all the necessary build artifacts and automatically creates the compilation database. There is no more worry for a mistake like "Oh, I forgot to build the kernel". Now, this warning is rather annoying. You can create compile_commands.json for an external module: $ make M=/path/to/your/external/module compile_commands.json Then, this warning is displayed since there are usually less than 300 files in a single module. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 92ea0a2ee59e..19963708bcf8 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -21,11 +21,6 @@ _FILENAME_PATTERN = r'^\..*\.cmd$' _LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$' _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] -# A kernel build generally has over 2000 entries in its compile_commands.json -# database. If this code finds 300 or fewer, then warn the user that they might -# not have all the .cmd files, and they might need to compile the kernel. -_LOW_COUNT_THRESHOLD = 300 - def parse_arguments(): """Sets up and parses command-line arguments. @@ -236,11 +231,6 @@ def main(): with open(output, 'wt') as f: json.dump(compile_commands, f, indent=2, sort_keys=True) - count = len(compile_commands) - if count < _LOW_COUNT_THRESHOLD: - logging.warning( - 'Found %s entries. Have you compiled the kernel?', count) - if __name__ == '__main__': main() -- cgit v1.2.3 From 6ad7cbc01527223f3f92baac9b122f15651cf76b Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Sat, 22 Aug 2020 23:56:18 +0900 Subject: Makefile: Add clang-tidy and static analyzer support to makefile This patch adds clang-tidy and the clang static-analyzer as make targets. The goal of this patch is to make static analysis tools usable and extendable by any developer or researcher who is familiar with basic c++. The current static analysis tools require intimate knowledge of the internal workings of the static analysis. Clang-tidy and the clang static analyzers expose an easy to use api and allow users unfamiliar with clang to write new checks with relative ease. ===Clang-tidy=== Clang-tidy is an easily extendable 'linter' that runs on the AST. Clang-tidy checks are easy to write and understand. A check consists of two parts, a matcher and a checker. The matcher is created using a domain specific language that acts on the AST (https://clang.llvm.org/docs/LibASTMatchersReference.html). When AST nodes are found by the matcher a callback is made to the checker. The checker can then execute additional checks and issue warnings. Here is an example clang-tidy check to report functions that have calls to local_irq_disable without calls to local_irq_enable and vice-versa. Functions flagged with __attribute((annotation("ignore_irq_balancing"))) are ignored for analysis. (https://reviews.llvm.org/D65828) ===Clang static analyzer=== The clang static analyzer is a more powerful static analysis tool that uses symbolic execution to find bugs. Currently there is a check that looks for potential security bugs from invalid uses of kmalloc and kfree. There are several more general purpose checks that are useful for the kernel. The clang static analyzer is well documented and designed to be extensible. (https://clang-analyzer.llvm.org/checker_dev_manual.html) (https://github.com/haoNoQ/clang-analyzer-guide/releases/download/v0.1/clang-analyzer-guide-v0.1.pdf) The main draw of the clang tools is how accessible they are. The clang documentation is very nice and these tools are built specifically to be easily extendable by any developer. They provide an accessible method of bug-finding and research to people who are not overly familiar with the kernel codebase. Signed-off-by: Nathan Huckleberry Reviewed-by: Nick Desaulniers Tested-by: Nick Desaulniers Tested-by: Lukas Bulwahn Signed-off-by: Masahiro Yamada --- scripts/clang-tools/gen_compile_commands.py | 236 ++++++++++++++++++++++++++++ scripts/clang-tools/run-clang-tools.py | 74 +++++++++ scripts/gen_compile_commands.py | 236 ---------------------------- 3 files changed, 310 insertions(+), 236 deletions(-) create mode 100755 scripts/clang-tools/gen_compile_commands.py create mode 100755 scripts/clang-tools/run-clang-tools.py delete mode 100755 scripts/gen_compile_commands.py (limited to 'scripts') diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py new file mode 100755 index 000000000000..19963708bcf8 --- /dev/null +++ b/scripts/clang-tools/gen_compile_commands.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) Google LLC, 2018 +# +# Author: Tom Roeder +# +"""A tool for generating compile_commands.json in the Linux kernel.""" + +import argparse +import json +import logging +import os +import re +import subprocess + +_DEFAULT_OUTPUT = 'compile_commands.json' +_DEFAULT_LOG_LEVEL = 'WARNING' + +_FILENAME_PATTERN = r'^\..*\.cmd$' +_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$' +_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] + + +def parse_arguments(): + """Sets up and parses command-line arguments. + + Returns: + log_level: A logging level to filter log output. + directory: The work directory where the objects were built. + ar: Command used for parsing .a archives. + output: Where to write the compile-commands JSON file. + paths: The list of files/directories to handle to find .cmd files. + """ + usage = 'Creates a compile_commands.json database from kernel .cmd files' + parser = argparse.ArgumentParser(description=usage) + + directory_help = ('specify the output directory used for the kernel build ' + '(defaults to the working directory)') + parser.add_argument('-d', '--directory', type=str, default='.', + help=directory_help) + + output_help = ('path to the output command database (defaults to ' + + _DEFAULT_OUTPUT + ')') + parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT, + help=output_help) + + log_level_help = ('the level of log messages to produce (defaults to ' + + _DEFAULT_LOG_LEVEL + ')') + parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS, + default=_DEFAULT_LOG_LEVEL, help=log_level_help) + + ar_help = 'command used for parsing .a archives' + parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help) + + paths_help = ('directories to search or files to parse ' + '(files should be *.o, *.a, or modules.order). ' + 'If nothing is specified, the current directory is searched') + parser.add_argument('paths', type=str, nargs='*', help=paths_help) + + args = parser.parse_args() + + return (args.log_level, + os.path.abspath(args.directory), + args.output, + args.ar, + args.paths if len(args.paths) > 0 else [args.directory]) + + +def cmdfiles_in_dir(directory): + """Generate the iterator of .cmd files found under the directory. + + Walk under the given directory, and yield every .cmd file found. + + Args: + directory: The directory to search for .cmd files. + + Yields: + The path to a .cmd file. + """ + + filename_matcher = re.compile(_FILENAME_PATTERN) + + for dirpath, _, filenames in os.walk(directory): + for filename in filenames: + if filename_matcher.match(filename): + yield os.path.join(dirpath, filename) + + +def to_cmdfile(path): + """Return the path of .cmd file used for the given build artifact + + Args: + Path: file path + + Returns: + The path to .cmd file + """ + dir, base = os.path.split(path) + return os.path.join(dir, '.' + base + '.cmd') + + +def cmdfiles_for_o(obj): + """Generate the iterator of .cmd files associated with the object + + Yield the .cmd file used to build the given object + + Args: + obj: The object path + + Yields: + The path to .cmd file + """ + yield to_cmdfile(obj) + + +def cmdfiles_for_a(archive, ar): + """Generate the iterator of .cmd files associated with the archive. + + Parse the given archive, and yield every .cmd file used to build it. + + Args: + archive: The archive to parse + + Yields: + The path to every .cmd file found + """ + for obj in subprocess.check_output([ar, '-t', archive]).decode().split(): + yield to_cmdfile(obj) + + +def cmdfiles_for_modorder(modorder): + """Generate the iterator of .cmd files associated with the modules.order. + + Parse the given modules.order, and yield every .cmd file used to build the + contained modules. + + Args: + modorder: The modules.order file to parse + + Yields: + The path to every .cmd file found + """ + with open(modorder) as f: + for line in f: + ko = line.rstrip() + base, ext = os.path.splitext(ko) + if ext != '.ko': + sys.exit('{}: module path must end with .ko'.format(ko)) + mod = base + '.mod' + # The first line of *.mod lists the objects that compose the module. + with open(mod) as m: + for obj in m.readline().split(): + yield to_cmdfile(obj) + + +def process_line(root_directory, command_prefix, file_path): + """Extracts information from a .cmd line and creates an entry from it. + + Args: + root_directory: The directory that was searched for .cmd files. Usually + used directly in the "directory" entry in compile_commands.json. + command_prefix: The extracted command line, up to the last element. + file_path: The .c file from the end of the extracted command. + Usually relative to root_directory, but sometimes absolute. + + Returns: + An entry to append to compile_commands. + + Raises: + ValueError: Could not find the extracted file based on file_path and + root_directory or file_directory. + """ + # The .cmd files are intended to be included directly by Make, so they + # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the + # kernel version). The compile_commands.json file is not interepreted + # by Make, so this code replaces the escaped version with '#'. + prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#') + + # Use os.path.abspath() to normalize the path resolving '.' and '..' . + abs_path = os.path.abspath(os.path.join(root_directory, file_path)) + if not os.path.exists(abs_path): + raise ValueError('File %s not found' % abs_path) + return { + 'directory': root_directory, + 'file': abs_path, + 'command': prefix + file_path, + } + + +def main(): + """Walks through the directory and finds and parses .cmd files.""" + log_level, directory, output, ar, paths = parse_arguments() + + level = getattr(logging, log_level) + logging.basicConfig(format='%(levelname)s: %(message)s', level=level) + + line_matcher = re.compile(_LINE_PATTERN) + + compile_commands = [] + + for path in paths: + # If 'path' is a directory, handle all .cmd files under it. + # Otherwise, handle .cmd files associated with the file. + # Most of built-in objects are linked via archives (built-in.a or lib.a) + # but some objects are linked to vmlinux directly. + # Modules are listed in modules.order. + if os.path.isdir(path): + cmdfiles = cmdfiles_in_dir(path) + elif path.endswith('.o'): + cmdfiles = cmdfiles_for_o(path) + elif path.endswith('.a'): + cmdfiles = cmdfiles_for_a(path, ar) + elif path.endswith('modules.order'): + cmdfiles = cmdfiles_for_modorder(path) + else: + sys.exit('{}: unknown file type'.format(path)) + + for cmdfile in cmdfiles: + with open(cmdfile, 'rt') as f: + result = line_matcher.match(f.readline()) + if result: + try: + entry = process_line(directory, result.group(1), + result.group(2)) + compile_commands.append(entry) + except ValueError as err: + logging.info('Could not add line from %s: %s', + cmdfile, err) + + with open(output, 'wt') as f: + json.dump(compile_commands, f, indent=2, sort_keys=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/clang-tools/run-clang-tools.py b/scripts/clang-tools/run-clang-tools.py new file mode 100755 index 000000000000..fa7655c7cec0 --- /dev/null +++ b/scripts/clang-tools/run-clang-tools.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) Google LLC, 2020 +# +# Author: Nathan Huckleberry +# +"""A helper routine run clang-tidy and the clang static-analyzer on +compile_commands.json. +""" + +import argparse +import json +import multiprocessing +import os +import subprocess +import sys + + +def parse_arguments(): + """Set up and parses command-line arguments. + Returns: + args: Dict of parsed args + Has keys: [path, type] + """ + usage = """Run clang-tidy or the clang static-analyzer on a + compilation database.""" + parser = argparse.ArgumentParser(description=usage) + + type_help = "Type of analysis to be performed" + parser.add_argument("type", + choices=["clang-tidy", "clang-analyzer"], + help=type_help) + path_help = "Path to the compilation database to parse" + parser.add_argument("path", type=str, help=path_help) + + return parser.parse_args() + + +def init(l, a): + global lock + global args + lock = l + args = a + + +def run_analysis(entry): + # Disable all checks, then re-enable the ones we want + checks = "-checks=-*," + if args.type == "clang-tidy": + checks += "linuxkernel-*" + else: + checks += "clang-analyzer-*" + p = subprocess.run(["clang-tidy", "-p", args.path, checks, entry["file"]], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=entry["directory"]) + with lock: + sys.stderr.buffer.write(p.stdout) + + +def main(): + args = parse_arguments() + + lock = multiprocessing.Lock() + pool = multiprocessing.Pool(initializer=init, initargs=(lock, args)) + # Read JSON data into the datastore variable + with open(args.path, "r") as f: + datastore = json.load(f) + pool.map(run_analysis, datastore) + + +if __name__ == "__main__": + main() diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py deleted file mode 100755 index 19963708bcf8..000000000000 --- a/scripts/gen_compile_commands.py +++ /dev/null @@ -1,236 +0,0 @@ -#!/usr/bin/env python -# SPDX-License-Identifier: GPL-2.0 -# -# Copyright (C) Google LLC, 2018 -# -# Author: Tom Roeder -# -"""A tool for generating compile_commands.json in the Linux kernel.""" - -import argparse -import json -import logging -import os -import re -import subprocess - -_DEFAULT_OUTPUT = 'compile_commands.json' -_DEFAULT_LOG_LEVEL = 'WARNING' - -_FILENAME_PATTERN = r'^\..*\.cmd$' -_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$' -_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] - - -def parse_arguments(): - """Sets up and parses command-line arguments. - - Returns: - log_level: A logging level to filter log output. - directory: The work directory where the objects were built. - ar: Command used for parsing .a archives. - output: Where to write the compile-commands JSON file. - paths: The list of files/directories to handle to find .cmd files. - """ - usage = 'Creates a compile_commands.json database from kernel .cmd files' - parser = argparse.ArgumentParser(description=usage) - - directory_help = ('specify the output directory used for the kernel build ' - '(defaults to the working directory)') - parser.add_argument('-d', '--directory', type=str, default='.', - help=directory_help) - - output_help = ('path to the output command database (defaults to ' + - _DEFAULT_OUTPUT + ')') - parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT, - help=output_help) - - log_level_help = ('the level of log messages to produce (defaults to ' + - _DEFAULT_LOG_LEVEL + ')') - parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS, - default=_DEFAULT_LOG_LEVEL, help=log_level_help) - - ar_help = 'command used for parsing .a archives' - parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help) - - paths_help = ('directories to search or files to parse ' - '(files should be *.o, *.a, or modules.order). ' - 'If nothing is specified, the current directory is searched') - parser.add_argument('paths', type=str, nargs='*', help=paths_help) - - args = parser.parse_args() - - return (args.log_level, - os.path.abspath(args.directory), - args.output, - args.ar, - args.paths if len(args.paths) > 0 else [args.directory]) - - -def cmdfiles_in_dir(directory): - """Generate the iterator of .cmd files found under the directory. - - Walk under the given directory, and yield every .cmd file found. - - Args: - directory: The directory to search for .cmd files. - - Yields: - The path to a .cmd file. - """ - - filename_matcher = re.compile(_FILENAME_PATTERN) - - for dirpath, _, filenames in os.walk(directory): - for filename in filenames: - if filename_matcher.match(filename): - yield os.path.join(dirpath, filename) - - -def to_cmdfile(path): - """Return the path of .cmd file used for the given build artifact - - Args: - Path: file path - - Returns: - The path to .cmd file - """ - dir, base = os.path.split(path) - return os.path.join(dir, '.' + base + '.cmd') - - -def cmdfiles_for_o(obj): - """Generate the iterator of .cmd files associated with the object - - Yield the .cmd file used to build the given object - - Args: - obj: The object path - - Yields: - The path to .cmd file - """ - yield to_cmdfile(obj) - - -def cmdfiles_for_a(archive, ar): - """Generate the iterator of .cmd files associated with the archive. - - Parse the given archive, and yield every .cmd file used to build it. - - Args: - archive: The archive to parse - - Yields: - The path to every .cmd file found - """ - for obj in subprocess.check_output([ar, '-t', archive]).decode().split(): - yield to_cmdfile(obj) - - -def cmdfiles_for_modorder(modorder): - """Generate the iterator of .cmd files associated with the modules.order. - - Parse the given modules.order, and yield every .cmd file used to build the - contained modules. - - Args: - modorder: The modules.order file to parse - - Yields: - The path to every .cmd file found - """ - with open(modorder) as f: - for line in f: - ko = line.rstrip() - base, ext = os.path.splitext(ko) - if ext != '.ko': - sys.exit('{}: module path must end with .ko'.format(ko)) - mod = base + '.mod' - # The first line of *.mod lists the objects that compose the module. - with open(mod) as m: - for obj in m.readline().split(): - yield to_cmdfile(obj) - - -def process_line(root_directory, command_prefix, file_path): - """Extracts information from a .cmd line and creates an entry from it. - - Args: - root_directory: The directory that was searched for .cmd files. Usually - used direct