From 4b2bd20c350a66d4f8a7c2da48b115fd4e06d15e Mon Sep 17 00:00:00 2001 From: Sumera Priyadarsini Date: Wed, 5 Aug 2020 14:27:40 +0530 Subject: scripts: coccicheck: Add chain mode to list of modes This patch adds chain mode to the list of available modes in coccicheck. Signed-off-by: Sumera Priyadarsini Signed-off-by: Julia Lawall --- scripts/coccicheck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/coccicheck b/scripts/coccicheck index e04d328210ac..6e37cf36caae 100755 --- a/scripts/coccicheck +++ b/scripts/coccicheck @@ -99,7 +99,7 @@ fi if [ "$MODE" = "" ] ; then if [ "$ONLINE" = "0" ] ; then echo 'You have not explicitly specified the mode to use. Using default "report" mode.' - echo 'Available modes are the following: patch, report, context, org' + echo 'Available modes are the following: patch, report, context, org, chain' echo 'You can specify the mode with "make coccicheck MODE="' echo 'Note however that some modes are not implemented by some semantic patches.' fi -- cgit v1.2.3 From 7a2624e6de03050c9f15af21c216818d5508b5e9 Mon Sep 17 00:00:00 2001 From: Alex Dewar Date: Fri, 21 Aug 2020 00:55:57 +0100 Subject: coccinelle: add patch rule for dma_alloc_coherent Commit dfd32cad146e ("dma-mapping: remove dma_zalloc_coherent()") removed the definition of dma_zalloc_coherent() and also removed the corresponding patch rule for replacing instances of dma_alloc_coherent + memset in zalloc-simple.cocci (though left the report rule). Add a new patch rule to remove unnecessary calls to memset after allocating with dma_alloc_coherent. While we're at it, fix a couple of typos. Fixes: dfd32cad146e ("dma-mapping: remove dma_zalloc_coherent()") Signed-off-by: Alex Dewar Signed-off-by: Julia Lawall --- scripts/coccinelle/api/alloc/zalloc-simple.cocci | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/coccinelle/api/alloc/zalloc-simple.cocci b/scripts/coccinelle/api/alloc/zalloc-simple.cocci index 26cda3f48f01..b3d0c3c230c1 100644 --- a/scripts/coccinelle/api/alloc/zalloc-simple.cocci +++ b/scripts/coccinelle/api/alloc/zalloc-simple.cocci @@ -127,6 +127,16 @@ statement S; if ((x==NULL) || ...) S - memset((T2)x,0,E1); +@depends on patch@ +type T, T2; +expression x; +expression E1,E2,E3,E4; +statement S; +@@ + x = (T)dma_alloc_coherent(E1, E2, E3, E4); + if ((x==NULL) || ...) S +- memset((T2)x, 0, E2); + //---------------------------------------------------------- // For org mode //---------------------------------------------------------- @@ -199,9 +209,9 @@ statement S; position p; @@ - x = (T)dma_alloc_coherent@p(E2,E1,E3,E4); + x = (T)dma_alloc_coherent@p(E1,E2,E3,E4); if ((x==NULL) || ...) S - memset((T2)x,0,E1); + memset((T2)x,0,E2); @script:python depends on org@ p << r2.p; @@ -217,7 +227,7 @@ p << r2.p; x << r2.x; @@ -msg="WARNING: dma_alloc_coherent use in %s already zeroes out memory, so memset is not needed" % (x) +msg="WARNING: dma_alloc_coherent used in %s already zeroes out memory, so memset is not needed" % (x) coccilib.report.print_report(p[0], msg) //----------------------------------------------------------------- -- cgit v1.2.3 From a2fc3718bc22e85378085568ecc5765fb28cabce Mon Sep 17 00:00:00 2001 From: Denis Efremov Date: Fri, 21 Aug 2020 23:11:37 +0300 Subject: coccinelle: api: add kobj_to_dev.cocci script Use kobj_to_dev() instead of container_of(). Signed-off-by: Denis Efremov Signed-off-by: Julia Lawall --- scripts/coccinelle/api/kobj_to_dev.cocci | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 scripts/coccinelle/api/kobj_to_dev.cocci (limited to 'scripts') diff --git a/scripts/coccinelle/api/kobj_to_dev.cocci b/scripts/coccinelle/api/kobj_to_dev.cocci new file mode 100644 index 000000000000..cd5d31c6fe76 --- /dev/null +++ b/scripts/coccinelle/api/kobj_to_dev.cocci @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0-only +/// +/// Use kobj_to_dev() instead of container_of() +/// +// Confidence: High +// Copyright: (C) 2020 Denis Efremov ISPRAS +// Options: --no-includes --include-headers +// +// Keywords: kobj_to_dev, container_of +// + +virtual context +virtual report +virtual org +virtual patch + + +@r depends on !patch@ +expression ptr; +symbol kobj; +position p; +@@ + +* container_of(ptr, struct device, kobj)@p + + +@depends on patch@ +expression ptr; +@@ + +- container_of(ptr, struct device, kobj) ++ kobj_to_dev(ptr) + + +@script:python depends on report@ +p << r.p; +@@ + +coccilib.report.print_report(p[0], "WARNING opportunity for kobj_to_dev()") + +@script:python depends on org@ +p << r.p; +@@ + +coccilib.org.print_todo(p[0], "WARNING opportunity for kobj_to_dev()") -- cgit v1.2.3 From 6e22ab9da79343532cd3cde39df25e5a5478c692 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Tue, 25 Aug 2020 21:21:20 +0200 Subject: bpf: Add d_path helper Adding d_path helper function that returns full path for given 'struct path' object, which needs to be the kernel BTF 'path' object. The path is returned in buffer provided 'buf' of size 'sz' and is zero terminated. bpf_d_path(&file->f_path, buf, size); The helper calls directly d_path function, so there's only limited set of function it can be called from. Adding just very modest set for the start. Updating also bpf.h tools uapi header and adding 'path' to bpf_helpers_doc.py script. Signed-off-by: Jiri Olsa Signed-off-by: Alexei Starovoitov Acked-by: Andrii Nakryiko Acked-by: KP Singh Link: https://lore.kernel.org/bpf/20200825192124.710397-11-jolsa@kernel.org --- scripts/bpf_helpers_doc.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/bpf_helpers_doc.py b/scripts/bpf_helpers_doc.py index 5bfa448b4704..08388173973f 100755 --- a/scripts/bpf_helpers_doc.py +++ b/scripts/bpf_helpers_doc.py @@ -432,6 +432,7 @@ class PrinterHelpers(Printer): 'struct __sk_buff', 'struct sk_msg_md', 'struct xdp_md', + 'struct path', ] known_types = { '...', @@ -472,6 +473,7 @@ class PrinterHelpers(Printer): 'struct tcp_request_sock', 'struct udp6_sock', 'struct task_struct', + 'struct path', } mapped_types = { 'u8': '__u8', -- cgit v1.2.3 From 23cd88c91343349a0d67a227f9effdde0cbe09af Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 21 Aug 2020 11:43:58 +0900 Subject: kbuild: hide commands to run Kconfig, and show short log for syncconfig Some targets (localyesconfig, localmodconfig, defconfig) hide the command running, but the others do not. Users know which Kconfig flavor they are running, so it is OK to hide the command. Add $(Q) to all commands consistently. If you want to see the full command running, pass V=1 from the command line. syncconfig is the exceptional case, which occurs without explicit command invocation by the user. Display the Kbuild-style log for it. The ugly bare log will go away. [Before] scripts/kconfig/conf --syncconfig Kconfig [After] SYNC include/config/auto.conf Signed-off-by: Masahiro Yamada --- scripts/kconfig/Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/kconfig/Makefile b/scripts/kconfig/Makefile index 52b59bf9efe4..e46df0a2d4f9 100644 --- a/scripts/kconfig/Makefile +++ b/scripts/kconfig/Makefile @@ -20,19 +20,19 @@ endif unexport CONFIG_ xconfig: $(obj)/qconf - $< $(silent) $(Kconfig) + $(Q)$< $(silent) $(Kconfig) gconfig: $(obj)/gconf - $< $(silent) $(Kconfig) + $(Q)$< $(silent) $(Kconfig) menuconfig: $(obj)/mconf - $< $(silent) $(Kconfig) + $(Q)$< $(silent) $(Kconfig) config: $(obj)/conf - $< $(silent) --oldaskconfig $(Kconfig) + $(Q)$< $(silent) --oldaskconfig $(Kconfig) nconfig: $(obj)/nconf - $< $(silent) $(Kconfig) + $(Q)$< $(silent) $(Kconfig) build_menuconfig: $(obj)/mconf @@ -68,12 +68,12 @@ simple-targets := oldconfig allnoconfig allyesconfig allmodconfig \ PHONY += $(simple-targets) $(simple-targets): $(obj)/conf - $< $(silent) --$@ $(Kconfig) + $(Q)$< $(silent) --$@ $(Kconfig) PHONY += savedefconfig defconfig savedefconfig: $(obj)/conf - $< $(silent) --$@=defconfig $(Kconfig) + $(Q)$< $(silent) --$@=defconfig $(Kconfig) defconfig: $(obj)/conf ifneq ($(wildcard $(srctree)/arch/$(SRCARCH)/configs/$(KBUILD_DEFCONFIG)),) @@ -111,7 +111,7 @@ tinyconfig: # CHECK: -o cache_dir= working? PHONY += testconfig testconfig: $(obj)/conf - $(PYTHON3) -B -m pytest $(srctree)/$(src)/tests \ + $(Q)$(PYTHON3) -B -m pytest $(srctree)/$(src)/tests \ -o cache_dir=$(abspath $(obj)/tests/.cache) \ $(if $(findstring 1,$(KBUILD_VERBOSE)),--capture=no) clean-files += tests/.cache -- cgit v1.2.3 From 8a685db32f2b3495c91efab6e9462c2fd934778f Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:09 +0900 Subject: gen_compile_commands: parse only the first line of .*.cmd files After the allmodconfig build, this script takes about 5 sec on my machine. Most of the run-time is consumed for needless regex matching. We know the format of .*.cmd file; the first line is the build command. There is no need to parse the rest. With this optimization, now it runs 4 times faster. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers Tested-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index c458696ef3a7..1bcf33a93cb9 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -125,11 +125,8 @@ def main(): filepath = os.path.join(dirpath, filename) with open(filepath, 'rt') as f: - for line in f: - result = line_matcher.match(line) - if not result: - continue - + result = line_matcher.match(f.readline()) + if result: try: entry = process_line(directory, dirpath, result.group(1), result.group(2)) -- cgit v1.2.3 From ea6cedc5b8a4e666da41f1173ea19970fa60683f Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:10 +0900 Subject: gen_compile_commands: use choices for --log_levels option Use 'choices' to check if the given parameter is valid. I also simplified the help message because, with 'choices', --help shows the list of valid parameters: --log_level {DEBUG,INFO,WARNING,ERROR,CRITICAL} I started the help message with a lower case, "the level of log ..." in order to be consistent with the -h option: -h, --help show this help message and exit The message "show this help ..." comes from the ArgumentParser library code, and I do not know how to change it. So, I changed our code. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 1bcf33a93cb9..535248cf2d7e 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -45,24 +45,18 @@ def parse_arguments(): 'compile_commands.json in the search directory)') parser.add_argument('-o', '--output', type=str, help=output_help) - log_level_help = ('The level of log messages to produce (one of ' + - ', '.join(_VALID_LOG_LEVELS) + '; defaults to ' + + log_level_help = ('the level of log messages to produce (defaults to ' + _DEFAULT_LOG_LEVEL + ')') - parser.add_argument( - '--log_level', type=str, default=_DEFAULT_LOG_LEVEL, - help=log_level_help) + parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS, + default=_DEFAULT_LOG_LEVEL, help=log_level_help) args = parser.parse_args() - log_level = args.log_level - if log_level not in _VALID_LOG_LEVELS: - raise ValueError('%s is not a valid log level' % log_level) - directory = args.directory or os.getcwd() output = args.output or os.path.join(directory, _DEFAULT_OUTPUT) directory = os.path.abspath(directory) - return log_level, directory, output + return args.log_level, directory, output def process_line(root_directory, file_directory, command_prefix, relative_path): -- cgit v1.2.3 From 6ca4c6d25949117dc5b4845612e290b6d89e70a8 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:11 +0900 Subject: gen_compile_commands: do not support .cmd files under tools/ directory The tools/ directory uses a different build system, and the format of .cmd files is different because the tools builds run in a different work directory. Supporting two formats compilicates the script. The only loss by this change is objtool. Also, rename the confusing variable 'relative_path' because it is not necessarily a relative path. When the output directory is not the direct child of the source tree (e.g. O=foo/bar), it is an absolute path. Rename it to 'file_path'. os.path.join(root_directory, file_path) works whether the file_path is relative or not. If file_path is already absolute, it returns it as-is. I used os.path.abspath() to normalize file paths. If you run this script against the kernel built with O=foo option, the file_path contains '../' patterns. os.path.abspath() fixes up 'foo/bar/../baz' into 'foo/baz', and produces a cleaner commands_database.json. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 535248cf2d7e..49fff0b0b385 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -59,23 +59,21 @@ def parse_arguments(): return args.log_level, directory, output -def process_line(root_directory, file_directory, command_prefix, relative_path): +def process_line(root_directory, command_prefix, file_path): """Extracts information from a .cmd line and creates an entry from it. Args: root_directory: The directory that was searched for .cmd files. Usually used directly in the "directory" entry in compile_commands.json. - file_directory: The path to the directory the .cmd file was found in. command_prefix: The extracted command line, up to the last element. - relative_path: The .c file from the end of the extracted command. - Usually relative to root_directory, but sometimes relative to - file_directory and sometimes neither. + file_path: The .c file from the end of the extracted command. + Usually relative to root_directory, but sometimes absolute. Returns: An entry to append to compile_commands. Raises: - ValueError: Could not find the extracted file based on relative_path and + ValueError: Could not find the extracted file based on file_path and root_directory or file_directory. """ # The .cmd files are intended to be included directly by Make, so they @@ -84,20 +82,14 @@ def process_line(root_directory, file_directory, command_prefix, relative_path): # by Make, so this code replaces the escaped version with '#'. prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#') - cur_dir = root_directory - expected_path = os.path.join(cur_dir, relative_path) - if not os.path.exists(expected_path): - # Try using file_directory instead. Some of the tools have a different - # style of .cmd file than the kernel. - cur_dir = file_directory - expected_path = os.path.join(cur_dir, relative_path) - if not os.path.exists(expected_path): - raise ValueError('File %s not in %s or %s' % - (relative_path, root_directory, file_directory)) + # Use os.path.abspath() to normalize the path resolving '.' and '..' . + abs_path = os.path.abspath(os.path.join(root_directory, file_path)) + if not os.path.exists(abs_path): + raise ValueError('File %s not found' % abs_path) return { - 'directory': cur_dir, - 'file': relative_path, - 'command': prefix + relative_path, + 'directory': root_directory, + 'file': abs_path, + 'command': prefix + file_path, } @@ -122,7 +114,7 @@ def main(): result = line_matcher.match(f.readline()) if result: try: - entry = process_line(directory, dirpath, + entry = process_line(directory, result.group(1), result.group(2)) compile_commands.append(entry) except ValueError as err: -- cgit v1.2.3 From 0a7d376d04a3c84b503f1efecde8f9d9a7430269 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:12 +0900 Subject: gen_compile_commands: reword the help message of -d option I think the help message of the -d option is somewhat misleading. Path to the kernel source directory to search (defaults to the working directory) The part "kernel source directory" is the source of the confusion. Some people misunderstand as if this script did not support separate output directories. Actually, this script also works for out-of-tree builds. You can use the -d option to point to the object output directory, not to the source directory. It should match to the O= option used in the previous kernel build, and then appears in the "directory" field of compile_commands.json. Reword the help message. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 49fff0b0b385..f37c1dac8db4 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -31,13 +31,13 @@ def parse_arguments(): Returns: log_level: A logging level to filter log output. - directory: The directory to search for .cmd files. + directory: The work directory where the objects were built. output: Where to write the compile-commands JSON file. """ usage = 'Creates a compile_commands.json database from kernel .cmd files' parser = argparse.ArgumentParser(description=usage) - directory_help = ('Path to the kernel source directory to search ' + directory_help = ('specify the output directory used for the kernel build ' '(defaults to the working directory)') parser.add_argument('-d', '--directory', type=str, help=directory_help) -- cgit v1.2.3 From 6fca36f1d82ad5bc385187f0aed93bbaefaa2172 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:13 +0900 Subject: gen_compile_commands: make -o option independent of -d option Change the -o option independent of the -d option, which is I think clearer behavior. Some people may like to use -d to specify a separate output directory, but still output the compile_commands.py in the source directory (unless the source tree is read-only) because it is the default location Clang Tools search for the compilation database. Also, move the default parameter to the default= argument of the .add_argument(). Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index f37c1dac8db4..71a0630ae188 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -39,11 +39,13 @@ def parse_arguments(): directory_help = ('specify the output directory used for the kernel build ' '(defaults to the working directory)') - parser.add_argument('-d', '--directory', type=str, help=directory_help) + parser.add_argument('-d', '--directory', type=str, default='.', + help=directory_help) - output_help = ('The location to write compile_commands.json (defaults to ' - 'compile_commands.json in the search directory)') - parser.add_argument('-o', '--output', type=str, help=output_help) + output_help = ('path to the output command database (defaults to ' + + _DEFAULT_OUTPUT + ')') + parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT, + help=output_help) log_level_help = ('the level of log messages to produce (defaults to ' + _DEFAULT_LOG_LEVEL + ')') @@ -52,11 +54,9 @@ def parse_arguments(): args = parser.parse_args() - directory = args.directory or os.getcwd() - output = args.output or os.path.join(directory, _DEFAULT_OUTPUT) - directory = os.path.abspath(directory) - - return args.log_level, directory, output + return (args.log_level, + os.path.abspath(args.directory), + args.output) def process_line(root_directory, command_prefix, file_path): -- cgit v1.2.3 From fc2cb22ec61c1a537337cd0eba458863564a0e1f Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:14 +0900 Subject: gen_compile_commands: move directory walk to a generator function Currently, this script walks under the specified directory (default to the current directory), then parses all .cmd files found. Split it into a separate helper function because the next commit will add more helpers to pick up .cmd files associated with given file(s). There is no point to build and return a huge list at once. I used a generator so it works in the for-loop with less memory. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 44 ++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 12 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 71a0630ae188..e45f17be8817 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -33,6 +33,7 @@ def parse_arguments(): log_level: A logging level to filter log output. directory: The work directory where the objects were built. output: Where to write the compile-commands JSON file. + paths: The list of directories to handle to find .cmd files. """ usage = 'Creates a compile_commands.json database from kernel .cmd files' parser = argparse.ArgumentParser(description=usage) @@ -56,7 +57,28 @@ def parse_arguments(): return (args.log_level, os.path.abspath(args.directory), - args.output) + args.output, + [args.directory]) + + +def cmdfiles_in_dir(directory): + """Generate the iterator of .cmd files found under the directory. + + Walk under the given directory, and yield every .cmd file found. + + Args: + directory: The directory to search for .cmd files. + + Yields: + The path to a .cmd file. + """ + + filename_matcher = re.compile(_FILENAME_PATTERN) + + for dirpath, _, filenames in os.walk(directory): + for filename in filenames: + if filename_matcher.match(filename): + yield os.path.join(dirpath, filename) def process_line(root_directory, command_prefix, file_path): @@ -95,31 +117,29 @@ def process_line(root_directory, command_prefix, file_path): def main(): """Walks through the directory and finds and parses .cmd files.""" - log_level, directory, output = parse_arguments() + log_level, directory, output, paths = parse_arguments() level = getattr(logging, log_level) logging.basicConfig(format='%(levelname)s: %(message)s', level=level) - filename_matcher = re.compile(_FILENAME_PATTERN) line_matcher = re.compile(_LINE_PATTERN) compile_commands = [] - for dirpath, _, filenames in os.walk(directory): - for filename in filenames: - if not filename_matcher.match(filename): - continue - filepath = os.path.join(dirpath, filename) - with open(filepath, 'rt') as f: + for path in paths: + cmdfiles = cmdfiles_in_dir(path) + + for cmdfile in cmdfiles: + with open(cmdfile, 'rt') as f: result = line_matcher.match(f.readline()) if result: try: - entry = process_line(directory, - result.group(1), result.group(2)) + entry = process_line(directory, result.group(1), + result.group(2)) compile_commands.append(entry) except ValueError as err: logging.info('Could not add line from %s: %s', - filepath, err) + cmdfile, err) with open(output, 'wt') as f: json.dump(compile_commands, f, indent=2, sort_keys=True) -- cgit v1.2.3 From ecca4fea1ede446e9ce3afa0b68b07fd1550f4f5 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:15 +0900 Subject: gen_compile_commands: support *.o, *.a, modules.order in positional argument This script currently searches the specified directory for .cmd files. One drawback is it may contain stale .cmd files after you rebuild the kernel several times without 'make clean'. This commit supports *.o, *.a, and modules.order as positional parameters. If such files are given, they are parsed to collect associated .cmd files. I added a generator helper for each of them. This feature is useful to get the list of active .cmd files from the last build, and will be used by the next commit to wire up the compile_commands.json rule to the Makefile. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 100 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index e45f17be8817..92ea0a2ee59e 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -12,6 +12,7 @@ import json import logging import os import re +import subprocess _DEFAULT_OUTPUT = 'compile_commands.json' _DEFAULT_LOG_LEVEL = 'WARNING' @@ -32,8 +33,9 @@ def parse_arguments(): Returns: log_level: A logging level to filter log output. directory: The work directory where the objects were built. + ar: Command used for parsing .a archives. output: Where to write the compile-commands JSON file. - paths: The list of directories to handle to find .cmd files. + paths: The list of files/directories to handle to find .cmd files. """ usage = 'Creates a compile_commands.json database from kernel .cmd files' parser = argparse.ArgumentParser(description=usage) @@ -53,12 +55,21 @@ def parse_arguments(): parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS, default=_DEFAULT_LOG_LEVEL, help=log_level_help) + ar_help = 'command used for parsing .a archives' + parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help) + + paths_help = ('directories to search or files to parse ' + '(files should be *.o, *.a, or modules.order). ' + 'If nothing is specified, the current directory is searched') + parser.add_argument('paths', type=str, nargs='*', help=paths_help) + args = parser.parse_args() return (args.log_level, os.path.abspath(args.directory), args.output, - [args.directory]) + args.ar, + args.paths if len(args.paths) > 0 else [args.directory]) def cmdfiles_in_dir(directory): @@ -81,6 +92,73 @@ def cmdfiles_in_dir(directory): yield os.path.join(dirpath, filename) +def to_cmdfile(path): + """Return the path of .cmd file used for the given build artifact + + Args: + Path: file path + + Returns: + The path to .cmd file + """ + dir, base = os.path.split(path) + return os.path.join(dir, '.' + base + '.cmd') + + +def cmdfiles_for_o(obj): + """Generate the iterator of .cmd files associated with the object + + Yield the .cmd file used to build the given object + + Args: + obj: The object path + + Yields: + The path to .cmd file + """ + yield to_cmdfile(obj) + + +def cmdfiles_for_a(archive, ar): + """Generate the iterator of .cmd files associated with the archive. + + Parse the given archive, and yield every .cmd file used to build it. + + Args: + archive: The archive to parse + + Yields: + The path to every .cmd file found + """ + for obj in subprocess.check_output([ar, '-t', archive]).decode().split(): + yield to_cmdfile(obj) + + +def cmdfiles_for_modorder(modorder): + """Generate the iterator of .cmd files associated with the modules.order. + + Parse the given modules.order, and yield every .cmd file used to build the + contained modules. + + Args: + modorder: The modules.order file to parse + + Yields: + The path to every .cmd file found + """ + with open(modorder) as f: + for line in f: + ko = line.rstrip() + base, ext = os.path.splitext(ko) + if ext != '.ko': + sys.exit('{}: module path must end with .ko'.format(ko)) + mod = base + '.mod' + # The first line of *.mod lists the objects that compose the module. + with open(mod) as m: + for obj in m.readline().split(): + yield to_cmdfile(obj) + + def process_line(root_directory, command_prefix, file_path): """Extracts information from a .cmd line and creates an entry from it. @@ -117,7 +195,7 @@ def process_line(root_directory, command_prefix, file_path): def main(): """Walks through the directory and finds and parses .cmd files.""" - log_level, directory, output, paths = parse_arguments() + log_level, directory, output, ar, paths = parse_arguments() level = getattr(logging, log_level) logging.basicConfig(format='%(levelname)s: %(message)s', level=level) @@ -127,7 +205,21 @@ def main(): compile_commands = [] for path in paths: - cmdfiles = cmdfiles_in_dir(path) + # If 'path' is a directory, handle all .cmd files under it. + # Otherwise, handle .cmd files associated with the file. + # Most of built-in objects are linked via archives (built-in.a or lib.a) + # but some objects are linked to vmlinux directly. + # Modules are listed in modules.order. + if os.path.isdir(path): + cmdfiles = cmdfiles_in_dir(path) + elif path.endswith('.o'): + cmdfiles = cmdfiles_for_o(path) + elif path.endswith('.a'): + cmdfiles = cmdfiles_for_a(path, ar) + elif path.endswith('modules.order'): + cmdfiles = cmdfiles_for_modorder(path) + else: + sys.exit('{}: unknown file type'.format(path)) for cmdfile in cmdfiles: with open(cmdfile, 'rt') as f: -- cgit v1.2.3 From 8b61f748e2a022f9028c612e44b68ad61caec474 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 22 Aug 2020 23:56:17 +0900 Subject: gen_compile_commands: remove the warning about too few .cmd files This warning was useful when users previously needed to manually build the kernel and run this script. Now you can simply do 'make compile_commands.json', which updates all the necessary build artifacts and automatically creates the compilation database. There is no more worry for a mistake like "Oh, I forgot to build the kernel". Now, this warning is rather annoying. You can create compile_commands.json for an external module: $ make M=/path/to/your/external/module compile_commands.json Then, this warning is displayed since there are usually less than 300 files in a single module. Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers --- scripts/gen_compile_commands.py | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'scripts') diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 92ea0a2ee59e..19963708bcf8 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -21,11 +21,6 @@ _FILENAME_PATTERN = r'^\..*\.cmd$' _LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$' _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] -# A kernel build generally has over 2000 entries in its compile_commands.json -# database. If this code finds 300 or fewer, then warn the user that they might -# not have all the .cmd files, and they might need to compile the kernel. -_LOW_COUNT_THRESHOLD = 300 - def parse_arguments(): """Sets up and parses command-line arguments. @@ -236,11 +231,6 @@ def main(): with open(output, 'wt') as f: json.dump(compile_commands, f, indent=2, sort_keys=True) - count = len(compile_commands) - if count < _LOW_COUNT_THRESHOLD: - logging.warning( - 'Found %s entries. Have you compiled the kernel?', count) - if __name__ == '__main__': main() -- cgit v1.2.3 From 6ad7cbc01527223f3f92baac9b122f15651cf76b Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Sat, 22 Aug 2020 23:56:18 +0900 Subject: Makefile: Add clang-tidy and static analyzer support to makefile This patch adds clang-tidy and the clang static-analyzer as make targets. The goal of this patch is to make static analysis tools usable and extendable by any developer or researcher who is familiar with basic c++. The current static analysis tools require intimate knowledge of the internal workings of the static analysis. Clang-tidy and the clang static analyzers expose an easy to use api and allow users unfamiliar with clang to write new checks with relative ease. ===Clang-tidy=== Clang-tidy is an easily extendable 'linter' that runs on the AST. Clang-tidy checks are easy to write and understand. A check consists of two parts, a matcher and a checker. The matcher is created using a domain specific language that acts on the AST (https://clang.llvm.org/docs/LibASTMatchersReference.html). When AST nodes are found by the matcher a callback is made to the checker. The checker can then execute additional checks and issue warnings. Here is an example clang-tidy check to report functions that have calls to local_irq_disable without calls to local_irq_enable and vice-versa. Functions flagged with __attribute((annotation("ignore_irq_balancing"))) are ignored for analysis. (https://reviews.llvm.org/D65828) ===Clang static analyzer=== The clang static analyzer is a more powerful static analysis tool that uses symbolic execution to find bugs. Currently there is a check that looks for potential security bugs from invalid uses of kmalloc and kfree. There are several more general purpose checks that are useful for the kernel. The clang static analyzer is well documented and designed to be extensible. (https://clang-analyzer.llvm.org/checker_dev_manual.html) (https://github.com/haoNoQ/clang-analyzer-guide/releases/download/v0.1/clang-analyzer-guide-v0.1.pdf) The main draw of the clang tools is how accessible they are. The clang documentation is very nice and these tools are built specifically to be easily extendable by any developer. They provide an accessible method of bug-finding and research to people who are not overly familiar with the kernel codebase. Signed-off-by: Nathan Huckleberry Reviewed-by: Nick Desaulniers Tested-by: Nick Desaulniers Tested-by: Lukas Bulwahn Signed-off-by: Masahiro Yamada --- scripts/clang-tools/gen_compile_commands.py | 236 ++++++++++++++++++++++++++++ scripts/clang-tools/run-clang-tools.py | 74 +++++++++ scripts/gen_compile_commands.py | 236 ---------------------------- 3 files changed, 310 insertions(+), 236 deletions(-) create mode 100755 scripts/clang-tools/gen_compile_commands.py create mode 100755 scripts/clang-tools/run-clang-tools.py delete mode 100755 scripts/gen_compile_commands.py (limited to 'scripts') diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py new file mode 100755 index 000000000000..19963708bcf8 --- /dev/null +++ b/scripts/clang-tools/gen_compile_commands.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) Google LLC, 2018 +# +# Author: Tom Roeder +# +"""A tool for generating compile_commands.json in the Linux kernel.""" + +import argparse +import json +import logging +import os +import re +import subprocess + +_DEFAULT_OUTPUT = 'compile_commands.json' +_DEFAULT_LOG_LEVEL = 'WARNING' + +_FILENAME_PATTERN = r'^\..*\.cmd$' +_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$' +_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] + + +def parse_arguments(): + """Sets up and parses command-line arguments. + + Returns: + log_level: A logging level to filter log output. + directory: The work directory where the objects were built. + ar: Command used for parsing .a archives. + output: Where to write the compile-commands JSON file. + paths: The list of files/directories to handle to find .cmd files. + """ + usage = 'Creates a compile_commands.json database from kernel .cmd files' + parser = argparse.ArgumentParser(description=usage) + + directory_help = ('specify the output directory used for the kernel build ' + '(defaults to the working directory)') + parser.add_argument('-d', '--directory', type=str, default='.', + help=directory_help) + + output_help = ('path to the output command database (defaults to ' + + _DEFAULT_OUTPUT + ')') + parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT, + help=output_help) + + log_level_help = ('the level of log messages to produce (defaults to ' + + _DEFAULT_LOG_LEVEL + ')') + parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS, + default=_DEFAULT_LOG_LEVEL, help=log_level_help) + + ar_help = 'command used for parsing .a archives' + parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help) + + paths_help = ('directories to search or files to parse ' + '(files should be *.o, *.a, or modules.order). ' + 'If nothing is specified, the current directory is searched') + parser.add_argument('paths', type=str, nargs='*', help=paths_help) + + args = parser.parse_args() + + return (args.log_level, + os.path.abspath(args.directory), + args.output, + args.ar, + args.paths if len(args.paths) > 0 else [args.directory]) + + +def cmdfiles_in_dir(directory): + """Generate the iterator of .cmd files found under the directory. + + Walk under the given directory, and yield every .cmd file found. + + Args: + directory: The directory to search for .cmd files. + + Yields: + The path to a .cmd file. + """ + + filename_matcher = re.compile(_FILENAME_PATTERN) + + for dirpath, _, filenames in os.walk(directory): + for filename in filenames: + if filename_matcher.match(filename): + yield os.path.join(dirpath, filename) + + +def to_cmdfile(path): + """Return the path of .cmd file used for the given build artifact + + Args: + Path: file path + + Returns: + The path to .cmd file + """ + dir, base = os.path.split(path) + return os.path.join(dir, '.' + base + '.cmd') + + +def cmdfiles_for_o(obj): + """Generate the iterator of .cmd files associated with the object + + Yield the .cmd file used to build the given object + + Args: + obj: The object path + + Yields: + The path to .cmd file + """ + yield to_cmdfile(obj) + + +def cmdfiles_for_a(archive, ar): + """Generate the iterator of .cmd files associated with the archive. + + Parse the given archive, and yield every .cmd file used to build it. + + Args: + archive: The archive to parse + + Yields: + The path to every .cmd file found + """ + for obj in subprocess.check_output([ar, '-t', archive]).decode().split(): + yield to_cmdfile(obj) + + +def cmdfiles_for_modorder(modorder): + """Generate the iterator of .cmd files associated with the modules.order. + + Parse the given modules.order, and yield every .cmd file used to build the + contained modules. + + Args: + modorder: The modules.order file to parse + + Yields: + The path to every .cmd file found + """ + with open(modorder) as f: + for line in f: + ko = line.rstrip() + base, ext = os.path.splitext(ko) + if ext != '.ko': + sys.exit('{}: module path must end with .ko'.format(ko)) + mod = base + '.mod' + # The first line of *.mod lists the objects that compose the module. + with open(mod) as m: + for obj in m.readline().split(): + yield to_cmdfile(obj) + + +def process_line(root_directory, command_prefix, file_path): + """Extracts information from a .cmd line and creates an entry from it. + + Args: + root_directory: The directory that was searched for .cmd files. Usually + used directly in the "directory" entry in compile_commands.json. + command_prefix: The extracted command line, up to the last element. + file_path: The .c file from the end of the extracted command. + Usually relative to root_directory, but sometimes absolute. + + Returns: + An entry to append to compile_commands. + + Raises: + ValueError: Could not find the extracted file based on file_path and + root_directory or file_directory. + """ + # The .cmd files are intended to be included directly by Make, so they + # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the + # kernel version). The compile_commands.json file is not interepreted + # by Make, so this code replaces the escaped version with '#'. + prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#') + + # Use os.path.abspath() to normalize the path resolving '.' and '..' . + abs_path = os.path.abspath(os.path.join(root_directory, file_path)) + if not os.path.exists(abs_path): + raise ValueError('File %s not found' % abs_path) + return { + 'directory': root_directory, + 'file': abs_path, + 'command': prefix + file_path, + } + + +def main(): + """Walks through the directory and finds and parses .cmd files.""" + log_level, directory, output, ar, paths = parse_arguments() + + level = getattr(logging, log_level) + logging.basicConfig(format='%(levelname)s: %(message)s', level=level) + + line_matcher = re.compile(_LINE_PATTERN) + + compile_commands = [] + + for path in paths: + # If 'path' is a directory, handle all .cmd files under it. + # Otherwise, handle .cmd files associated with the file. + # Most of built-in objects are linked via archives (built-in.a or lib.a) + # but some objects are linked to vmlinux directly. + # Modules are listed in modules.order. + if os.path.isdir(path): + cmdfiles = cmdfiles_in_dir(path) + elif path.endswith('.o'): + cmdfiles = cmdfiles_for_o(path) + elif path.endswith('.a'): + cmdfiles = cmdfiles_for_a(path, ar) + elif path.endswith('modules.order'): + cmdfiles = cmdfiles_for_modorder(path) + else: + sys.exit('{}: unknown file type'.format(path)) + + for cmdfile in cmdfiles: + with open(cmdfile, 'rt') as f: + result = line_matcher.match(f.readline()) + if result: + try: + entry = process_line(directory, result.group(1), + result.group(2)) + compile_commands.append(entry) + except ValueError as err: + logging.info('Could not add line from %s: %s', + cmdfile, err) + + with open(output, 'wt') as f: + json.dump(compile_commands, f, indent=2, sort_keys=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/clang-tools/run-clang-tools.py b/scripts/clang-tools/run-clang-tools.py new file mode 100755 index 000000000000..fa7655c7cec0 --- /dev/null +++ b/scripts/clang-tools/run-clang-tools.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) Google LLC, 2020 +# +# Author: Nathan Huckleberry +# +"""A helper routine run clang-tidy and the clang static-analyzer on +compile_commands.json. +""" + +import argparse +import json +import multiprocessing +import os +import subprocess +import sys + + +def parse_arguments(): + """Set up and parses command-line arguments. + Returns: + args: Dict of parsed args + Has keys: [path, type] + """ + usage = """Run clang-tidy or the clang static-analyzer on a + compilation database.""" + parser = argparse.ArgumentParser(description=usage) + + type_help = "Type of analysis to be performed" + parser.add_argument("type", + choices=["clang-tidy", "clang-analyzer"], + help=type_help) + path_help = "Path to the compilation database to parse" + parser.add_argument("path", type=str, help=path_help) + + return parser.parse_args() + + +def init(l, a): + global lock + global args + lock = l + args = a + + +def run_analysis(entry): + # Disable all checks, then re-enable the ones we want + checks = "-checks=-*," + if args.type == "clang-tidy": + checks += "linuxkernel-*" + else: + checks += "clang-analyzer-*" + p = subprocess.run(["clang-tidy", "-p", args.path, checks, entry["file"]], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=entry["directory"]) + with lock: + sys.stderr.buffer.write(p.stdout) + + +def main(): + args = parse_arguments() + + lock = multiprocessing.Lock() + pool = multiprocessing.Pool(initializer=init, initargs=(lock, args)) + # Read JSON data into the datastore variable + with open(args.path, "r") as f: + datastore = json.load(f) + pool.map(run_analysis, datastore) + + +if __name__ == "__main__": + main() diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py deleted file mode 100755 index 19963708bcf8..000000000000 --- a/scripts/gen_compile_commands.py +++ /dev/null @@ -1,236 +0,0 @@ -#!/usr/bin/env python -# SPDX-License-Identifier: GPL-2.0 -# -# Copyright (C) Google LLC, 2018 -# -# Author: Tom Roeder -# -"""A tool for generating compile_commands.json in the Linux kernel.""" - -import argparse -import json -import logging -import os -import re -import subprocess - -_DEFAULT_OUTPUT = 'compile_commands.json' -_DEFAULT_LOG_LEVEL = 'WARNING' - -_FILENAME_PATTERN = r'^\..*\.cmd$' -_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$' -_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] - - -def parse_arguments(): - """Sets up and parses command-line arguments. - - Returns: - log_level: A logging level to filter log output. - directory: The work directory where the objects were built. - ar: Command used for parsing .a archives. - output: Where to write the compile-commands JSON file. - paths: The list of files/directories to handle to find .cmd files. - """ - usage = 'Creates a compile_commands.json database from kernel .cmd files' - parser = argparse.ArgumentParser(description=usage) - - directory_help = ('specify the output directory used for the kernel build ' - '(defaults to the working directory)') - parser.add_argument('-d', '--directory', type=str, default='.', - help=directory_help) - - output_help = ('path to the output command database (defaults to ' + - _DEFAULT_OUTPUT + ')') - parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT, - help=output_help) - - log_level_help = ('the level of log messages to produce (defaults to ' + - _DEFAULT_LOG_LEVEL + ')') - parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS, - default=_DEFAULT_LOG_LEVEL, help=log_level_help) - - ar_help = 'command used for parsing .a archives' - parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help) - - paths_help = ('directories to search or files to parse ' - '(files should be *.o, *.a, or modules.order). ' - 'If nothing is specified, the current directory is searched') - parser.add_argument('paths', type=str, nargs='*', help=paths_help) - - args = parser.parse_args() - - return (args.log_level, - os.path.abspath(args.directory), - args.output, - args.ar, - args.paths if len(args.paths) > 0 else [args.directory]) - - -def cmdfiles_in_dir(directory): - """Generate the iterator of .cmd files found under the directory. - - Walk under the given directory, and yield every .cmd file found. - - Args: - directory: The directory to search for .cmd files. - - Yields: - The path to a .cmd file. - """ - - filename_matcher = re.compile(_FILENAME_PATTERN) - - for dirpath, _, filenames in os.walk(directory): - for filename in filenames: - if filename_matcher.match(filename): - yield os.path.join(dirpath, filename) - - -def to_cmdfile(path): - """Return the path of .cmd file used for the given build artifact - - Args: - Path: file path - - Returns: - The path to .cmd file - """ - dir, base = os.path.split(path) - return os.path.join(dir, '.' + base + '.cmd') - - -def cmdfiles_for_o(obj): - """Generate the iterator of .cmd files associated with the object - - Yield the .cmd file used to build the given object - - Args: - obj: The object path - - Yields: - The path to .cmd file - """ - yield to_cmdfile(obj) - - -def cmdfiles_for_a(archive, ar): - """Generate the iterator of .cmd files associated with the archive. - - Parse the given archive, and yield every .cmd file used to build it. - - Args: - archive: The archive to parse - - Yields: - The path to every .cmd file found - """ - for obj in subprocess.check_output([ar, '-t', archive]).decode().split(): - yield to_cmdfile(obj) - - -def cmdfiles_for_modorder(modorder): - """Generate the iterator of .cmd files associated with the modules.order. - - Parse the given modules.order, and yield every .cmd file used to build the - contained modules. - - Args: - modorder: The modules.order file to parse - - Yields: - The path to every .cmd file found - """ - with open(modorder) as f: - for line in f: - ko = line.rstrip() - base, ext = os.path.splitext(ko) - if ext != '.ko': - sys.exit('{}: module path must end with .ko'.format(ko)) - mod = base + '.mod' - # The first line of *.mod lists the objects that compose the module. - with open(mod) as m: - for obj in m.readline().split(): - yield to_cmdfile(obj) - - -def process_line(root_directory, command_prefix, file_path): - """Extracts information from a .cmd line and creates an entry from it. - - Args: - root_directory: The directory that was searched for .cmd files. Usually - used directly in the "directory" entry in compile_commands.json. - command_prefix: The extracted command line, up to the last element. - file_path: The .c file from the end of the extracted command. - Usually relative to root_directory, but sometimes absolute. - - Returns: - An entry to append to compile_commands. - - Raises: - ValueError: Could not find the extracted file based on file_path and - root_directory or file_directory. - """ - # The .cmd files are intended to be included directly by Make, so they - # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the - # kernel version). The compile_commands.json file is not interepreted - # by Make, so this code replaces the escaped version with '#'. - prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#') - - # Use os.path.abspath() to normalize the path resolving '.' and '..' . - abs_path = os.path.abspath(os.path.join(root_directory, file_path)) - if not os.path.exists(abs_path): - raise ValueError('File %s not found' % abs_path) - return { - 'directory': root_directory, - 'file': abs_path, - 'command': prefix + file_path, - } - - -def main(): - """Walks through the directory and finds and parses .cmd files.""" - log_level, directory, output, ar, paths = parse_arguments() - - level = getattr(logging, log_level) - logging.basicConfig(format='%(levelname)s: %(message)s', level=level) - - line_matcher = re.compile(_LINE_PATTERN) - - compile_commands = [] - - for path in paths: - # If 'path' is a directory, handle all .cmd files under it. - # Otherwise, handle .cmd files associated with the file. - # Most of built-in objects are linked via archives (built-in.a or lib.a) - # but some objects are linked to vmlinux directly. - # Modules are listed in modules.order. - if os.path.isdir(path): - cmdfiles = cmdfiles_in_dir(path) - elif path.endswith('.o'): - cmdfiles = cmdfiles_for_o(path) - elif path.endswith('.a'): - cmdfiles = cmdfiles_for_a(path, ar) - elif path.endswith('modules.order'): - cmdfiles = cmdfiles_for_modorder(path) - else: - sys.exit('{}: unknown file type'.format(path)) - - for cmdfile in cmdfiles: - with open(cmdfile, 'rt') as f: - result = line_matcher.match(f.readline()) - if result: - try: - entry = process_line(directory, result.group(1), - result.group(2)) - compile_commands.append(entry) - except ValueError as err: - logging.info('Could not add line from %s: %s', - cmdfile, err) - - with open(output, 'wt') as f: - json.dump(compile_commands, f, indent=2, sort_keys=True) - - -if __name__ == '__main__': - main() -- cgit v1.2.3 From 4fcadb39c6b298e105d827741e0b7a6b146633e5 Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Wed, 5 Aug 2020 11:14:38 -0700 Subject: spelling.txt: Remove some duplicate entries Found when Colin King fixed a typo for falied/failed and a git grep showed 2 entries in this file. Signed-off-by: Joe Perches Acked-by: Colin Ian King Signed-off-by: Jiri Kosina --- scripts/spelling.txt | 4 ---- 1 file changed, 4 deletions(-) (limited to 'scripts') diff --git a/scripts/spelling.txt b/scripts/spelling.txt index f253681e7e2a..b703e4fd51e8 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -596,7 +596,6 @@ extenstion||extension extracter||extractor faied||failed faield||failed -falied||failed faild||failed failded||failed failer||failure @@ -793,7 +792,6 @@ interrup||interrupt interrups||interrupts interruptted||interrupted interupted||interrupted -interupt||interrupt intial||initial intialisation||initialisation intialised||initialised @@ -968,7 +966,6 @@ occurd||occurred occured||occurred occurence||occurrence occure||occurred -occured||occurred occuring||occurring offser||offset offet||offset @@ -1437,7 +1434,6 @@ udpate||update uesd||used uknown||unknown usccess||success -usupported||unsupported uncommited||uncommitted uncompatible||incompatible unconditionaly||unconditionally -- cgit v1.2.3 From 5e0c074e5b4be02d57d1b60abc3391afe7edd088 Mon Sep 17 00:00:00 2001 From: Denis Efremov Date: Wed, 2 Sep 2020 07:40:59 +0300 Subject: coccinelle: ifnullfree: add vfree(), kvfree*() functions Extend the list of free functions with kvfree(), kvfree_sensitive(), vfree(). Signed-off-by: Denis Efremov Signed-off-by: Julia Lawall --- scripts/coccinelle/free/ifnullfree.cocci | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/coccinelle/free/ifnullfree.cocci b/scripts/coccinelle/free/ifnullfree.cocci index 2045391e36a0..285b92d5c665 100644 --- a/scripts/coccinelle/free/ifnullfree.cocci +++ b/scripts/coccinelle/free/ifnullfree.cocci @@ -20,8 +20,14 @@ expression E; - if (E != NULL) ( kfree(E); +| + kvfree(E); | kfree_sensitive(E); +| + kvfree_sensitive(E, ...); +| + vfree(E); | debugfs_remove(E); | @@ -42,9 +48,10 @@ position p; @@ * if (E != NULL) -* \(kfree@p\|kfree_sensitive@p\|debugfs_remove@p\|debugfs_remove_recursive@p\| +* \(kfree@p\|kvfree@p\|kfree_sensitive@p\|kvfree_sensitive@p\|vfree@p\| +* debugfs_remove@p\|debugfs_remove_recursive@p\| * usb_free_urb@p\|kmem_cache_destroy@p\|mempool_destroy@p\| -* dma_pool_destroy@p\)(E); +* dma_pool_destroy@p\)(E, ...); @script:python depends on org@ p << r.p; -- cgit v1.2.3 From 7c9dc603d55617edcc3190f9a6d7b5cf763feb57 Mon Sep 17 00:00:00 2001 From: Denis Efremov Date: Tue, 1 Sep 2020 12:48:12 +0300 Subject: coccinelle: misc: add uninitialized_var.cocci script uninitialized_var() macro was removed from the sources [1] and other warning-silencing tricks were deprecated [2]. The purpose of this cocci script is to prevent new occurrences of uninitialized_var() open-coded variants. [1] commit 63a0895d960a ("compiler: Remove uninitialized_var() macro") [2] commit 4b19bec97c88 ("docs: deprecated.rst: Add uninitialized_var()") Cc: Kees Cook Cc: Gustavo A. R. Silva Signed-off-by: Denis Efremov Signed-off-by: Julia Lawall --- scripts/coccinelle/misc/uninitialized_var.cocci | 51 +++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 scripts/coccinelle/misc/uninitialized_var.cocci (limited to 'scripts') diff --git a/scripts/coccinelle/misc/uninitialized_var.cocci b/scripts/coccinelle/misc/uninitialized_var.cocci new file mode 100644 index 000000000000..8fa845cefe11 --- /dev/null +++ b/scripts/coccinelle/misc/uninitialized_var.cocci @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only +/// +/// Please, don't reintroduce uninitialized_var(). +/// From Documentation/process/deprecated.rst: +/// For any compiler warnings about uninitialized variables, just add +/// an initializer. Using warning-silencing tricks is dangerous as it +/// papers over real bugs (or can in the future), and suppresses unrelated +/// compiler warnings (e.g. "unused variable"). If the compiler thinks it +/// is uninitialized, either simply initialize the variable or make compiler +/// changes. Keep in mind that in most cases, if an initialization is +/// obviously redundant, the compiler's dead-store elimination pass will make +/// sure there are no needless variable writes. +/// +// Confidence: High +// Copyright: (C) 2020 Denis Efremov ISPRAS +// Options: --no-includes --include-headers +// + +virtual context +virtual report +virtual org + +@r@ +identifier var; +type T; +position p; +@@ + +( +* T var =@p var; +| +* T var =@p *(&(var)); +| +* var =@p var +| +* var =@p *(&(var)) +) + +@script:python depends on report@ +p << r.p; +@@ + +coccilib.report.print_report(p[0], + "WARNING this kind of initialization is deprecated (https://www.kernel.org/doc/html/latest/process/deprecated.html#uninitialized-var)") + +@script:python depends on org@ +p << r.p; +@@ + +coccilib.org.print_todo(p[0], + "WARNING this kind of initialization is deprecated (https://www.kernel.org/doc/html/latest/process/deprecated.html#uninitialized-var)") -- cgit v1.2.3 From 6519a5ab1a9ffe5cf8056f688a69960bf126e723 Mon Sep 17 00:00:00 2001 From: Denis Efremov Date: Tue, 11 Aug 2020 10:49:53 +0300 Subject: coccinelle: api: update kzfree script to kfree_sensitive Commit 453431a54934 ("mm, treewide: rename kzfree() to kfree_sensitive()") renames kzfree to kfree_sensitive and uses memzero_explicit(...) instead of memset(..., 0, ...) internally. Update cocci script to reflect these changes. Signed-off-by: Denis Efremov Signed-off-by: Julia Lawall --- scripts/coccinelle/api/kfree_sensitive.cocci | 98 ++++++++++++++++++++++++++ scripts/coccinelle/api/kzfree.cocci | 101 --------------------------- 2 files changed, 98 insertions(+), 101 deletions(-) create mode 100644 scripts/coccinelle/api/kfree_sensitive.cocci delete mode 100644 scripts/coccinelle/api/kzfree.cocci (limited to 'scripts') diff --git a/scripts/coccinelle/api/kfree_sensitive.cocci b/scripts/coccinelle/api/kfree_sensitive.cocci new file mode 100644 index 000000000000..e4a066a0b77d --- /dev/null +++ b/scripts/coccinelle/api/kfree_sensitive.cocci @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only +/// +/// Use kfree_sensitive, kvfree_sensitive rather than memset or +/// memzero_explicit followed by kfree. +/// +// Confidence: High +// Copyright: (C) 2020 Denis Efremov ISPRAS +// Options: --no-includes --include-headers +// +// Keywords: kfree_sensitive, kvfree_sensitive +// + +virtual context +virtual patch +virtual org +virtual report + +@initialize:python@ +@@ +# kmalloc_oob_in_memset uses memset to explicitly trigger out-of-bounds access +filter = frozenset(['kmalloc_oob_in_memset', + 'kfree_sensitive', 'kvfree_sensitive']) + +def relevant(p): + return not (filter & {el.current_element for el in p}) + +@cond@ +position ok; +@@ + +if (...) + \(memset@ok\|memzero_explicit@ok\)(...); + +@r depends on !patch forall@ +expression E; +position p : script:python() { relevant(p) }; +position m != cond.ok; +type T; +@@ + +( +* memset@m((T)E, 0, ...); +| +* memzero_explicit@m((T)E, ...); +) + ... when != E + when strict +* \(kfree\|vfree\|kvfree\)(E)@p; + +@rp_memzero depends on patch@ +expression E, size; +position p : script:python() { relevant(p) }; +position m != cond.ok; +type T; +@@ + +- memzero_explicit@m(