From 8962b6b475bddc011c414f40ffd02f0ed4e02771 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 23 Dec 2022 01:25:32 +0900 Subject: kbuild: print short log in addition to the whole command with V=1 "make V=1" prints the whole command instead of the short log, but I think it is nicer to print both so that you can easily spot the build rule of your interest. This commit changes V=1 to print the short log (the line starts with '#'), followed by the full log. In parallel builds, the short/full logs from the same build rule may be interspersed. If you want to avoid it, please add -Otarget option. Kbuild will never set it by default because Make would buffer the logs and lose the escape sequences. (Modern compilers print warnings and errors in color, but only when they write to a terminal.) This is also a preparation for supporting V=12 because V=2 appends the reason for rebuilding to the short log. Signed-off-by: Masahiro Yamada Tested-by: Nicolas Schier Reviewed-by: Nicolas Schier --- scripts/Kbuild.include | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'scripts') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 2f7356b2990b..faae3c7986b0 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -125,15 +125,15 @@ dtbinst := -f $(srctree)/scripts/Makefile.dtbinst obj # $(Q)$(MAKE) $(clean)=dir clean := -f $(srctree)/scripts/Makefile.clean obj -# echo command. -# Short version is used, if $(quiet) equals `quiet_', otherwise full one. -echo-cmd = $(if $($(quiet)cmd_$(1)),\ - echo ' $(call escsq,$($(quiet)cmd_$(1)))$(echo-why)';) - -# sink stdout for 'make -s' - redirect := - quiet_redirect := -silent_redirect := exec >/dev/null; +# pring log +# +# If quiet is "silent_", print nothing and sink stdout +# If quiet is "quiet_", print short log +# If quiet is empty, print short log and whole command +silent_log_print = exec >/dev/null; + quiet_log_print = $(if $(quiet_cmd_$1), echo ' $(call escsq,$(quiet_cmd_$1)$(why))';) + log_print = echo '$(pound) $(call escsq,$(or $(quiet_cmd_$1),cmd_$1 $@))'; \ + echo ' $(call escsq,$(cmd_$1))'; # Delete the target on interruption # @@ -156,8 +156,8 @@ delete-on-interrupt = \ $(foreach sig, HUP INT QUIT TERM PIPE, \ trap 'rm -f $@; trap - $(sig); kill -s $(sig) $$$$' $(sig);)) -# printing commands -cmd = @set -e; $(echo-cmd) $($(quiet)redirect) $(delete-on-interrupt) $(cmd_$(1)) +# print and execute commands +cmd = @$(if $(cmd_$(1)),set -e; $($(quiet)log_print) $(delete-on-interrupt) $(cmd_$(1)),:) ### # if_changed - execute command if any prerequisite is newer than @@ -234,7 +234,7 @@ if_changed_rule = $(if $(if-changed-cond),$(rule_$(1)),@:) # (6) No dir/.target.cmd file and target not listed in $(targets) # This is a good hint that there is a bug in the kbuild file ifeq ($(KBUILD_VERBOSE),2) -why = \ +_why = \ $(if $(filter $@, $(PHONY)),- due to target is PHONY, \ $(if $(wildcard $@), \ $(if $(newer-prereqs),- due to: $(newer-prereqs), \ @@ -251,7 +251,7 @@ why = \ ) \ ) -echo-why = $(call escsq, $(strip $(why))) +why = $(space)$(strip $(_why)) endif ############################################################################### -- cgit v1.2.3 From c0d3b83100c896e1b0909023df58a0ebdd428d61 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 23 Dec 2022 01:25:33 +0900 Subject: kbuild: do not print extra logs for V=2 Some scripts increase the verbose level when V=1, but others when not V=0. I think the former is correct because V=2 is not a log level but a switch to print the reason for rebuilding. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/asn1_compiler.c | 4 ++-- scripts/kernel-doc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/asn1_compiler.c b/scripts/asn1_compiler.c index 71d4a7c87900..7b6756a8c15d 100644 --- a/scripts/asn1_compiler.c +++ b/scripts/asn1_compiler.c @@ -567,8 +567,8 @@ int main(int argc, char **argv) int fd; kbuild_verbose = getenv("KBUILD_VERBOSE"); - if (kbuild_verbose) - verbose_opt = atoi(kbuild_verbose); + if (kbuild_verbose && strchr(kbuild_verbose, '1')) + verbose_opt = true; while (argc > 4) { if (strcmp(argv[1], "-v") == 0) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 54b0893cae66..8ad0a7d68d9a 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -175,8 +175,8 @@ my $declaration_start_line; my ($type, $declaration_name, $return_type); my ($newsection, $newcontents, $prototype, $brcount, %source_map); -if (defined($ENV{'KBUILD_VERBOSE'})) { - $verbose = "$ENV{'KBUILD_VERBOSE'}"; +if (defined($ENV{'KBUILD_VERBOSE'}) && $ENV{'KBUILD_VERBOSE'} =~ '1') { + $verbose = 1; } if (defined($ENV{'KCFLAGS'})) { -- cgit v1.2.3 From 6ae4b9868a8f723cae2600722eea033fafadd399 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 23 Dec 2022 01:25:34 +0900 Subject: kbuild: allow to combine multiple V= levels Commit a6de553da01c ("kbuild: Allow to combine multiple W= levels") supported W=123 to enable all the extra warning groups. I think a similar idea is applicable to the V= option. V=1 echos the whole command V=2 prints the reason for rebuilding These are orthogonal, and can be enabled at the same time. This commit supports V=12 to enable both of them. Signed-off-by: Masahiro Yamada Tested-by: Nicolas Schier Reviewed-by: Nicolas Schier --- scripts/Kbuild.include | 4 ++-- scripts/tags.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index faae3c7986b0..10cf8d2d82ef 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -132,7 +132,7 @@ clean := -f $(srctree)/scripts/Makefile.clean obj # If quiet is empty, print short log and whole command silent_log_print = exec >/dev/null; quiet_log_print = $(if $(quiet_cmd_$1), echo ' $(call escsq,$(quiet_cmd_$1)$(why))';) - log_print = echo '$(pound) $(call escsq,$(or $(quiet_cmd_$1),cmd_$1 $@))'; \ + log_print = echo '$(pound) $(call escsq,$(or $(quiet_cmd_$1),cmd_$1 $@)$(why))'; \ echo ' $(call escsq,$(cmd_$1))'; # Delete the target on interruption @@ -233,7 +233,7 @@ if_changed_rule = $(if $(if-changed-cond),$(rule_$(1)),@:) # (5) No dir/.target.cmd file (used to store command line) # (6) No dir/.target.cmd file and target not listed in $(targets) # This is a good hint that there is a bug in the kbuild file -ifeq ($(KBUILD_VERBOSE),2) +ifneq ($(findstring 2, $(KBUILD_VERBOSE)),) _why = \ $(if $(filter $@, $(PHONY)),- due to target is PHONY, \ $(if $(wildcard $@), \ diff --git a/scripts/tags.sh b/scripts/tags.sh index e137cf15aae9..2e756bee1fa9 100755 --- a/scripts/tags.sh +++ b/scripts/tags.sh @@ -8,7 +8,7 @@ # Uses the following environment variables: # SUBARCH, SRCARCH, srctree -if [ "$KBUILD_VERBOSE" = "1" ]; then +if [[ "$KBUILD_VERBOSE" =~ 1 ]]; then set -x fi -- cgit v1.2.3 From 8f99eb857a96d474bfbb715ff9073b276b87ad2d Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 29 Dec 2022 16:06:33 +0900 Subject: kbuild: clean up stale file removal More than one year has passed since the copied *.[cS] files were removed from arch/*/boot/compressed/. Signed-off-by: Masahiro Yamada --- scripts/remove-stale-files | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) (limited to 'scripts') diff --git a/scripts/remove-stale-files b/scripts/remove-stale-files index 64b14aa5aebf..cdbdde89a271 100755 --- a/scripts/remove-stale-files +++ b/scripts/remove-stale-files @@ -21,31 +21,9 @@ set -e # then will be really dead and removed from the code base entirely. rm -f arch/powerpc/purgatory/kexec-purgatory.c - -# These were previously generated source files. When you are building the kernel -# with O=, make sure to remove the stale files in the output tree. Otherwise, -# the build system wrongly compiles the stale ones. -if [ -n "${building_out_of_srctree}" ]; then - for f in fdt_rw.c fdt_ro.c fdt_wip.c fdt.c ashldi3.S bswapsdi2.S font.c lib1funcs.S hyp-stub.S - do - rm -f arch/arm/boot/compressed/${f} - done - - for f in uart-ath79.c ashldi3.c bswapdi.c bswapsi.c - do - rm -f arch/mips/boot/compressed/${f} - done - - for f in firmware.c real2.S - do - rm -f arch/parisc/boot/compressed/${f} - done -fi - rm -f arch/riscv/purgatory/kexec-purgatory.c +rm -f arch/x86/purgatory/kexec-purgatory.c rm -f scripts/extract-cert -rm -f arch/x86/purgatory/kexec-purgatory.c - rm -f scripts/kconfig/[gmnq]conf-cfg -- cgit v1.2.3 From 91ecf7ff1b036f3fe1183809661119b1ee109b19 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 29 Dec 2022 16:43:10 +0900 Subject: kbuild: make W=1 warn files that are tracked but ignored by git The top .gitignore comments about how to detect files breaking .gitignore rules, but people rarely care about it. Add a new W=1 warning to detect files that are tracked but ignored by git. If git is not installed or the source tree is not tracked by git at all, this script does not print anything. Running it on v6.2-rc1 detected the following: $ make W=1 misc-check Documentation/devicetree/bindings/.yamllint: warning: ignored by one of the .gitignore files drivers/clk/.kunitconfig: warning: ignored by one of the .gitignore files drivers/gpu/drm/tests/.kunitconfig: warning: ignored by one of the .gitignore files drivers/hid/.kunitconfig: warning: ignored by one of the .gitignore files fs/ext4/.kunitconfig: warning: ignored by one of the .gitignore files fs/fat/.kunitconfig: warning: ignored by one of the .gitignore files kernel/kcsan/.kunitconfig: warning: ignored by one of the .gitignore files lib/kunit/.kunitconfig: warning: ignored by one of the .gitignore files mm/kfence/.kunitconfig: warning: ignored by one of the .gitignore files tools/testing/selftests/arm64/tags/.gitignore: warning: ignored by one of the .gitignore files tools/testing/selftests/arm64/tags/Makefile: warning: ignored by one of the .gitignore files tools/testing/selftests/arm64/tags/run_tags_test.sh: warning: ignored by one of the .gitignore files tools/testing/selftests/arm64/tags/tags_test.c: warning: ignored by one of the .gitignore files These are ignored by the '.*' or 'tags' in the top .gitignore, but there is no rule to negate it. You might be tempted to do 'git add -f' but I want to have the real issue fixed (by fixing a .gitignore, or by renaming files, etc.). Signed-off-by: Masahiro Yamada Reviewed-by: Nathan Chancellor Reviewed-by: Nicolas Schier --- scripts/misc-check | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 scripts/misc-check (limited to 'scripts') diff --git a/scripts/misc-check b/scripts/misc-check new file mode 100755 index 000000000000..d40d5484e0c5 --- /dev/null +++ b/scripts/misc-check @@ -0,0 +1,19 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only + +set -e + +# Detect files that are tracked but ignored by git. This is checked only when +# ${KBUILD_EXTRA_WARN} contains 1, git is installed, and the source tree is +# tracked by git. +check_tracked_ignored_files () { + case "${KBUILD_EXTRA_WARN}" in + *1*) ;; + *) return;; + esac + + git -C ${srctree:-.} ls-files -i -c --exclude-per-directory=.gitignore 2>/dev/null | + sed 's/$/: warning: ignored by one of the .gitignore files/' >&2 +} + +check_tracked_ignored_files -- cgit v1.2.3 From 92215e7a801da7b89037a185c98f5ebb86a415b9 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 29 Dec 2022 18:15:00 +0900 Subject: kbuild: rename cmd_$@ to savedcmd_$@ in *.cmd files The cmd-check macro compares $(cmd_$@) and $(cmd_$1), but a pitfall is that you cannot use cmd_ as the variable name for the command. For example, the following code will not work in the top Makefile or ./Kbuild. quiet_cmd_foo = GEN $@ cmd_foo = touch $@ targets += foo foo: FORCE $(call if_changed,foo) In this case, both $@ and $1 are expanded to 'foo', so $(cmd_check) is always empty. We do not need to use the same prefix for cmd_$@ and cmd_$1. Rename the former to savedcmd_$@. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Kbuild.include | 8 ++++---- scripts/Makefile.modfinal | 2 +- scripts/basic/fixdep.c | 4 ++-- scripts/clang-tools/gen_compile_commands.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 10cf8d2d82ef..1a7514f49089 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -170,10 +170,10 @@ cmd = @$(if $(cmd_$(1)),set -e; $($(quiet)log_print) $(delete-on-interrupt) $(cm ifneq ($(KBUILD_NOCMDDEP),1) # Check if both commands are the same including their order. Result is empty # string if equal. User may override this check using make KBUILD_NOCMDDEP=1 -cmd-check = $(filter-out $(subst $(space),$(space_escape),$(strip $(cmd_$@))), \ +cmd-check = $(filter-out $(subst $(space),$(space_escape),$(strip $(savedcmd_$@))), \ $(subst $(space),$(space_escape),$(strip $(cmd_$1)))) else -cmd-check = $(if $(strip $(cmd_$@)),,1) +cmd-check = $(if $(strip $(savedcmd_$@)),,1) endif # Replace >$< with >$$< to preserve $ when reloading the .cmd file @@ -199,7 +199,7 @@ if_changed = $(if $(if-changed-cond),$(cmd_and_savecmd),@:) cmd_and_savecmd = \ $(cmd); \ - printf '%s\n' 'cmd_$@ := $(make-cmd)' > $(dot-target).cmd + printf '%s\n' 'savedcmd_$@ := $(make-cmd)' > $(dot-target).cmd # Execute the command and also postprocess generated .d dependencies file. if_changed_dep = $(if $(if-changed-cond),$(cmd_and_fixdep),@:) @@ -239,7 +239,7 @@ _why = \ $(if $(wildcard $@), \ $(if $(newer-prereqs),- due to: $(newer-prereqs), \ $(if $(cmd-check), \ - $(if $(cmd_$@),- due to command line change, \ + $(if $(savedcmd_$@),- due to command line change, \ $(if $(filter $@, $(targets)), \ - due to missing .cmd file, \ - due to $(notdir $@) not in $$(targets) \ diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal index a30d5b08eee9..4703f652c009 100644 --- a/scripts/Makefile.modfinal +++ b/scripts/Makefile.modfinal @@ -54,7 +54,7 @@ newer_prereqs_except = $(filter-out $(PHONY) $(1),$?) # Same as if_changed, but allows to exclude specified extra dependencies if_changed_except = $(if $(call newer_prereqs_except,$(2))$(cmd-check), \ $(cmd); \ - printf '%s\n' 'cmd_$@ := $(make-cmd)' > $(dot-target).cmd, @:) + printf '%s\n' 'savedcmd_$@ := $(make-cmd)' > $(dot-target).cmd, @:) # Re-generate module BTFs if either module's .ko or vmlinux changed %.ko: %.o %.mod.o scripts/module.lds $(and $(CONFIG_DEBUG_INFO_BTF_MODULES),$(KBUILD_BUILTIN),vmlinux) FORCE diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index f932aeaba71a..37782a632494 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -70,7 +70,7 @@ * * It first generates a line * - * cmd_ = + * savedcmd_ = * * and then basically copies the ..d file to stdout, in the * process filtering out the dependency on autoconf.h and adding @@ -343,7 +343,7 @@ int main(int argc, char *argv[]) target = argv[2]; cmdline = argv[3]; - printf("cmd_%s := %s\n\n", target, cmdline); + printf("savedcmd_%s := %s\n\n", target, cmdline); buf = read_file(depfile); parse_dep_file(buf, target); diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py index 0227522959a4..15ba56527acd 100755 --- a/scripts/clang-tools/gen_compile_commands.py +++ b/scripts/clang-tools/gen_compile_commands.py @@ -19,7 +19,7 @@ _DEFAULT_OUTPUT = 'compile_commands.json' _DEFAULT_LOG_LEVEL = 'WARNING' _FILENAME_PATTERN = r'^\..*\.cmd$' -_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' +_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] # The tools/ directory adopts a different build system, and produces .cmd # files in a different format. Do not support it. -- cgit v1.2.3 From ee2162bd4fe85a62dcd3b9133fd0c537ae2eb3c5 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 29 Dec 2022 18:15:01 +0900 Subject: kbuild: add more comments for KBUILD_NOCMDDEP=1 The cmd-check for KBUILD_NOCMDDEP=1 may not be clear until you see commit c4d5ee13984f ("kbuild: make KBUILD_NOCMDDEP=1 handle empty built-in.o"). When a phony target (i.e. FORCE) is the only prerequisite, Kbuild uses a tricky way to detect that the target does not exist. Add more comments. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Kbuild.include | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'scripts') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 1a7514f49089..4648ab8f11d4 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -170,9 +170,13 @@ cmd = @$(if $(cmd_$(1)),set -e; $($(quiet)log_print) $(delete-on-interrupt) $(cm ifneq ($(KBUILD_NOCMDDEP),1) # Check if both commands are the same including their order. Result is empty # string if equal. User may override this check using make KBUILD_NOCMDDEP=1 +# If the target does not exist, the *.cmd file should not be included so +# $(savedcmd_$@) gets empty. Then, target will be built even if $(newer-prereqs) +# happens to become empty. cmd-check = $(filter-out $(subst $(space),$(space_escape),$(strip $(savedcmd_$@))), \ $(subst $(space),$(space_escape),$(strip $(cmd_$1)))) else +# We still need to detect missing targets. cmd-check = $(if $(strip $(savedcmd_$@)),,1) endif @@ -186,6 +190,8 @@ make-cmd = $(call escsq,$(subst $(pound),$$(pound),$(subst $$,$$$$,$(cmd_$(1)))) # Find any prerequisites that are newer than target or that do not exist. # PHONY targets skipped in both cases. +# If there is no prerequisite other than phony targets, $(newer-prereqs) becomes +# empty even if the target does not exist. cmd-check saves this corner case. newer-prereqs = $(filter-out $(PHONY),$?) # It is a typical mistake to forget the FORCE prerequisite. Check it here so -- cgit v1.2.3 From ecd42fba5765bba5955e3e1f80265295c0f5c32d Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 30 Dec 2022 03:46:50 +0900 Subject: kbuild: unify cmd_dt_S_dtb and cmd_dt_S_dtbo cmd_dt_S_dtb and cmd_dt_S_dtbo are almost the same; the only difference is the prefix of the begin/end symbols. (__dtb vs __dtbo) Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Makefile.lib | 45 +++++++++++++++------------------------------ 1 file changed, 15 insertions(+), 30 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 4a4a5f67c1a6..100a386fcd71 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -368,40 +368,25 @@ DTC_FLAGS += $(DTC_FLAGS_$(basetarget)) DTC_FLAGS += $(if $(filter $(patsubst $(obj)/%,%,$@), $(base-dtb-y)), -@) # Generate an assembly file to wrap the output of the device tree compiler -quiet_cmd_dt_S_dtb= DTBS $@ -cmd_dt_S_dtb= \ -{ \ - echo '\#include '; \ - echo '.section .dtb.init.rodata,"a"'; \ - echo '.balign STRUCT_ALIGNMENT'; \ - echo '.global __dtb_$(subst -,_,$(*F))_begin'; \ - echo '__dtb_$(subst -,_,$(*F))_begin:'; \ - echo '.incbin "$<" '; \ - echo '__dtb_$(subst -,_,$(*F))_end:'; \ - echo '.global __dtb_$(subst -,_,$(*F))_end'; \ - echo '.balign STRUCT_ALIGNMENT'; \ -} > $@ +quiet_cmd_wrap_S_dtb = WRAP $@ + cmd_wrap_S_dtb = { \ + symbase=__$(patsubst .%,%,$(suffix $<))_$(subst -,_,$(notdir $*)); \ + echo '\#include '; \ + echo '.section .dtb.init.rodata,"a"'; \ + echo '.balign STRUCT_ALIGNMENT'; \ + echo ".global $${symbase}_begin"; \ + echo "$${symbase}_begin:"; \ + echo '.incbin "$<" '; \ + echo ".global $${symbase}_end"; \ + echo "$${symbase}_end:"; \ + echo '.balign STRUCT_ALIGNMENT'; \ + } > $@ $(obj)/%.dtb.S: $(obj)/%.dtb FORCE - $(call if_changed,dt_S_dtb) - -# Generate an assembly file to wrap the output of the device tree compiler -quiet_cmd_dt_S_dtbo= DTBOS $@ -cmd_dt_S_dtbo= \ -{ \ - echo '\#include '; \ - echo '.section .dtb.init.rodata,"a"'; \ - echo '.balign STRUCT_ALIGNMENT'; \ - echo '.global __dtbo_$(subst -,_,$(*F))_begin'; \ - echo '__dtbo_$(subst -,_,$(*F))_begin:'; \ - echo '.incbin "$<" '; \ - echo '__dtbo_$(subst -,_,$(*F))_end:'; \ - echo '.global __dtbo_$(subst -,_,$(*F))_end'; \ - echo '.balign STRUCT_ALIGNMENT'; \ -} > $@ + $(call if_changed,wrap_S_dtb) $(obj)/%.dtbo.S: $(obj)/%.dtbo FORCE - $(call if_changed,dt_S_dtbo) + $(call if_changed,wrap_S_dtb) quiet_cmd_dtc = DTC $@ cmd_dtc = $(HOSTCC) -E $(dtc_cpp_flags) -x assembler-with-cpp -o $(dtc-tmp) $< ; \ -- cgit v1.2.3 From 16169a47d5c36046041527faafb5a3f5c86701c6 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:14 +0900 Subject: kbuild: refactor host*_flags Remove _host*_flags. No functional change is intended. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda --- scripts/Makefile.host | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.host b/scripts/Makefile.host index da133780b751..4a02b31cd102 100644 --- a/scripts/Makefile.host +++ b/scripts/Makefile.host @@ -80,25 +80,23 @@ host-rust := $(addprefix $(obj)/,$(host-rust)) ##### # Handle options to gcc. Support building with separate output directory -_hostc_flags = $(KBUILD_HOSTCFLAGS) $(HOST_EXTRACFLAGS) \ +hostc_flags = -Wp,-MMD,$(depfile) \ + $(KBUILD_HOSTCFLAGS) $(HOST_EXTRACFLAGS) \ $(HOSTCFLAGS_$(target-stem).o) -_hostcxx_flags = $(KBUILD_HOSTCXXFLAGS) $(HOST_EXTRACXXFLAGS) \ +hostcxx_flags = -Wp,-MMD,$(depfile) \ + $(KBUILD_HOSTCXXFLAGS) $(HOST_EXTRACXXFLAGS) \ $(HOSTCXXFLAGS_$(target-stem).o) -_hostrust_flags = $(KBUILD_HOSTRUSTFLAGS) $(HOST_EXTRARUSTFLAGS) \ - $(HOSTRUSTFLAGS_$(target-stem)) +hostrust_flags = $(KBUILD_HOSTRUSTFLAGS) $(HOST_EXTRARUSTFLAGS) \ + $(HOSTRUSTFLAGS_$(target-stem)) # $(objtree)/$(obj) for including generated headers from checkin source files ifeq ($(KBUILD_EXTMOD),) ifdef building_out_of_srctree -_hostc_flags += -I $(objtree)/$(obj) -_hostcxx_flags += -I $(objtree)/$(obj) +hostc_flags += -I $(objtree)/$(obj) +hostcxx_flags += -I $(objtree)/$(obj) endif endif -hostc_flags = -Wp,-MMD,$(depfile) $(_hostc_flags) -hostcxx_flags = -Wp,-MMD,$(depfile) $(_hostcxx_flags) -hostrust_flags = $(_hostrust_flags) - ##### # Compile programs on the host -- cgit v1.2.3 From 295d8398c67e314d99bb070f38883f83fe94a97a Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:15 +0900 Subject: kbuild: specify output names separately for each emission type from rustc In Kbuild, two different rules must not write to the same file, but it happens when compiling rust source files. For example, set CONFIG_SAMPLE_RUST_MINIMAL=m and run the following: $ make -j$(nproc) samples/rust/rust_minimal.o samples/rust/rust_minimal.rsi \ samples/rust/rust_minimal.s samples/rust/rust_minimal.ll [snip] RUSTC [M] samples/rust/rust_minimal.o RUSTC [M] samples/rust/rust_minimal.rsi RUSTC [M] samples/rust/rust_minimal.s RUSTC [M] samples/rust/rust_minimal.ll mv: cannot stat 'samples/rust/rust_minimal.d': No such file or directory make[3]: *** [scripts/Makefile.build:334: samples/rust/rust_minimal.ll] Error 1 make[3]: *** Waiting for unfinished jobs.... mv: cannot stat 'samples/rust/rust_minimal.d': No such file or directory make[3]: *** [scripts/Makefile.build:309: samples/rust/rust_minimal.o] Error 1 mv: cannot stat 'samples/rust/rust_minimal.d': No such file or directory make[3]: *** [scripts/Makefile.build:326: samples/rust/rust_minimal.s] Error 1 make[2]: *** [scripts/Makefile.build:504: samples/rust] Error 2 make[1]: *** [scripts/Makefile.build:504: samples] Error 2 make: *** [Makefile:2008: .] Error 2 The reason for the error is that 4 threads running in parallel renames the same file, samples/rust/rust_minimal.d. This does not happen when compiling C or assembly files because -Wp,-MMD,$(depfile) explicitly specifies the dependency filepath. $(depfile) is a unique path for each target. Currently, rustc is only given --out-dir and --emit= So, all the rust build rules output the dep-info into the default .d, which causes the path conflict. Fortunately, the --emit option is able to specify the output path individually, with the form --emit==. Add --emit=dep-info=$(depfile) to the common part. Also, remove the redundant --out-dir because the output path is specified for each type. The code gets much cleaner because we do not need to rename *.d files. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda Reviewed-by: Vincenzo Palazzo --- scripts/Makefile.build | 14 +++++++------- scripts/Makefile.host | 7 +++---- 2 files changed, 10 insertions(+), 11 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.build b/scripts/Makefile.build index a0d5c6cca76d..40de20246e50 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -285,11 +285,11 @@ rust_common_cmd = \ -Zcrate-attr=no_std \ -Zcrate-attr='feature($(rust_allowed_features))' \ --extern alloc --extern kernel \ - --crate-type rlib --out-dir $(obj) -L $(objtree)/rust/ \ - --crate-name $(basename $(notdir $@)) + --crate-type rlib -L $(objtree)/rust/ \ + --crate-name $(basename $(notdir $@)) \ + --emit=dep-info=$(depfile) rust_handle_depfile = \ - mv $(obj)/$(basename $(notdir $@)).d $(depfile); \ sed -i '/^\#/d' $(depfile) # `--emit=obj`, `--emit=asm` and `--emit=llvm-ir` imply a single codegen unit @@ -302,7 +302,7 @@ rust_handle_depfile = \ quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_o_rs = \ - $(rust_common_cmd) --emit=dep-info,obj $<; \ + $(rust_common_cmd) --emit=obj=$@ $<; \ $(rust_handle_depfile) $(obj)/%.o: $(src)/%.rs FORCE @@ -310,7 +310,7 @@ $(obj)/%.o: $(src)/%.rs FORCE quiet_cmd_rustc_rsi_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_rsi_rs = \ - $(rust_common_cmd) --emit=dep-info -Zunpretty=expanded $< >$@; \ + $(rust_common_cmd) -Zunpretty=expanded $< >$@; \ command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@; \ $(rust_handle_depfile) @@ -319,7 +319,7 @@ $(obj)/%.rsi: $(src)/%.rs FORCE quiet_cmd_rustc_s_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_s_rs = \ - $(rust_common_cmd) --emit=dep-info,asm $<; \ + $(rust_common_cmd) --emit=asm=$@ $<; \ $(rust_handle_depfile) $(obj)/%.s: $(src)/%.rs FORCE @@ -327,7 +327,7 @@ $(obj)/%.s: $(src)/%.rs FORCE quiet_cmd_rustc_ll_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_ll_rs = \ - $(rust_common_cmd) --emit=dep-info,llvm-ir $<; \ + $(rust_common_cmd) --emit=llvm-ir=$@ $<; \ $(rust_handle_depfile) $(obj)/%.ll: $(src)/%.rs FORCE diff --git a/scripts/Makefile.host b/scripts/Makefile.host index 4a02b31cd102..d812241144d4 100644 --- a/scripts/Makefile.host +++ b/scripts/Makefile.host @@ -86,7 +86,8 @@ hostc_flags = -Wp,-MMD,$(depfile) \ hostcxx_flags = -Wp,-MMD,$(depfile) \ $(KBUILD_HOSTCXXFLAGS) $(HOST_EXTRACXXFLAGS) \ $(HOSTCXXFLAGS_$(target-stem).o) -hostrust_flags = $(KBUILD_HOSTRUSTFLAGS) $(HOST_EXTRARUSTFLAGS) \ +hostrust_flags = --emit=dep-info=$(depfile) \ + $(KBUILD_HOSTRUSTFLAGS) $(HOST_EXTRARUSTFLAGS) \ $(HOSTRUSTFLAGS_$(target-stem)) # $(objtree)/$(obj) for including generated headers from checkin source files @@ -147,9 +148,7 @@ $(host-cxxobjs): $(obj)/%.o: $(src)/%.cc FORCE # host-rust -> Executable quiet_cmd_host-rust = HOSTRUSTC $@ cmd_host-rust = \ - $(HOSTRUSTC) $(hostrust_flags) --emit=dep-info,link \ - --out-dir=$(obj)/ $<; \ - mv $(obj)/$(target-stem).d $(depfile); \ + $(HOSTRUSTC) $(hostrust_flags) --emit=link=$@ $<; \ sed -i '/^\#/d' $(depfile) $(host-rust): $(obj)/%: $(src)/%.rs FORCE $(call if_changed_dep,host-rust) -- cgit v1.2.3 From bc6df812a1529ab0cbac7f17ac6c4f4aff2a8bbf Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:16 +0900 Subject: fixdep: parse Makefile more correctly to handle comments etc. fixdep parses dependency files (*.d) emitted by the compiler. *.d files are Makefiles describing the dependencies of the main source file. fixdep understands minimal Makefile syntax. It works well enough for GCC and Clang, but not for rustc. This commit improves the parser a little more for better processing comments, escape sequences, etc. My main motivation is to drop comments. rustc may output comments (e.g. env-dep). Currentyly, rustc build rules invoke sed to remove comments, but it is more efficient to do it in fixdep. Signed-off-by: Masahiro Yamada Acked-by: Miguel Ojeda Tested-by: Miguel Ojeda --- scripts/basic/fixdep.c | 173 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 119 insertions(+), 54 deletions(-) (limited to 'scripts') diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index 37782a632494..f5a51770eb74 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -94,6 +94,7 @@ #include #include #include +#include #include #include #include @@ -251,75 +252,139 @@ static int is_ignored_file(const char *s, int len) * assignments are parsed not only by make, but also by the rather simple * parser in scripts/mod/sumversion.c. */ -static void parse_dep_file(char *m, const char *target) +static void parse_dep_file(char *p, const char *target) { - char *p; - int is_last, is_target; - int saw_any_target = 0; - int is_first_dep = 0; - void *buf; - - while (1) { - /* Skip any "white space" */ - while (*m == ' ' || *m == '\\' || *m == '\n') - m++; - - if (!*m) + bool saw_any_target = false; + bool is_target = true; + bool is_source = false; + bool need_parse; + char *q, saved_c; + + while (*p) { + /* handle some special characters first. */ + switch (*p) { + case '#': + /* + * skip comments. + * rustc may emit comments to dep-info. + */ + p++; + while (*p != '\0' && *p != '\n') { + /* + * escaped newlines continue the comment across + * multiple lines. + */ + if (*p == '\\') + p++; + p++; + } + continue; + case ' ': + case '\t': + /* skip whitespaces */ + p++; + continue; + case '\\': + /* + * backslash/newline combinations continue the + * statement. Skip it just like a whitespace. + */ + if (*(p + 1) == '\n') { + p += 2; + continue; + } break; - - /* Find next "white space" */ - p = m; - while (*p && *p != ' ' && *p != '\\' && *p != '\n') + case '\n': + /* + * Makefiles use a line-based syntax, where the newline + * is the end of a statement. After seeing a newline, + * we expect the next token is a target. + */ p++; - is_last = (*p == '\0'); - /* Is the token we found a target name? */ - is_target = (*(p-1) == ':'); - /* Don't write any target names into the dependency file */ - if (is_target) { - /* The /next/ file is the first dependency */ - is_first_dep = 1; - } else if (!is_ignored_file(m, p - m)) { - *p = '\0'; - + is_target = true; + continue; + case ':': /* - * Do not list the source file as dependency, so that - * kbuild is not confused if a .c file is rewritten - * into .S or vice versa. Storing it in source_* is - * needed for modpost to compute srcversions. + * assume the first dependency after a colon as the + * source file. */ - if (is_first_dep) { + p++; + is_target = false; + is_source = true; + continue; + } + + /* find the end of the token */ + q = p; + while (*q != ' ' && *q != '\t' && *q != '\n' && *q != '#' && *q != ':') { + if (*q == '\\') { /* - * If processing the concatenation of multiple - * dependency files, only process the first - * target name, which will be the original - * source name, and ignore any other target - * names, which will be intermediate temporary - * files. + * backslash/newline combinations work like as + * a whitespace, so this is the end of token. */ - if (!saw_any_target) { - saw_any_target = 1; - printf("source_%s := %s\n\n", - target, m); - printf("deps_%s := \\\n", target); + if (*(q + 1) == '\n') + break; + + /* escaped special characters */ + if (*(q + 1) == '#' || *(q + 1) == ':') { + memmove(p + 1, p, q - p); + p++; } - is_first_dep = 0; - } else { - printf(" %s \\\n", m); + + q++; } - buf = read_file(m); - parse_config_file(buf); - free(buf); + if (*q == '\0') + break; + q++; } - if (is_last) - break; + /* Just discard the target */ + if (is_target) { + p = q; + continue; + } + + saved_c = *q; + *q = '\0'; + need_parse = false; /* - * Start searching for next token immediately after the first - * "whitespace" character that follows this token. + * Do not list the source file as dependency, so that kbuild is + * not confused if a .c file is rewritten into .S or vice versa. + * Storing it in source_* is needed for modpost to compute + * srcversions. */ - m = p + 1; + if (is_source) { + /* + * The DT build rule concatenates multiple dep files. + * When processing them, only process the first source + * name, which will be the original one, and ignore any + * other source names, which will be intermediate + * temporary files. + */ + if (!saw_any_target) { + saw_any_target = true; + printf("source_%s := %s\n\n", target, p); + printf("deps_%s := \\\n", target); + need_parse = true; + } + } else if (!is_ignored_file(p, q - p)) { + printf(" %s \\\n", p); + need_parse = true; + } + + if (need_parse) { + void *buf; + + buf = read_file(p); + parse_config_file(buf); + free(buf); + } + + is_source = false; + *q = saved_c; + p = q; } if (!saw_any_target) { -- cgit v1.2.3 From 2185242faddd12a1ba1060be5caf584fe5aba93a Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:17 +0900 Subject: kbuild: remove sed commands after rustc rules rustc may put comments in dep-info, so sed is used to drop them before passing it to fixdep. Now that fixdep can remove comments, Makefiles do not need to run sed. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda Reviewed-by: Vincenzo Palazzo --- scripts/Makefile.build | 18 ++++-------------- scripts/Makefile.host | 3 +-- 2 files changed, 5 insertions(+), 16 deletions(-) (limited to 'scripts') diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 40de20246e50..76323201232a 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -289,9 +289,6 @@ rust_common_cmd = \ --crate-name $(basename $(notdir $@)) \ --emit=dep-info=$(depfile) -rust_handle_depfile = \ - sed -i '/^\#/d' $(depfile) - # `--emit=obj`, `--emit=asm` and `--emit=llvm-ir` imply a single codegen unit # will be used. We explicitly request `-Ccodegen-units=1` in any case, and # the compiler shows a warning if it is not 1. However, if we ever stop @@ -301,9 +298,7 @@ rust_handle_depfile = \ # would not match each other. quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_o_rs = \ - $(rust_common_cmd) --emit=obj=$@ $<; \ - $(rust_handle_depfile) + cmd_rustc_o_rs = $(rust_common_cmd) --emit=obj=$@ $< $(obj)/%.o: $(src)/%.rs FORCE $(call if_changed_dep,rustc_o_rs) @@ -311,24 +306,19 @@ $(obj)/%.o: $(src)/%.rs FORCE quiet_cmd_rustc_rsi_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_rsi_rs = \ $(rust_common_cmd) -Zunpretty=expanded $< >$@; \ - command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@; \ - $(rust_handle_depfile) + command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@ $(obj)/%.rsi: $(src)/%.rs FORCE $(call if_changed_dep,rustc_rsi_rs) quiet_cmd_rustc_s_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_s_rs = \ - $(rust_common_cmd) --emit=asm=$@ $<; \ - $(rust_handle_depfile) + cmd_rustc_s_rs = $(rust_common_cmd) --emit=asm=$@ $< $(obj)/%.s: $(src)/%.rs FORCE $(call if_changed_dep,rustc_s_rs) quiet_cmd_rustc_ll_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_ll_rs = \ - $(rust_common_cmd) --emit=llvm-ir=$@ $<; \ - $(rust_handle_depfile) + cmd_rustc_ll_rs = $(rust_common_cmd) --emit=llvm-ir=$@ $< $(obj)/%.ll: $(src)/%.rs FORCE $(call if_changed_dep,rustc_ll_rs) diff --git a/scripts/Makefile.host b/scripts/Makefile.host index d812241144d4..7aea9005e497 100644 --- a/scripts/Makefile.host +++ b/scripts/Makefile.host @@ -148,8 +148,7 @@ $(host-cxxobjs): $(obj)/%.o: $(src)/%.cc FORCE # host-rust -> Executable quiet_cmd_host-rust = HOSTRUSTC $@ cmd_host-rust = \ - $(HOSTRUSTC) $(hostrust_flags) --emit=link=$@ $<; \ - sed -i '/^\#/d' $(depfile) + $(HOSTRUSTC) $(hostrust_flags) --emit=link=$@ $< $(host-rust): $(obj)/%: $(src)/%.rs FORCE $(call if_changed_dep,host-rust) -- cgit v1.2.3 From 871d657385466d380133e457fc86672bedf14cd9 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:18 +0900 Subject: fixdep: refactor hash table lookup Change the hash table code so it will be easier to add the second table. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda --- scripts/basic/fixdep.c | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) (limited to 'scripts') diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index f5a51770eb74..74f90a0deeb9 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -113,7 +113,7 @@ struct item { }; #define HASHSZ 256 -static struct item *hashtab[HASHSZ]; +static struct item *config_hashtab[HASHSZ]; static unsigned int strhash(const char *str, unsigned int sz) { @@ -125,25 +125,11 @@ static unsigned int strhash(const char *str, unsigned int sz) return hash; } -/* - * Lookup a value in the configuration string. - */ -static int is_defined_config(const char *name, int len, unsigned int hash) -{ - struct item *aux; - - for (aux = hashtab[hash % HASHSZ]; aux; aux = aux->next) { - if (aux->hash == hash && aux->len == len && - memcmp(aux->name, name, len) == 0) - return 1; - } - return 0; -} - /* * Add a new value to the configuration string. */ -static void define_config(const char *name, int len, unsigned int hash) +static void add_to_hashtable(const char *name, int len, unsigned int hash, + struct item *hashtab[]) { struct item *aux = malloc(sizeof(*aux) + len); @@ -158,17 +144,34 @@ static void define_config(const char *name, int len, unsigned int hash) hashtab[hash % HASHSZ] = aux; } +/* + * Lookup a string in the hash table. If found, just return true. + * If not, add it to the hashtable and return false. + */ +static bool in_hashtable(const char *name, int len, struct item *hashtab[]) +{ + struct item *aux; + unsigned int hash = strhash(name, len); + + for (aux = hashtab[hash % HASHSZ]; aux; aux = aux->next) { + if (aux->hash == hash && aux->len == len && + memcmp(aux->name, name, len) == 0) + return true; + } + + add_to_hashtable(name, len, hash, hashtab); + + return false; +} + /* * Record the use of a CONFIG_* word. */ static void use_config(const char *m, int slen) { - unsigned int hash = strhash(m, slen); - - if (is_defined_config(m, slen, hash)) - return; + if (in_hashtable(m, slen, config_hashtab)) + return; - define_config(m, slen, hash); /* Print out a dependency path from a symbol name. */ printf(" $(wildcard include/config/%.*s) \\\n", slen, m); } -- cgit v1.2.3 From faa91c472be8ffa3121c9db803d3e529d48e246a Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:19 +0900 Subject: fixdep: avoid parsing the same file over again The dep files (*.d files) emitted by C compilers usually contain the deduplicated list of included files. One exceptional case is when a header is included by the -include command line option, and also by #include directive. For example, the top Makefile adds the command line option, "-include $(srctree)/include/linux/kconfig.h". You do not need to include in every source file. In fact, include/linux/kconfig.h is listed twice in many .*.cmd files due to include/linux/xarray.h having "#include ". I did not fix that since it is a small redundancy. However, this is more annoying for rustc. rustc emits the dependency for each emission type. For example, cmd_rustc_library emits dep-info, obj, and metadata. So, the emitted *.d file contains the dependency for those 3 targets, which makes fixdep parse the same file 3 times. $ grep rust/alloc/raw_vec.rs rust/.alloc.o.cmd rust/alloc/raw_vec.rs \ rust/alloc/raw_vec.rs \ rust/alloc/raw_vec.rs \ To skip the second parsing, this commit adds a hash table for parsed files, just like we did for CONFIG options. Signed-off-by: Masahiro Yamada Acked-by: Miguel Ojeda Tested-by: Miguel Ojeda Reviewed-by: Vincenzo Palazzo --- scripts/basic/fixdep.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index 74f90a0deeb9..e22e689de61e 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -113,7 +113,7 @@ struct item { }; #define HASHSZ 256 -static struct item *config_hashtab[HASHSZ]; +static struct item *config_hashtab[HASHSZ], *file_hashtab[HASHSZ]; static unsigned int strhash(const char *str, unsigned int sz) { @@ -365,6 +365,10 @@ static void parse_dep_file(char *p, const char *target) * name, which will be the original one, and ignore any * other source names, which will be intermediate * temporary files. + * + * rustc emits the same dependency list for each + * emission type. It is enough to list the source name + * just once. */ if (!saw_any_target) { saw_any_target = true; @@ -372,7 +376,8 @@ static void parse_dep_file(char *p, const char *target) printf("deps_%s := \\\n", target); need_parse = true; } - } else if (!is_ignored_file(p, q - p)) { + } else if (!is_ignored_file(p, q - p) && + !in_hashtable(p, q - p, file_hashtab)) { printf(" %s \\\n", p); need_parse = true; } -- cgit v1.2.3 From 93c656de8da8b53317bd88ecb7cce592310995d8 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:18:20 +0900 Subject: fixdep: do not parse *.rlib, *.rmeta, *.so fixdep is designed only for parsing text files. read_file() appends a terminating null byte ('\0') and parse_config_file() calls strstr() to search for CONFIG options. rustc outputs *.rlib, *.rmeta, *.so to dep-info. fixdep needs them in the dependency, but there is no point in parsing such binary files. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda --- scripts/basic/fixdep.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/basic/fixdep.c b/scripts/basic/fixdep.c index e22e689de61e..fa562806c2be 100644 --- a/scripts/basic/fixdep.c +++ b/scripts/basic/fixdep.c @@ -250,6 +250,15 @@ static int is_ignored_file(const char *s, int len) str_ends_with(s, len, "include/generated/autoksyms.h"); } +/* Do not parse these files */ +static int is_no_parse_file(const char *s, int len) +{ + /* rustc may list binary files in dep-info */ + return str_ends_with(s, len, ".rlib") || + str_ends_with(s, len, ".rmeta") || + str_ends_with(s, len, ".so"); +} + /* * Important: The below generated source_foo.o and deps_foo.o variable * assignments are parsed not only by make, but also by the rather simple @@ -382,7 +391,7 @@ static void parse_dep_file(char *p, const char *target) need_parse = true; } - if (need_parse) { + if (need_parse && !is_no_parse_file(p, q - p)) { void *buf; buf = read_file(p); -- cgit v1.2.3 From c83b16cefd78f55071840e1159ead9fe62747769 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:45:45 +0900 Subject: kbuild: rust: move rust/target.json to scripts/ scripts/ is a better place to generate files used treewide. With target.json moved to scripts/, you do not need to add target.json to no-clean-files or MRPROPER_FILES. 'make clean' does not visit scripts/, but 'make mrproper' does. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda --- scripts/.gitignore | 1 + scripts/Makefile | 8 +++++++- scripts/remove-stale-files | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/.gitignore b/scripts/.gitignore index b7aec8eb1bd4..11bf3c075fb6 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -8,4 +8,5 @@ /recordmcount /sign-file /sorttable +/target.json /unifdef diff --git a/scripts/Makefile b/scripts/Makefile index 1575af84d557..0e0ae3c06ed7 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -10,8 +10,14 @@ hostprogs-always-$(CONFIG_BUILDTIME_TABLE_SORT) += sorttable hostprogs-always-$(CONFIG_ASN1) += asn1_compiler hostprogs-always-$(CONFIG_MODULE_SIG_FORMAT) += sign-file hostprogs-always-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE) += insert-sys-cert -hostprogs-always-$(CONFIG_RUST) += generate_rust_target +always-$(CONFIG_RUST) += target.json +filechk_rust_target = $< < include/config/auto.conf + +$(obj)/target.json: scripts/generate_rust_target include/config/auto.conf FORCE + $(call filechk,rust_target) + +hostprogs += generate_rust_target generate_rust_target-rust := y HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include diff --git a/scripts/remove-stale-files b/scripts/remove-stale-files index cdbdde89a271..c71bf2f68360 100755 --- a/scripts/remove-stale-files +++ b/scripts/remove-stale-files @@ -27,3 +27,5 @@ rm -f arch/x86/purgatory/kexec-purgatory.c rm -f scripts/extract-cert rm -f scripts/kconfig/[gmnq]conf-cfg + +rm -f rust/target.json -- cgit v1.2.3 From 12fec3d60118237e4674a463c2c2923b8279ad67 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 9 Jan 2023 22:40:40 +0900 Subject: kbuild: replace $(dot-target).tmp in filechk with $(tmp-target) $(tmp-target) is a better fit for local use like this. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Kbuild.include | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 4648ab8f11d4..7778cc97a4e0 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -99,11 +99,11 @@ define filechk $(check-FORCE) $(Q)set -e; \ mkdir -p $(dir $@); \ - trap "rm -f $(dot-target).tmp" EXIT; \ - { $(filechk_$(1)); } > $(dot-target).tmp; \ - if [ ! -r $@ ] || ! cmp -s $@ $(dot-target).tmp; then \ + trap "rm -f $(tmp-target)" EXIT; \ + { $(filechk_$(1)); } > $(tmp-target); \ + if [ ! -r $@ ] || ! cmp -s $@ $(tmp-target); then \ $(kecho) ' UPD $@'; \ - mv -f $(dot-target).tmp $@; \ + mv -f $(tmp-target) $@; \ fi endef -- cgit v1.2.3 From d5c8d6e0fa61401a729e9eb6a9c7077b2d3aebb0 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Wed, 11 Jan 2023 20:05:01 -0700 Subject: kbuild: Update assembler calls to use proper flags and language target as-instr uses KBUILD_AFLAGS, but as-option uses KBUILD_CFLAGS. This can cause as-option to fail unexpectedly when CONFIG_WERROR is set, because clang will emit -Werror,-Wunused-command-line-argument for various -m and -f flags in KBUILD_CFLAGS for assembler sources. Callers of as-option and as-instr should be adding flags to KBUILD_AFLAGS / aflags-y, not KBUILD_CFLAGS / cflags-y. Use KBUILD_AFLAGS in all macros to clear up the initial problem. Unfortunately, -Wunused-command-line-argument can still be triggered with clang by the presence of warning flags or macro definitions because '-x assembler' is used, instead of '-x assembler-with-cpp', which will consume these flags. Switch to '-x assembler-with-cpp' in places where '-x assembler' is used, as the compiler is always used as the driver for out of line assembler sources in the kernel. Finally, add -Werror to these macros so that they behave consistently whether or not CONFIG_WERROR is set. [nathan: Reworded and expanded on problems in commit message Use '-x assembler-with-cpp' in a couple more places] Link: https://github.com/ClangBuiltLinux/linux/issues/1699 Suggested-by: Masahiro Yamada Signed-off-by: Nick Desaulniers Signed-off-by: Nathan Chancellor Tested-by: Linux Kernel Functional Testing Tested-by: Anders Roxell Signed-off-by: Masahiro Yamada --- scripts/Kconfig.include | 2 +- scripts/Makefile.compiler | 8 ++++---- scripts/as-version.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/Kconfig.include b/scripts/Kconfig.include index 274125307ebd..5a84b6443875 100644 --- a/scripts/Kconfig.include +++ b/scripts/Kconfig.include @@ -33,7 +33,7 @@ ld-option = $(success,$(LD) -v $(1)) # $(as-instr,) # Return y if the assembler supports , n otherwise -as-instr = $(success,printf "%b\n" "$(1)" | $(CC) $(CLANG_FLAGS) -c -x assembler -o /dev/null -) +as-instr = $(success,printf "%b\n" "$(1)" | $(CC) $(CLANG_FLAGS) -c -x assembler-with-cpp -o /dev/null -) # check if $(CC) and $(LD) exist $(error-if,$(failure,command -v $(CC)),C compiler '$(CC)' not found) diff --git a/scripts/Makefile.compiler b/scripts/Makefile.compiler index 3d8adfd34af1..7aa1fbc4aafe 100644 --- a/scripts/Makefile.compiler +++ b/scripts/Makefile.compiler @@ -29,16 +29,16 @@ try-run = $(shell set -e; \ fi) # as-option -# Usage: cflags-y += $(call as-option,-Wa$(comma)-isa=foo,) +# Usage: aflags-y += $(call as-option,-Wa$(comma)-isa=foo,) as-option = $(call try-run,\ - $(CC) $(KBUILD_CFLAGS) $(1) -c -x assembler /dev/null -o "$$TMP",$(1),$(2)) + $(CC) -Werror $(KBUILD_AFLAGS) $(1) -c -x assembler-with-cpp /dev/null -o "$$TMP",$(1),$(2)) # as-instr -# Usage: cflags-y += $(call as-instr,instr,option1,option2) +# Usage: aflags-y += $(call as-instr,instr,option1,option2) as-instr = $(call try-run,\ - printf "%b\n" "$(1)" | $(CC) $(KBUILD_AFLAGS) -c -x assembler -o "$$TMP" -,$(2),$(3)) + printf "%b\n" "$(1)" | $(CC) -Werror $(KBUILD_AFLAGS) -c -x assembler-with-cpp -o "$$TMP" -,$(2),$(3)) # __cc-option # Usage: MY_CFLAGS += $(call __cc-option,$(CC),$(MY_CFLAGS),-march=winchip-c6,-march=i586) diff --git a/scripts/as-version.sh b/scripts/as-version.sh index 1a21495e9ff0..af717476152d 100755 --- a/scripts/as-version.sh +++ b/scripts/as-version.sh @@ -45,7 +45,7 @@ orig_args="$@" # Get the first line of the --version output. IFS=' ' -set -- $(LC_ALL=C "$@" -Wa,--version -c -x assembler /dev/null -o /dev/null 2>/dev/null) +set -- $(LC_ALL=C "$@" -Wa,--version -c -x assembler-with-cpp /dev/null -o /dev/null 2>/dev/null) # Split the line on spaces. IFS=' ' -- cgit v1.2.3 From db1547c56886742283d7566c872f89cbad76a14c Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 11 Jan 2023 20:05:10 -0700 Subject: kbuild: Turn a couple more of clang's unused option warnings into errors Currently, these warnings are hidden with -Qunused-arguments in KBUILD_CPPFLAGS. Once that option is removed, these warnings should be turned into hard errors to make unconditionally added but unsupported flags for the current compilation mode or target obvious due to a failed build; otherwise, the warnings might just be ignored if the build log is not checked. Link: https://github.com/ClangBuiltLinux/linux/issues/1587 Signed-off-by: Nathan Chancellor Reviewed-by: Nick Desaulniers Tested-by: Linux Kernel Functional Testing Tested-by: Anders Roxell Signed-off-by: Masahiro Yamada --- scripts/Makefile.clang | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/Makefile.clang b/scripts/Makefile.clang index 87285b76adb2..70b354fa1cb4 100644 --- a/scripts/Makefile.clang +++ b/scripts/Makefile.clang @@ -36,6 +36,8 @@ endif # so they can be implemented or wrapped in cc-option. CLANG_FLAGS += -Werror=unknown-warning-option CLANG_FLAGS += -Werror=ignored-optimization-argument +CLANG_FLAGS += -Werror=option-ignored +CLANG_FLAGS += -Werror=unused-command-line-argument KBUILD_CFLAGS += $(CLANG_FLAGS) KBUILD_AFLAGS += $(CLANG_FLAGS) export CLANG_FLAGS -- cgit v1.2.3 From 87c7ee67deb7fce9951a5f9d80641138694aad17 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 12 Jan 2023 11:30:06 +0900 Subject: scripts: handle BrokenPipeError for python scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the follow-up of commit fb3041d61f68 ("kbuild: fix SIGPIPE error message for AR=gcc-ar and AR=llvm-ar"), Kees Cook pointed out that tools should _not_ catch their own SIGPIPEs [1] [2]. Based on his feedback, LLVM was fixed [3]. However, Python's default behavior is to show noisy bracktrace when SIGPIPE is sent. So, scripts written in Python are basically in the same situation as the buggy llvm tools. Example: $ make -s allnoconfig $ make -s allmodconfig $ scripts/diffconfig .config.old .config | head -n1 -ALIX n Traceback (most recent call last): File "/home/masahiro/linux/scripts/diffconfig", line 132, in main() File "/home/masahiro/linux/scripts/diffconfig", line 130, in main print_config("+", config, None, b[config]) File "/home/masahiro/linux/scripts/diffconfig", line 64, in print_config print("+%s %s" % (config, new_value)) BrokenPipeError: [Errno 32] Broken pipe Python documentation [4] notes how to make scripts die immediately and silently: """ Piping output of your program to tools like head(1) will cause a SIGPIPE signal to be sent to your process when the receiver of its standard output closes early. This results in an exception like BrokenPipeError: [Errno 32] Broken pipe. To handle this case, wrap your entry point to catch this exception as follows: import os import sys def main(): try: # simulate large output (your code replaces this loop) for x in range(10000): print("y") # flush output here to force SIGPIPE to be triggered # while inside this try block. sys.stdout.flush() except BrokenPipeError: # Python flushes standard streams on exit; redirect remaining output # to devnull to avoid another BrokenPipeError at shutdown devnull = os.open(os.devnull, os.O_WRONLY) os.dup2(devnull, sys.stdout.fileno()) sys.exit(1) # Python exits with error code 1 on EPIPE if __name__ == '__main__': main() Do not set SIGPIPE’s disposition to SIG_DFL in order to avoid BrokenPipeError. Doing that would cause your program to exit unexpectedly whenever any socket connection is interrupted while your program is still writing to it. """ Currently, tools/perf/scripts/python/intel-pt-events.py seems to be the only script that fixes the issue that way. tools/perf/scripts/python/compaction-times.py uses another approach signal.signal(signal.SIGPIPE, signal.SIG_DFL) but the Python documentation clearly says "Don't do it". I cannot fix all Python scripts since there are so many. I fixed some in the scripts/ directory. [1]: https://lore.kernel.org/all/202211161056.1B9611A@keescook/ [2]: https://github.com/llvm/llvm-project/issues/59037 [3]: https://github.com/llvm/llvm-project/commit/4787efa38066adb51e2c049499d25b3610c0877b [4]: https://docs.python.org/3/library/signal.html#note-on-sigpipe Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers Reviewed-by: Nicolas Schier --- scripts/checkkconfigsymbols.py | 13 ++++++++++++- scripts/clang-tools/run-clang-tools.py | 21 ++++++++++++++------- scripts/diffconfig | 16 ++++++++++++++-- 3 files changed, 40 insertions(+), 10 deletions(-) (limited to 'scripts') diff --git a/scripts/checkkconfigsymbols.py b/scripts/checkkconfigsymbols.py index 217d21abc86e..36c920e71313 100755 --- a/scripts/checkkconfigsymbols.py +++ b/scripts/checkkconfigsymbols.py @@ -115,7 +115,7 @@ def parse_options(): return args -def main(): +def print_undefined_symbols(): """Main function of this module.""" args = parse_options() @@ -467,5 +467,16 @@ def parse_kconfig_file(kfile): return defined, references +def main(): + try: + print_undefined_symbols() + except BrokenPipeError: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(1) # Python exits with error code 1 on EPIPE + + if __name__ == "__main__": main() diff --git a/scripts/clang-tools/run-clang-tools.py b/scripts/clang-tools/run-clang-tools.py index 56f2ec8f0f40..3266708a8658 100755 --- a/scripts/clang-tools/run-clang-tools.py +++ b/scripts/clang-tools/run-clang-tools.py @@ -61,14 +61,21 @@ def run_analysis(entry): def main(): - args = parse_arguments() + try: + args = parse_arguments() - lock = multiprocessing.Lock() - pool = multiprocessing.Pool(initializer=init, initargs=(lock, args)) - # Read JSON data into the datastore variable - with open(args.path, "r") as f: - datastore = json.load(f) - pool.map(run_analysis, datastore) + lock = multiprocessing.Lock() + pool = multiprocessing.Pool(initializer=init, initargs=(lock, args)) + # Read JSON data into the datastore variable + with open(args.path, "r") as f: + datastore = json.load(f) + pool.map(run_analysis, datastore) + except BrokenPipeError: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(1) # Python exits with error code 1 on EPIPE if __name__ == "__main__": diff --git a/scripts/diffconfig b/scripts/diffconfig index d5da5fa05d1d..43f0f3d273ae 100755 --- a/scripts/diffconfig +++ b/scripts/diffconfig @@ -65,7 +65,7 @@ def print_config(op, config, value, new_value): else: print(" %s %s -> %s" % (config, value, new_value)) -def main(): +def show_diff(): global merge_style # parse command line args @@ -129,4 +129,16 @@ def main(): for config in new: print_config("+", config, None, b[config]) -main() +def main(): + try: + show_diff() + except BrokenPipeError: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(1) # Python exits with error code 1 on EPIPE + + +if __name__ == '__main__': + main() -- cgit v1.2.3 From ec61452aaad6bb0d4f3f4d4b78d27bed90eb2494 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 19 Jan 2023 16:12:15 +0900 Subject: scripts: remove bin2c Commit 80f8be7af03f ("tomoyo: Omit use of bin2c") removed the last use of bin2c. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier Reviewed-by: Sedat Dilek --- scripts/.gitignore | 1 - scripts/Makefile | 1 - scripts/bin2c.c | 36 ------------------------------------ scripts/remove-stale-files | 2 ++ 4 files changed, 2 insertions(+), 38 deletions(-) delete mode 100644 scripts/bin2c.c (limited to 'scripts') diff --git a/scripts/.gitignore b/scripts/.gitignore index 11bf3c075fb6..6e9ce6720a05 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only /asn1_compiler -/bin2c /generate_rust_target /insert-sys-cert /kallsyms diff --git a/scripts/Makefile b/scripts/Makefile index 0e0ae3c06ed7..32b6ba722728 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -3,7 +3,6 @@ # scripts contains sources for various helper programs used throughout # the kernel for the build process. -hostprogs-always-$(CONFIG_BUILD_BIN2C) += bin2c hostprogs-always-$(CONFIG_KALLSYMS) += kallsyms hostprogs-always-$(BUILD_C_RECORDMCOUNT) += recordmcount hostprogs-always-$(CONFIG_BUILDTIME_TABLE_SORT) += sorttable diff --git a/scripts/bin2c.c b/scripts/bin2c.c deleted file mode 100644 index c3d7eef3ad06..000000000000 --- a/scripts/bin2c.c +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Unloved program to convert a binary on stdin to a C include on stdout - * - * Jan 1999 Matt Mackall - * - * This software may be used and distributed according to the terms - * of the GNU General Public License, incorporated herein by reference. - */ - -#include - -int main(int argc, char *argv[]) -{ - int ch, total = 0; - - if (argc > 1) - printf("const char %s[] %s=\n", - argv[1], argc > 2 ? argv[2] : ""); - - do { - printf("\t\""); - while ((ch = getchar()) != EOF) { - total++; - printf("\\x%02x", ch); - if (total % 16