Skip to content

bpo-45020: Make "make all" output less noisy. #28554

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ericsnowcurrently wants to merge 5 commits into from
Closed

bpo-45020: Make "make all" output less noisy. #28554

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2fbef5c
Don't write the generated files if they haven't changed.
ericsnowcurrently Sep 24, 2021
18016f8
Make "make all" less noisy..
ericsnowcurrently Sep 24, 2021
857f380
Make the recipes a little easier to read.
ericsnowcurrently Sep 24, 2021
ee13e18
Show the frozen file if it is missing (but others aren't).
ericsnowcurrently Sep 24, 2021
5caaacc
Use a common make recipe for all the frozen modules.
ericsnowcurrently Sep 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ Tools/ssl/win32
Python/frozen_modules/*.h
# The manifest can be generated at any time with "make regen-frozen".
Python/frozen_modules/MANIFEST
Python/frozen_modules/ready-stamp
Python/frozen_modules/generated-stamp

# Two-trick pony for OSX and other case insensitive file systems:
# Ignore ./python binary on Unix but still look into ./Python/ directory.
Expand Down
141 changes: 98 additions & 43 deletions Makefile.pre.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -772,58 +772,112 @@ FROZEN_FILES_OUT = \
Python/frozen_modules/__hello__.h
# End FROZEN_FILES_OUT

Programs/_freeze_module.o: Programs/_freeze_module.c Makefile

Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN)
$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)

# BEGIN: freezing modules

Python/frozen_modules/importlib._bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py
Programs/_freeze_module importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/frozen_modules/importlib._bootstrap.h
# the 3.0 release (6046c5e0298c25515ea58abc8ab87f7413e3f743)
OLD_TIMESTAMP=200812031729.10

Python/frozen_modules/importlib._bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py
Programs/_freeze_module importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py $(srcdir)/Python/frozen_modules/importlib._bootstrap_external.h
FROZEN_DATA=Python/frozen_modules
FROZEN_READY=$(FROZEN_DATA)/ready-stamp
FROZEN_GENERATED=$(FROZEN_DATA)/generated-stamp

Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py
Programs/_freeze_module zipimport $(srcdir)/Lib/zipimport.py $(srcdir)/Python/frozen_modules/zipimport.h
.PHONY: clean-frozen
clean-frozen:
rm -rf $(FROZEN_DATA)/*-stamp
# rm -f $(FROZEN_DATA)/*.h
# rm -f $(FROZEN_FILES_OUT)

Python/frozen_modules/abc.h: Programs/_freeze_module Lib/abc.py
Programs/_freeze_module abc $(srcdir)/Lib/abc.py $(srcdir)/Python/frozen_modules/abc.h
.FORCE:

Python/frozen_modules/codecs.h: Programs/_freeze_module Lib/codecs.py
Programs/_freeze_module codecs $(srcdir)/Lib/codecs.py $(srcdir)/Python/frozen_modules/codecs.h

Python/frozen_modules/io.h: Programs/_freeze_module Lib/io.py
Programs/_freeze_module io $(srcdir)/Lib/io.py $(srcdir)/Python/frozen_modules/io.h

Python/frozen_modules/_collections_abc.h: Programs/_freeze_module Lib/_collections_abc.py
Programs/_freeze_module _collections_abc $(srcdir)/Lib/_collections_abc.py $(srcdir)/Python/frozen_modules/_collections_abc.h

Python/frozen_modules/_sitebuiltins.h: Programs/_freeze_module Lib/_sitebuiltins.py
Programs/_freeze_module _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py $(srcdir)/Python/frozen_modules/_sitebuiltins.h

Python/frozen_modules/genericpath.h: Programs/_freeze_module Lib/genericpath.py
Programs/_freeze_module genericpath $(srcdir)/Lib/genericpath.py $(srcdir)/Python/frozen_modules/genericpath.h

Python/frozen_modules/ntpath.h: Programs/_freeze_module Lib/ntpath.py
Programs/_freeze_module ntpath $(srcdir)/Lib/ntpath.py $(srcdir)/Python/frozen_modules/ntpath.h
# This should always run before any frozen module .h rules.
# It gets all the frozen-related stamp files in the proper state.
#
# This file is created by this rule, with an old timestamp.
# It is deleted as soon as the state of the files changes.
$(FROZEN_READY): .FORCE
@mkdir -p $(FROZEN_DATA)
@touch -t $(OLD_TIMESTAMP) $@
@rm -rf $(FROZEN_GENERATED)

# This records that one or more frozen module .h files was built. It also
# prints an informative header when the first frozen module is handled.
# It is run at the beginning of the recipe for each frozen module .h file.
# Note that the rule for each module will print a progress marker on
# the same line as the others.
.PHONY: .generating-frozen-module
.generating-frozen-module:
@if ( mkdir $(FROZEN_GENERATED) ) 2> /dev/null; then \
chmod 555 $(FROZEN_GENERATED); \
echo "=== Re-generating the frozen modules. ==="; \
echo "(For verbose output, run make regen-frozen.)"; \
else \
touch $(FROZEN_GENERATED); \
fi
@rm -f $(FROZEN_READY)

Python/frozen_modules/posixpath.h: Programs/_freeze_module Lib/posixpath.py
Programs/_freeze_module posixpath $(srcdir)/Lib/posixpath.py $(srcdir)/Python/frozen_modules/posixpath.h
# This should always run after all frozen module .h rules finish.
# If at least one frozen module rules actually ran then this rule
# finishes printing the line where the progress markers were printed.
# This rule also removes all the other frozen-related stamp files.
#
# This file is created by the .generating-frozen-module rule if at least
# one of the frozen module rules is run. Otherwise it is created here,
# with an old timestamp. It is always an empty directory.
# timestamp. It is updated to "now" in each frozem module .h rule
# if the module was actually updated.
$(FROZEN_GENERATED): $(FROZEN_FILES_OUT) .FORCE
@if test -e $(FROZEN_GENERATED); then \
echo ""; \
else \
touch $@; \
chmod 555 $(FROZEN_GENERATED); \
fi
@touch -t $(OLD_TIMESTAMP) $@
@rm -f $(FROZEN_READY)

Python/frozen_modules/os.h: Programs/_freeze_module Lib/os.py
Programs/_freeze_module os $(srcdir)/Lib/os.py $(srcdir)/Python/frozen_modules/os.h
Programs/_freeze_module.o: Programs/_freeze_module.c Makefile

Python/frozen_modules/site.h: Programs/_freeze_module Lib/site.py
Programs/_freeze_module site $(srcdir)/Lib/site.py $(srcdir)/Python/frozen_modules/site.h
Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(FROZEN_READY)
$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)

Python/frozen_modules/stat.h: Programs/_freeze_module Lib/stat.py
Programs/_freeze_module stat $(srcdir)/Lib/stat.py $(srcdir)/Python/frozen_modules/stat.h
have_frozen_files=$(wildcard $(FROZEN_FILES_OUT))

Python/frozen_modules/__hello__.h: Programs/_freeze_module Lib/__hello__.py
Programs/_freeze_module __hello__ $(srcdir)/Lib/__hello__.py $(srcdir)/Python/frozen_modules/__hello__.h
$(FROZEN_DATA)/%.h: Programs/_freeze_module

@gvanrossum gvanrossum Sep 27, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW, the commands in the block below this line give me the creeps (== I've never seen such make syntax before :-).

But perhaps we can define a pattern rule that tells how to generate a frozen .h file from a Lib .py file? That won't change the output volume, but it will simplify the Makefile.

$(eval exists = $(wildcard $@))
$(eval pyfile = Lib/$(subst .,/,$(*F)).py)
$(eval pyfilefull = $(srcdir)/$(pyfile))
@if ( mkdir $(FROZEN_GENERATED) ) 2> /dev/null; then \
chmod 555 $(FROZEN_GENERATED); \
echo "=== Re-generating the frozen modules. ==="; \
echo "(For verbose output, run make regen-frozen.)"; \
else \
touch $(FROZEN_GENERATED); \
fi
@rm -f $(FROZEN_READY)
@Programs/_freeze_module $(*F) "$(pyfilefull)" $@.new
@if $(UPDATE_FILE) --create --exitcode $@ $@.new; then \
if test $@ -ot Python/frozen.o; then touch -r Python/frozen.o $@; fi; \
if test $@ -ot "$(pyfilefull)"; then touch -r "$(pyfilefull)" $@; fi; \
fi
@if test -n "$(exists)" && test -n "$(findstring $(pyfile),$?)"; then echo -n " $(@F) "; \
elif test -z "$(have_frozen_files)"; then echo -n "."; \
elif test -z "$(exists)"; then echo -n " $(@F) "; \
else echo -n "."; fi

# BEGIN: freezing modules
Python/frozen_modules/importlib._bootstrap.h: $(srcdir)/Lib/importlib/_bootstrap.py
Python/frozen_modules/importlib._bootstrap_external.h: $(srcdir)/Lib/importlib/_bootstrap_external.py
Python/frozen_modules/zipimport.h: $(srcdir)/Lib/zipimport.py
Python/frozen_modules/abc.h: $(srcdir)/Lib/abc.py
Python/frozen_modules/codecs.h: $(srcdir)/Lib/codecs.py
Python/frozen_modules/io.h: $(srcdir)/Lib/io.py
Python/frozen_modules/_collections_abc.h: $(srcdir)/Lib/_collections_abc.py
Python/frozen_modules/_sitebuiltins.h: $(srcdir)/Lib/_sitebuiltins.py
Python/frozen_modules/genericpath.h: $(srcdir)/Lib/genericpath.py
Python/frozen_modules/ntpath.h: $(srcdir)/Lib/ntpath.py
Python/frozen_modules/posixpath.h: $(srcdir)/Lib/posixpath.py
Python/frozen_modules/os.h: $(srcdir)/Lib/os.py
Python/frozen_modules/site.h: $(srcdir)/Lib/site.py
Python/frozen_modules/stat.h: $(srcdir)/Lib/stat.py
Python/frozen_modules/__hello__.h: $(srcdir)/Lib/__hello__.py
# END: freezing modules

Tools/scripts/freeze_modules.py: Programs/_freeze_module
Expand Down Expand Up @@ -1062,7 +1116,7 @@ regen-opcode-targets:
Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \
$(srcdir)/Python/condvar.h

Python/frozen.o: $(FROZEN_FILES_OUT)
Python/frozen.o: $(FROZEN_FILES_OUT) $(FROZEN_GENERATED)

# Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to
# follow our naming conventions. dtrace(1) uses the output filename to generate
Expand Down Expand Up @@ -1988,6 +2042,7 @@ clean-retain-profile: pycremoval
-find build -type f -a ! -name '*.gc??' -exec rm -f {} ';'
-rm -f Include/pydtrace_probes.h
-rm -f profile-gen-stamp
-rm -rf $(FROZEN_DATA)/*-stamp

profile-removal:
find . -name '*.gc??' -exec rm -f {} ';'
Expand Down
8 changes: 2 additions & 6 deletions Tools/scripts/freeze_modules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -545,20 +545,16 @@ def regen_frozen(modules):
def regen_makefile(modules):
pyfiles = []
frozenfiles = []
rules = ['']
rules = []
for src in _iter_sources(modules):
header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
frozenfiles.append(f'\t\t{header} \\')

pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
pyfiles.append(f'\t\t{pyfile} \\')

freeze = (f'Programs/_freeze_module {src.frozenid} '
f'$(srcdir)/{pyfile} $(srcdir)/{header}')
rules.extend([
f'{header}: Programs/_freeze_module {pyfile}',
f'\t{freeze}',
'',
f'{header}: $(srcdir)/{pyfile}',
])
pyfiles[-1] = pyfiles[-1].rstrip(" \\")
frozenfiles[-1] = frozenfiles[-1].rstrip(" \\")
Expand Down