From 30eb1e6579559f5fc3127b32c5e9b55b5b418cd6 Mon Sep 17 00:00:00 2001 From: Erik Cervin-Edin Date: Tue, 26 May 2026 12:47:43 +0200 Subject: [PATCH 001/259] commit: allow -m/-F for all kinds of --fixup The ability to provide a commit message for git commit --fixup and its variations is limited: * Plain --fixup only allows using the -m flag * The amend/reword --fixup variants only allow supplying the message using an editor For amend/reword, the -m and -F flags are rejected: -m is caught by a die() in prepare_to_commit(), and -F is caught by die_for_incompatible_opt4() which groups -F with --fixup as mutually exclusive. This makes these modes poorly suited for non-interactive workflows -- notably when using AI coding agents. When support to use the -m option was introduced in [1] it was noted that there could be support for other options but at the time the use case was deemed too niche. Later, when the amend suboption was introduced in [2] -m support for amend fixups was discussed but not pursued, and -F was already caught by the higher-layer incompatibility check grouping it with --fixup. The rejections of these options hark back to when --fixup was introduced in [3] and as noted in [1] -- there's nothing inherently preventing support for them. The current patchwork of which flags work with which --fixup variants has no strong logic to it, and allowing all of them simplifies both the code and the interface. Allow -m and -F to supply the message body for all --fixup variations, mirroring the flow of a regular commit. -c and -C, which are blocked by the same incompatibility check, are handled in the next commit. 1. 30884c9afc (commit: add support for --fixup -m"", 2017-12-22) 2. 494d314a05 (commit: add amend suboption to --fixup to create amend! commit, 2021-03-15) 3. d71b8ba7c9 (commit: --fixup option for use with rebase --autosquash, 2010-11-02) Helped-by: Junio C Hamano Suggested-by: Phillip Wood Signed-off-by: Erik Cervin-Edin Signed-off-by: Junio C Hamano --- Documentation/git-commit.adoc | 19 ++++---- builtin/commit.c | 34 +++++++------- t/t7500-commit-template-squash-signoff.sh | 56 +++++++++++++++++------ 3 files changed, 69 insertions(+), 40 deletions(-) diff --git a/Documentation/git-commit.adoc b/Documentation/git-commit.adoc index 8329c1034b9b30..61efd29e66e7d0 100644 --- a/Documentation/git-commit.adoc +++ b/Documentation/git-commit.adoc @@ -103,20 +103,21 @@ include::diff-context-options.adoc[] The commit created by plain `--fixup=` has a title composed of "fixup!" followed by the title of __, and is recognized specially by `git rebase --autosquash`. The `-m` -option may be used to supplement the log message of the created -commit, but the additional commentary will be thrown away once the -"fixup!" commit is squashed into __ by +or `-F` option may be used to supplement the log message +of the created commit, but the additional commentary will be thrown +away once the "fixup!" commit is squashed into __ by `git rebase --autosquash`. + The commit created by `--fixup=amend:` is similar but its title is instead prefixed with "amend!". The log message of __ is copied into the log message of the "amend!" commit and -opened in an editor so it can be refined. When `git rebase ---autosquash` squashes the "amend!" commit into __, the -log message of __ is replaced by the refined log message -from the "amend!" commit. It is an error for the "amend!" commit's -log message to be empty unless `--allow-empty-message` is -specified. +opened in an editor so it can be refined. The replacement message may +also be supplied directly using `-m` or `-F`, bypassing the +need to open an editor. When `git rebase +--autosquash` squashes the "amend!" commit into __, the log +message of __ is replaced by the refined log message from the +"amend!" commit. It is an error for the "amend!" commit's log message +to be empty unless `--allow-empty-message` is specified. + `--fixup=reword:` is shorthand for `--fixup=amend: --only`. It creates an "amend!" commit with only a log message diff --git a/builtin/commit.c b/builtin/commit.c index 28f61745034506..3f1fca291968de 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -804,18 +804,18 @@ static int prepare_to_commit(const char *index_file, const char *prefix, if (have_option_m && !fixup_message) { strbuf_addbuf(&sb, &message); hook_arg1 = "message"; - } else if (logfile && !strcmp(logfile, "-")) { + } else if (logfile && !fixup_message && !strcmp(logfile, "-")) { if (isatty(0)) fprintf(stderr, _("(reading log message from standard input)\n")); if (strbuf_read(&sb, 0, 0) < 0) die_errno(_("could not read log from standard input")); hook_arg1 = "message"; - } else if (logfile) { + } else if (logfile && !fixup_message) { if (strbuf_read_file(&sb, logfile, 0) < 0) die_errno(_("could not read log file '%s'"), logfile); hook_arg1 = "message"; - } else if (use_message) { + } else if (use_message && !fixup_message) { const char *buffer; buffer = strstr(use_message_buffer, "\n\n"); if (buffer) @@ -837,20 +837,21 @@ static int prepare_to_commit(const char *index_file, const char *prefix, hook_arg1 = "message"; /* - * Only `-m` commit message option is checked here, as - * it supports `--fixup` to append the commit message. - * - * The other commit message options `-c`/`-C`/`-F` are - * incompatible with all the forms of `--fixup` and - * have already errored out while parsing the `git commit` - * options. + * Only `-m` and `-F` are handled here. `-c`/`-C` are + * incompatible with --fixup and have already errored out + * during option parsing. */ - if (have_option_m && !strcmp(fixup_prefix, "fixup")) + if (have_option_m) { strbuf_addbuf(&sb, &message); - - if (!strcmp(fixup_prefix, "amend")) { - if (have_option_m) - die(_("options '%s' and '%s:%s' cannot be used together"), "-m", "--fixup", fixup_message); + } else if (logfile && !strcmp(logfile, "-")) { + if (isatty(0)) + fprintf(stderr, _("(reading log message from standard input)\n")); + if (strbuf_read(&sb, 0, 0) < 0) + die_errno(_("could not read log from standard input")); + } else if (logfile) { + if (strbuf_read_file(&sb, logfile, 0) < 0) + die_errno(_("could not read log file '%s'"), logfile); + } else if (!strcmp(fixup_prefix, "amend")) { prepare_amend_commit(commit, &sb, &ctx); } } else if (!stat(git_path_merge_msg(the_repository), &statbuf)) { @@ -1338,9 +1339,8 @@ static int parse_and_validate_options(int argc, const char *argv[], } if (fixup_message && squash_message) die(_("options '%s' and '%s' cannot be used together"), "--squash", "--fixup"); - die_for_incompatible_opt4(!!use_message, "-C", + die_for_incompatible_opt3(!!use_message, "-C", !!edit_message, "-c", - !!logfile, "-F", !!fixup_message, "--fixup"); die_for_incompatible_opt4(have_option_m, "-m", !!edit_message, "-c", diff --git a/t/t7500-commit-template-squash-signoff.sh b/t/t7500-commit-template-squash-signoff.sh index 66aff8e0976e79..01c7400136d437 100755 --- a/t/t7500-commit-template-squash-signoff.sh +++ b/t/t7500-commit-template-squash-signoff.sh @@ -384,18 +384,24 @@ test_expect_success '--fixup=reword: ignores staged changes' ' test_cmp foo actual ' -test_expect_success '--fixup=reword: error out with -m option' ' +test_expect_success 'commit --fixup=reword: works with -m' ' commit_for_rebase_autosquash_setup && - echo "fatal: options '\''-m'\'' and '\''--fixup:reword'\'' cannot be used together" >expect && - test_must_fail git commit --fixup=reword:HEAD~ -m "reword commit message" 2>actual && - test_cmp expect actual + git commit --fixup=reword:HEAD~ -m "reword commit message" && + test_commit_message HEAD <<-EOF + amend! $(git log -1 --format=%s HEAD~2) + + reword commit message + EOF ' -test_expect_success '--fixup=amend: error out with -m option' ' +test_expect_success 'commit --fixup=amend: works with -m' ' commit_for_rebase_autosquash_setup && - echo "fatal: options '\''-m'\'' and '\''--fixup:amend'\'' cannot be used together" >expect && - test_must_fail git commit --fixup=amend:HEAD~ -m "amend commit message" 2>actual && - test_cmp expect actual + git commit --fixup=amend:HEAD~ -m "amend commit message" && + test_commit_message HEAD <<-EOF + amend! $(git log -1 --format=%s HEAD~2) + + amend commit message + EOF ' test_expect_success 'consecutive amend! commits remove amend! line from commit msg body' ' @@ -432,6 +438,13 @@ test_expect_success 'deny to create amend! commit if its commit msg body is empt test_cmp expected actual ' +test_expect_success 'deny to create amend! commit if -m is empty' ' + commit_for_rebase_autosquash_setup && + echo "Aborting commit due to empty commit message body." >expect && + test_must_fail git commit --fixup=amend:HEAD~ -m "" 2>actual && + test_cmp expect actual +' + test_expect_success 'amend! commit allows empty commit msg body with --allow-empty-message' ' commit_for_rebase_autosquash_setup && cat >expected <<-EOF && @@ -468,10 +481,26 @@ test_expect_success '--fixup=reword: give error with pathsec' ' test_cmp expect actual ' -test_expect_success '--fixup=reword: -F give error message' ' - echo "fatal: options '\''-F'\'' and '\''--fixup'\'' cannot be used together" >expect && - test_must_fail git commit --fixup=reword:HEAD~ -F msg 2>actual && - test_cmp expect actual +test_expect_success 'commit --fixup works with -F' ' + commit_for_rebase_autosquash_setup && + echo "message" >msgfile && + git commit --fixup HEAD~ -F msgfile && + test_commit_message HEAD <<-EOF + fixup! $(git log -1 --format=%s HEAD~2) + + message + EOF +' + +test_expect_success 'commit --fixup=reword: works with -F' ' + commit_for_rebase_autosquash_setup && + echo "message from file" >msgfile && + git commit --fixup=reword:HEAD~ -F msgfile && + test_commit_message HEAD <<-EOF + amend! $(git log -1 --format=%s HEAD~2) + + $(cat msgfile) + EOF ' test_expect_success 'commit --squash works with -F' ' @@ -526,8 +555,7 @@ test_expect_success 'invalid message options when using --fixup' ' git add foo && test_must_fail git commit --fixup HEAD~1 --squash HEAD~2 && test_must_fail git commit --fixup HEAD~1 -C HEAD~2 && - test_must_fail git commit --fixup HEAD~1 -c HEAD~2 && - test_must_fail git commit --fixup HEAD~1 -F log + test_must_fail git commit --fixup HEAD~1 -c HEAD~2 ' cat >expected-template < Date: Tue, 26 May 2026 12:47:44 +0200 Subject: [PATCH 002/259] commit: allow -c/-C for all kinds of --fixup The previous commit allowed -m and -F for all --fixup variations. The -c/-C flags were blocked by the same higher-layer incompatibility check that previously caught -F, namely die_for_incompatible_opt4() grouping them with --fixup. Drop --fixup from that check and route the resolved commit through prepare_amend_commit() in the fixup path, mirroring the no-message-source behaviour of --fixup=amend. With this in place, -m/-F/-c/-C all behave consistently across the plain, amend, and reword --fixup forms. Signed-off-by: Erik Cervin-Edin Signed-off-by: Junio C Hamano --- Documentation/git-commit.adoc | 9 ++-- builtin/commit.c | 13 +++-- t/t7500-commit-template-squash-signoff.sh | 60 +++++++++++++++++++++-- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/Documentation/git-commit.adoc b/Documentation/git-commit.adoc index 61efd29e66e7d0..98c50a3be5c81c 100644 --- a/Documentation/git-commit.adoc +++ b/Documentation/git-commit.adoc @@ -102,8 +102,8 @@ include::diff-context-options.adoc[] + The commit created by plain `--fixup=` has a title composed of "fixup!" followed by the title of __, -and is recognized specially by `git rebase --autosquash`. The `-m` -or `-F` option may be used to supplement the log message +and is recognized specially by `git rebase --autosquash`. The `-m`, +`-F`, `-C`, or `-c` option may be used to supplement the log message of the created commit, but the additional commentary will be thrown away once the "fixup!" commit is squashed into __ by `git rebase --autosquash`. @@ -112,8 +112,9 @@ The commit created by `--fixup=amend:` is similar but its title is instead prefixed with "amend!". The log message of __ is copied into the log message of the "amend!" commit and opened in an editor so it can be refined. The replacement message may -also be supplied directly using `-m` or `-F`, bypassing the -need to open an editor. When `git rebase +also be supplied directly using `-m`, `-F`, or `-C`, bypassing the +need to open an editor, or using `-c` to open the editor pre-populated +with the referenced commit's message. When `git rebase --autosquash` squashes the "amend!" commit into __, the log message of __ is replaced by the refined log message from the "amend!" commit. It is an error for the "amend!" commit's log message diff --git a/builtin/commit.c b/builtin/commit.c index 3f1fca291968de..fcf148eb21b95b 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -837,9 +837,9 @@ static int prepare_to_commit(const char *index_file, const char *prefix, hook_arg1 = "message"; /* - * Only `-m` and `-F` are handled here. `-c`/`-C` are - * incompatible with --fixup and have already errored out - * during option parsing. + * `-m`, `-F`, `-C`, and `-c` provide the message body. + * If none was given and this is an amend, use the target + * commit's body instead. */ if (have_option_m) { strbuf_addbuf(&sb, &message); @@ -851,6 +851,11 @@ static int prepare_to_commit(const char *index_file, const char *prefix, } else if (logfile) { if (strbuf_read_file(&sb, logfile, 0) < 0) die_errno(_("could not read log file '%s'"), logfile); + } else if (use_message) { + struct commit *c = lookup_commit_reference_by_name(use_message); + if (!c) + die(_("could not lookup commit '%s'"), use_message); + prepare_amend_commit(c, &sb, &ctx); } else if (!strcmp(fixup_prefix, "amend")) { prepare_amend_commit(commit, &sb, &ctx); } @@ -1341,7 +1346,7 @@ static int parse_and_validate_options(int argc, const char *argv[], die(_("options '%s' and '%s' cannot be used together"), "--squash", "--fixup"); die_for_incompatible_opt3(!!use_message, "-C", !!edit_message, "-c", - !!fixup_message, "--fixup"); + !!logfile, "-F"); die_for_incompatible_opt4(have_option_m, "-m", !!edit_message, "-c", !!use_message, "-C", diff --git a/t/t7500-commit-template-squash-signoff.sh b/t/t7500-commit-template-squash-signoff.sh index 01c7400136d437..48e1247d9ebb5a 100755 --- a/t/t7500-commit-template-squash-signoff.sh +++ b/t/t7500-commit-template-squash-signoff.sh @@ -492,6 +492,62 @@ test_expect_success 'commit --fixup works with -F' ' EOF ' +test_expect_success 'commit --fixup works with -C' ' + commit_for_rebase_autosquash_setup && + git commit --fixup HEAD~ -C HEAD && + test_commit_message HEAD <<-EOF + fixup! $(git log -1 --format=%s HEAD~2) + + $(get_commit_msg HEAD~) + EOF +' + +test_expect_success 'commit --fixup=amend: works with -c' ' + commit_for_rebase_autosquash_setup && + test_set_editor : && + git commit --fixup=amend:HEAD -c HEAD~ && + test_commit_message HEAD <<-EOF + amend! intermediate commit + + target message subject line + + target message body line 1 + target message body line 2 + EOF +' + +test_expect_success 'commit --fixup=amend:HEAD with -C HEAD and without have the same message' ' + commit_for_rebase_autosquash_setup && + start=$(git rev-parse HEAD) && + + git commit --fixup=amend:HEAD -C HEAD && + git commit --fixup=amend:HEAD -C HEAD && + git log -1 --pretty=%B >with-c && + + git reset --hard "$start" && + test_set_editor : && + git commit --fixup=amend:HEAD && + git commit --fixup=amend:HEAD && + git log -1 --pretty=%B >without-c && + + test_cmp with-c without-c +' + +test_expect_success 'commit --fixup=amend: with -C copies full subject + body of squash commit' ' + commit_for_rebase_autosquash_setup && + git commit --squash HEAD~ -m "inner body" && + echo "extra" >>foo && + git add foo && + git commit --fixup=amend:HEAD -C HEAD && + test_commit_message HEAD <<-EOF + amend! squash! $(git log -1 --format=%s HEAD~3) + + squash! $(git log -1 --format=%s HEAD~3) + + inner body + EOF +' + test_expect_success 'commit --fixup=reword: works with -F' ' commit_for_rebase_autosquash_setup && echo "message from file" >msgfile && @@ -553,9 +609,7 @@ test_expect_success 'invalid message options when using --fixup' ' echo changes >>foo && echo "message" >log && git add foo && - test_must_fail git commit --fixup HEAD~1 --squash HEAD~2 && - test_must_fail git commit --fixup HEAD~1 -C HEAD~2 && - test_must_fail git commit --fixup HEAD~1 -c HEAD~2 + test_must_fail git commit --fixup HEAD~1 --squash HEAD~2 ' cat >expected-template < Date: Fri, 12 Jun 2026 16:07:08 -0400 Subject: [PATCH 003/259] t5334: expose shared `nth_line()` helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit 0cd2255e64b (midx: support custom `--base` for incremental MIDX writes, 2026-05-19), t5334 has referred to a non-existent helper function 'nth_line', which is defined in t5335, but not here. Move the helper to lib-midx.sh so that both tests can use the same implementation. Ensure likewise that `nth_line()` remains visible from within t5335 by sourcing lib-midx.sh there appropriately. Curiously, t5334 passes both before and after this change. Before this change, the failed command substitution leaves '--base' with an empty value, and after this change, the custom base value is still ignored by the normal incremental write path. The following commits will explain and address that behavior. Noticed-by: SZEDER Gábor Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- t/lib-midx.sh | 6 ++++++ t/t5335-compact-multi-pack-index.sh | 7 +------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/t/lib-midx.sh b/t/lib-midx.sh index e38c609604c301..b522dbdb0f431a 100644 --- a/t/lib-midx.sh +++ b/t/lib-midx.sh @@ -34,3 +34,9 @@ compare_results_with_midx () { midx_git_two_modes "cat-file --batch-all-objects --batch-check --unordered" sorted ' } + +nth_line() { + local n="$1" + shift + awk "NR==$n" "$@" +} diff --git a/t/t5335-compact-multi-pack-index.sh b/t/t5335-compact-multi-pack-index.sh index ec1dafe89fcfce..6a4b799b9c9f49 100755 --- a/t/t5335-compact-multi-pack-index.sh +++ b/t/t5335-compact-multi-pack-index.sh @@ -3,6 +3,7 @@ test_description='multi-pack-index compaction' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-midx.sh GIT_TEST_MULTI_PACK_INDEX=0 GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0 @@ -13,12 +14,6 @@ packdir=$objdir/pack midxdir=$packdir/multi-pack-index.d midx_chain=$midxdir/multi-pack-index-chain -nth_line() { - local n="$1" - shift - awk "NR==$n" "$@" -} - write_packs () { for c in "$@" do From 8e519b87565a8689d509693a049bc6dad632dbb3 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 12 Jun 2026 16:07:11 -0400 Subject: [PATCH 004/259] midx: pass custom '--base' through incremental writes The 'multi-pack-index' builtin parses '--base' for incremental writes, but the normal write path does not pass that value through to `write_midx_file()`. As a result, something like: $ git multi-pack-index write --incremental --base= behaves as if no custom base had been given (unless the caller used the '--stdin-packs' path). Thread the parsed base through `write_midx_file()`, and update the repack caller to pass NULL for the new argument where no custom base selection is needed. This exposes a pre-existing problem in incremental writes with custom bases: the writer skips packs from the full existing MIDX chain, even when the caller selected an older base or no base at all. The affected t5334 cases fail while trying to write MIDX bitmaps. The detached layer omits packs above the selected base, and thus the resulting MIDX does not have a reachability closure, making it impossible to generate reachability bitmaps. Mark those tests as expected failures accordingly. The following commit will fix the broken behavior and restore these tests. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- builtin/multi-pack-index.c | 3 ++- builtin/repack.c | 2 +- midx-write.c | 2 ++ midx.h | 2 +- t/t5334-incremental-multi-pack-index.sh | 24 +++++++++++++++++++----- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 00ffb36394d08c..949bfa796b28a5 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -224,7 +224,8 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, } ret = write_midx_file(source, opts.preferred_pack, - opts.refs_snapshot, opts.flags); + opts.refs_snapshot, opts.incremental_base, + opts.flags); free(opts.refs_snapshot); return ret; diff --git a/builtin/repack.c b/builtin/repack.c index 1524a9c13ad5b8..0092a72a996cae 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -629,7 +629,7 @@ int cmd_repack(int argc, unsigned flags = 0; if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0)) flags |= MIDX_WRITE_INCREMENTAL; - write_midx_file(existing.source, NULL, NULL, flags); + write_midx_file(existing.source, NULL, NULL, NULL, flags); } cleanup: diff --git a/midx-write.c b/midx-write.c index 561e9eedc0e6ef..aa438775ebd23f 100644 --- a/midx-write.c +++ b/midx-write.c @@ -1850,12 +1850,14 @@ static int write_midx_internal(struct write_midx_opts *opts) int write_midx_file(struct odb_source *source, const char *preferred_pack_name, const char *refs_snapshot, + const char *incremental_base, unsigned flags) { struct write_midx_opts opts = { .source = source, .preferred_pack_name = preferred_pack_name, .refs_snapshot = refs_snapshot, + .incremental_base = incremental_base, .flags = flags, }; diff --git a/midx.h b/midx.h index 63853a03a47fd1..92ed29d913d9a3 100644 --- a/midx.h +++ b/midx.h @@ -131,7 +131,7 @@ int prepare_multi_pack_index_one(struct odb_source *source); */ int write_midx_file(struct odb_source *source, const char *preferred_pack_name, const char *refs_snapshot, - unsigned flags); + const char *incremental_base, unsigned flags); int write_midx_file_only(struct odb_source *source, struct string_list *packs_to_include, const char *preferred_pack_name, diff --git a/t/t5334-incremental-multi-pack-index.sh b/t/t5334-incremental-multi-pack-index.sh index 68a103d13d23c3..69e96bf8d93713 100755 --- a/t/t5334-incremental-multi-pack-index.sh +++ b/t/t5334-incremental-multi-pack-index.sh @@ -119,7 +119,7 @@ test_expect_success 'write MIDX layer with --base without --no-write-chain-file' test_grep "cannot use --base without --no-write-chain-file" err ' -test_expect_success 'write MIDX layer with --base=none and --no-write-chain-file' ' +test_expect_failure 'write MIDX layer with --base=none and --no-write-chain-file' ' test_commit base-none && git repack -d && @@ -128,19 +128,33 @@ test_expect_success 'write MIDX layer with --base=none and --no-write-chain-file --no-write-chain-file --base=none)" && test_cmp "$midx_chain.bak" "$midx_chain" && - test_path_is_file "$midxdir/multi-pack-index-$layer.midx" + test_path_is_file "$midxdir/multi-pack-index-$layer.midx" && + + echo "$layer" >"$midx_chain" && + test-tool read-midx --show-objects "$objdir" "$layer" >midx.objects && + test_grep "^$(git rev-parse 2.2) " midx.objects && + cp "$midx_chain.bak" "$midx_chain" ' -test_expect_success 'write MIDX layer with --base= and --no-write-chain-file' ' +test_expect_failure 'write MIDX layer with --base= and --no-write-chain-file' ' test_commit base-hash && git repack -d && cp "$midx_chain" "$midx_chain.bak" && + base="$(nth_line 1 "$midx_chain")" && layer="$(git multi-pack-index write --bitmap --incremental \ - --no-write-chain-file --base="$(nth_line 1 "$midx_chain")")" && + --no-write-chain-file --base="$base")" && test_cmp "$midx_chain.bak" "$midx_chain" && - test_path_is_file "$midxdir/multi-pack-index-$layer.midx" + test_path_is_file "$midxdir/multi-pack-index-$layer.midx" && + + { + echo "$base" && + echo "$layer" + } >"$midx_chain" && + test-tool read-midx --show-objects "$objdir" "$layer" >midx.objects && + test_grep "^$(git rev-parse 2.2) " midx.objects && + cp "$midx_chain.bak" "$midx_chain" ' for reuse in false single multi From 6afc679fa62362ddaab955778d973c85a6cbf004 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 12 Jun 2026 16:07:14 -0400 Subject: [PATCH 005/259] midx-write: include packs above custom incremental base The previous commit made '--base' take effect on the normal incremental write path, which exposed an existing assumption in our helper function `should_include_pack()`, which is that any pack already present in `ctx->m` was skipped. That is only correct for non-incremental writes. For incremental writes, `ctx->base_midx` is the boundary that should be excluded from the new layer. If the caller selects an older base, or no base at all, then packs from layers above that base have to be included in the detached layer so that its bitmap has reachability closure. Teach `should_include_pack()` to choose the MIDX used for pack exclusion based on whether or not we are performing an incremental write. When doing so, use `ctx->base_midx`, and use `ctx->m` otherwise. The t5334 cases from the previous commit can now be marked as successful. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- midx-write.c | 16 +++++++++++----- t/t5334-incremental-multi-pack-index.sh | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/midx-write.c b/midx-write.c index aa438775ebd23f..c50fdb5c6d17c3 100644 --- a/midx-write.c +++ b/midx-write.c @@ -133,8 +133,17 @@ static uint32_t midx_pack_perm(struct write_midx_context *ctx, static int should_include_pack(const struct write_midx_context *ctx, const char *file_name) { + struct multi_pack_index *m = ctx->m; /* - * Note that at most one of ctx->m and ctx->to_include are set, + * When writing incrementally, ctx->m may contain layers above + * the selected base MIDX, which must be included in the new + * layer. + */ + if (ctx->incremental) + m = ctx->base_midx; + + /* + * Note that at most one of m and ctx->to_include are set, * so we are testing midx_contains_pack() and * string_list_has_string() independently (guarded by the * appropriate NULL checks). @@ -148,10 +157,7 @@ static int should_include_pack(const struct write_midx_context *ctx, * should be performed independently (likely checking * to_include before the existing MIDX). */ - if (ctx->m && midx_contains_pack(ctx->m, file_name)) - return 0; - else if (ctx->base_midx && midx_contains_pack(ctx->base_midx, - file_name)) + if (m && midx_contains_pack(m, file_name)) return 0; else if (ctx->to_include && !string_list_has_string(ctx->to_include, file_name)) diff --git a/t/t5334-incremental-multi-pack-index.sh b/t/t5334-incremental-multi-pack-index.sh index 69e96bf8d93713..84ff612097888f 100755 --- a/t/t5334-incremental-multi-pack-index.sh +++ b/t/t5334-incremental-multi-pack-index.sh @@ -119,7 +119,7 @@ test_expect_success 'write MIDX layer with --base without --no-write-chain-file' test_grep "cannot use --base without --no-write-chain-file" err ' -test_expect_failure 'write MIDX layer with --base=none and --no-write-chain-file' ' +test_expect_success 'write MIDX layer with --base=none and --no-write-chain-file' ' test_commit base-none && git repack -d && @@ -136,7 +136,7 @@ test_expect_failure 'write MIDX layer with --base=none and --no-write-chain-file cp "$midx_chain.bak" "$midx_chain" ' -test_expect_failure 'write MIDX layer with --base= and --no-write-chain-file' ' +test_expect_success 'write MIDX layer with --base= and --no-write-chain-file' ' test_commit base-hash && git repack -d && From 6fa5fbaf2fc88582d070adc2b0e765682206a984 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:28:55 +0000 Subject: [PATCH 006/259] diff: rename and group the line-range filter for clarity The line-range filter that mm/line-log-cleanup added uses names that obscure its model. The cursors lno_post/lno_pre and the index lno_0 share an lno_ prefix but conflate the pre/post-image axis with the 0-based/1-based axis, the hunk state is a flat set of rhunk_* fields, and the filter-state pointer is just s. The filter bridges two layers of diff.c, and its fields already used each layer's vocabulary, but in cryptic abbreviations. Spell them out to the form the rest of the file uses, so that the patches that follow can simplify and fix it with those clearer names in place: - lno_post/lno_pre -> lno_in_postimage/lno_in_preimage, the line-number cursors, matching the counters in struct emit_callback - lno_0 -> idx_in_postimage, the 0-based range index - the hunk-header geometry stays old/new (old_begin, new_begin, and counts) to match the xdiff_emit_hunk_fn callback and the "@@ - + @@" header it feeds, but moves from flat rhunk_* fields into a "hunk" sub-struct, so accesses read filter->hunk.old_begin - flush_rhunk -> flush_range_hunk - the filter-state pointer in each callback: s -> filter Also rename the struct line_range_callback to line_range_filter: it is a filter over xdiff output, not merely a callback. No behavior change. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- diff.c | 192 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 97 insertions(+), 95 deletions(-) diff --git a/diff.c b/diff.c index 5a584fa1d569e7..1e043c959ff2b4 100644 --- a/diff.c +++ b/diff.c @@ -623,15 +623,15 @@ struct emit_callback { * reveals whether they precede an in-range line (flush into range hunk) or * an out-of-range line (discard). */ -struct line_range_callback { +struct line_range_filter { xdiff_emit_line_fn orig_line_fn; void *orig_cb_data; const struct range_set *ranges; /* 0-based [start, end) */ unsigned int cur_range; /* index into the range_set */ /* Post/pre-image line counters (1-based, set from hunk headers) */ - long lno_post; - long lno_pre; + long lno_in_postimage; + long lno_in_preimage; /* * Function name from most recent xdiff hunk header; @@ -640,12 +640,14 @@ struct line_range_callback { char func[80]; long funclen; - /* Range hunk being accumulated for the current range */ - struct strbuf rhunk; - long rhunk_old_begin, rhunk_old_count; - long rhunk_new_begin, rhunk_new_count; - int rhunk_active; - int rhunk_has_changes; /* any '+' or '-' lines? */ + /* The range hunk being accumulated for the current range. */ + struct { + struct strbuf lines; /* buffered in-range diff lines */ + long old_begin, old_count; + long new_begin, new_count; + int active; + int has_changes; /* any '+' or '-' line? */ + } hunk; /* Removal lines not yet known to be in-range */ struct strbuf pending_rm; @@ -2540,26 +2542,26 @@ static int quick_consume(void *priv, char *line UNUSED, unsigned long len UNUSED return 1; } -static void discard_pending_rm(struct line_range_callback *s) +static void discard_pending_rm(struct line_range_filter *filter) { - strbuf_reset(&s->pending_rm); - s->pending_rm_count = 0; + strbuf_reset(&filter->pending_rm); + filter->pending_rm_count = 0; } -static void flush_rhunk(struct line_range_callback *s) +static void flush_range_hunk(struct line_range_filter *filter) { struct strbuf hdr = STRBUF_INIT; const char *p, *end; - if (!s->rhunk_active || s->ret) + if (!filter->hunk.active || filter->ret) return; /* Drain any pending removal lines into the range hunk */ - if (s->pending_rm_count) { - strbuf_addbuf(&s->rhunk, &s->pending_rm); - s->rhunk_old_count += s->pending_rm_count; - s->rhunk_has_changes = 1; - discard_pending_rm(s); + if (filter->pending_rm_count) { + strbuf_addbuf(&filter->hunk.lines, &filter->pending_rm); + filter->hunk.old_count += filter->pending_rm_count; + filter->hunk.has_changes = 1; + discard_pending_rm(filter); } /* @@ -2568,22 +2570,22 @@ static void flush_rhunk(struct line_range_callback *s) * ctxlen causes xdiff to emit context covering a range that * has no changes in this commit. */ - if (!s->rhunk_has_changes) { - s->rhunk_active = 0; - strbuf_reset(&s->rhunk); + if (!filter->hunk.has_changes) { + filter->hunk.active = 0; + strbuf_reset(&filter->hunk.lines); return; } strbuf_addf(&hdr, "@@ -%ld,%ld +%ld,%ld @@", - s->rhunk_old_begin, s->rhunk_old_count, - s->rhunk_new_begin, s->rhunk_new_count); - if (s->funclen > 0) { + filter->hunk.old_begin, filter->hunk.old_count, + filter->hunk.new_begin, filter->hunk.new_count); + if (filter->funclen > 0) { strbuf_addch(&hdr, ' '); - strbuf_add(&hdr, s->func, s->funclen); + strbuf_add(&hdr, filter->func, filter->funclen); } strbuf_addch(&hdr, '\n'); - s->ret = s->orig_line_fn(s->orig_cb_data, hdr.buf, hdr.len); + filter->ret = filter->orig_line_fn(filter->orig_cb_data, hdr.buf, hdr.len); strbuf_release(&hdr); /* @@ -2591,18 +2593,18 @@ static void flush_rhunk(struct line_range_callback *s) * The cast discards const because xdiff_emit_line_fn takes * char *, though fn_out_consume does not modify the buffer. */ - p = s->rhunk.buf; - end = p + s->rhunk.len; - while (!s->ret && p < end) { + p = filter->hunk.lines.buf; + end = p + filter->hunk.lines.len; + while (!filter->ret && p < end) { const char *eol = memchr(p, '\n', end - p); unsigned long line_len = eol ? (unsigned long)(eol - p + 1) : (unsigned long)(end - p); - s->ret = s->orig_line_fn(s->orig_cb_data, (char *)p, line_len); + filter->ret = filter->orig_line_fn(filter->orig_cb_data, (char *)p, line_len); p += line_len; } - s->rhunk_active = 0; - strbuf_reset(&s->rhunk); + filter->hunk.active = 0; + strbuf_reset(&filter->hunk.lines); } static void line_range_hunk_fn(void *data, @@ -2610,7 +2612,7 @@ static void line_range_hunk_fn(void *data, long new_begin, long new_nr UNUSED, const char *func, long funclen) { - struct line_range_callback *s = data; + struct line_range_filter *filter = data; /* * When count > 0, begin is 1-based. When count == 0, begin is @@ -2622,104 +2624,104 @@ static void line_range_hunk_fn(void *data, * flush or discard them when the next content line reveals * whether the removals precede in-range content. */ - s->lno_post = new_begin; - s->lno_pre = old_begin; + filter->lno_in_postimage = new_begin; + filter->lno_in_preimage = old_begin; if (funclen > 0) { - if (funclen > (long)sizeof(s->func)) - funclen = sizeof(s->func); - memcpy(s->func, func, funclen); + if (funclen > (long)sizeof(filter->func)) + funclen = sizeof(filter->func); + memcpy(filter->func, func, funclen); } - s->funclen = funclen; + filter->funclen = funclen; } static int line_range_line_fn(void *priv, char *line, unsigned long len) { - struct line_range_callback *s = priv; + struct line_range_filter *filter = priv; const struct range *cur; - long lno_0, cur_pre; + long idx_in_postimage, cur_pre; - if (s->ret) - return s->ret; + if (filter->ret) + return filter->ret; if (line[0] == '-') { - if (!s->pending_rm_count) - s->pending_rm_pre_begin = s->lno_pre; - s->lno_pre++; - strbuf_add(&s->pending_rm, line, len); - s->pending_rm_count++; - return s->ret; + if (!filter->pending_rm_count) + filter->pending_rm_pre_begin = filter->lno_in_preimage; + filter->lno_in_preimage++; + strbuf_add(&filter->pending_rm, line, len); + filter->pending_rm_count++; + return filter->ret; } if (line[0] == '\\') { - if (s->pending_rm_count) - strbuf_add(&s->pending_rm, line, len); - else if (s->rhunk_active) - strbuf_add(&s->rhunk, line, len); + if (filter->pending_rm_count) + strbuf_add(&filter->pending_rm, line, len); + else if (filter->hunk.active) + strbuf_add(&filter->hunk.lines, line, len); /* otherwise outside tracked range; drop silently */ - return s->ret; + return filter->ret; } if (line[0] != '+' && line[0] != ' ') BUG("unexpected diff line type '%c'", line[0]); - lno_0 = s->lno_post - 1; - cur_pre = s->lno_pre; /* save before advancing for context lines */ - s->lno_post++; + idx_in_postimage = filter->lno_in_postimage - 1; + cur_pre = filter->lno_in_preimage; /* save before advancing for context lines */ + filter->lno_in_postimage++; if (line[0] == ' ') - s->lno_pre++; + filter->lno_in_preimage++; /* Advance past ranges we've passed */ - while (s->cur_range < s->ranges->nr && - lno_0 >= s->ranges->ranges[s->cur_range].end) { - if (s->rhunk_active) - flush_rhunk(s); - discard_pending_rm(s); - s->cur_range++; + while (filter->cur_range < filter->ranges->nr && + idx_in_postimage >= filter->ranges->ranges[filter->cur_range].end) { + if (filter->hunk.active) + flush_range_hunk(filter); + discard_pending_rm(filter); + filter->cur_range++; } /* Past all ranges */ - if (s->cur_range >= s->ranges->nr) { - discard_pending_rm(s); - return s->ret; + if (filter->cur_range >= filter->ranges->nr) { + discard_pending_rm(filter); + return filter->ret; } - cur = &s->ranges->ranges[s->cur_range]; + cur = &filter->ranges->ranges[filter->cur_range]; /* Before current range */ - if (lno_0 < cur->start) { - discard_pending_rm(s); - return s->ret; + if (idx_in_postimage < cur->start) { + discard_pending_rm(filter); + return filter->ret; } /* In range so start a new range hunk if needed */ - if (!s->rhunk_active) { - s->rhunk_active = 1; - s->rhunk_has_changes = 0; - s->rhunk_new_begin = lno_0 + 1; - s->rhunk_old_begin = s->pending_rm_count - ? s->pending_rm_pre_begin : cur_pre; - s->rhunk_old_count = 0; - s->rhunk_new_count = 0; - strbuf_reset(&s->rhunk); + if (!filter->hunk.active) { + filter->hunk.active = 1; + filter->hunk.has_changes = 0; + filter->hunk.new_begin = idx_in_postimage + 1; + filter->hunk.old_begin = filter->pending_rm_count + ? filter->pending_rm_pre_begin : cur_pre; + filter->hunk.old_count = 0; + filter->hunk.new_count = 0; + strbuf_reset(&filter->hunk.lines); } /* Flush pending removals into range hunk */ - if (s->pending_rm_count) { - strbuf_addbuf(&s->rhunk, &s->pending_rm); - s->rhunk_old_count += s->pending_rm_count; - s->rhunk_has_changes = 1; - discard_pending_rm(s); + if (filter->pending_rm_count) { + strbuf_addbuf(&filter->hunk.lines, &filter->pending_rm); + filter->hunk.old_count += filter->pending_rm_count; + filter->hunk.has_changes = 1; + discard_pending_rm(filter); } - strbuf_add(&s->rhunk, line, len); - s->rhunk_new_count++; + strbuf_add(&filter->hunk.lines, line, len); + filter->hunk.new_count++; if (line[0] == '+') - s->rhunk_has_changes = 1; + filter->hunk.has_changes = 1; else - s->rhunk_old_count++; + filter->hunk.old_count++; - return s->ret; + return filter->ret; } static void pprint_rename(struct strbuf *name, const char *a, const char *b) @@ -4086,7 +4088,7 @@ static void builtin_diff(const char *name_a, xdi_diff_outf(&mf1, &mf2, NULL, quick_consume, &ecbdata, &xpp, &xecfg); } else if (line_ranges) { - struct line_range_callback lr_state; + struct line_range_filter lr_state; unsigned int i; long max_span = 0; @@ -4094,7 +4096,7 @@ static void builtin_diff(const char *name_a, lr_state.orig_line_fn = fn_out_consume; lr_state.orig_cb_data = &ecbdata; lr_state.ranges = line_ranges; - strbuf_init(&lr_state.rhunk, 0); + strbuf_init(&lr_state.hunk.lines, 0); strbuf_init(&lr_state.pending_rm, 0); /* @@ -4125,11 +4127,11 @@ static void builtin_diff(const char *name_a, die("unable to generate diff for %s", one->path); - flush_rhunk(&lr_state); + flush_range_hunk(&lr_state); if (lr_state.ret) die("unable to generate diff for %s", one->path); - strbuf_release(&lr_state.rhunk); + strbuf_release(&lr_state.hunk.lines); strbuf_release(&lr_state.pending_rm); } else if (xdi_diff_outf(&mf1, &mf2, NULL, fn_out_consume, &ecbdata, &xpp, &xecfg)) From 5a508c13ac159a6c8e938ba6ce544a33f234e58b Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:28:56 +0000 Subject: [PATCH 007/259] diff: simplify the line-range filter by classifying removals immediately The filter buffered '-' lines in a pending_rm strbuf, deferring their classification until a '+' or ' ' line revealed the post-image position. That buffering is unnecessary: a removal occupies no post-image line, so it does not advance lno_in_postimage, and xdiff emits removals before additions within a change. A '-' therefore arrives while lno_in_postimage already holds the index the following '+'/' ' will occupy, and can be classified against the ranges as it arrives. The buffering also hid a bug: flush_range_hunk() drained pending_rm into the range hunk whenever the hunk was active, even after lno_in_postimage had advanced past the tracked range, so a deletion just after the tracked function leaked into the patch. Classifying each line as it arrives removes the pending_rm buffer, the discard_pending_rm() helper, three struct fields, and makes that bug impossible by construction. With every line classified on arrival, the buffered lines are the hunk's single source of truth, so the old/new counts need not be kept alongside them: flush_range_hunk() derives the counts (and whether the hunk holds any change) from the buffer when it builds the header. Drop the per-line counting and the old_count, new_count, and has_changes fields; there is no longer a second tally that could fall out of sync with the buffer. Add begin_range_hunk() to open the accumulator at the first in-range line, seeding both begins from the live image cursors, as the counterpart to flush_range_hunk(). With the counting gone too, line_range_line_fn() now only appends an in-range line. Document the coordinate model: a block comment on struct line_range_filter states it (the pre/post-image cursors, the 0-based idx_in_postimage, removals classified by the following line) with a worked example. Add tests for the leaked trailing deletion this fixes, the symmetric leading-deletion case, and the filter's range boundaries (a change at the first and last line of a range, and a pure in-range deletion). Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- diff.c | 215 ++++++++++++++++++++++++-------------------- t/t4211-line-log.sh | 125 ++++++++++++++++++++++++++ 2 files changed, 243 insertions(+), 97 deletions(-) diff --git a/diff.c b/diff.c index 1e043c959ff2b4..ee765d7ac2183d 100644 --- a/diff.c +++ b/diff.c @@ -610,18 +610,58 @@ struct emit_callback { }; /* - * State for the line-range callback wrappers that sit between - * xdi_diff_outf() and fn_out_consume(). xdiff produces a normal, - * unfiltered diff; the wrappers intercept each hunk header and line, - * track post-image position, and forward only lines that fall within - * the requested ranges. Contiguous in-range lines are collected into - * range hunks and flushed with a synthetic @@ header so that - * fn_out_consume() sees well-formed unified-diff fragments. + * Line-range filter: scopes "git log -L" output to the tracked ranges. * - * Removal lines ('-') cannot be classified by post-image position, so - * they are buffered in pending_rm until the next '+' or ' ' line - * reveals whether they precede an in-range line (flush into range hunk) or - * an out-of-range line (discard). + * It sits between xdi_diff_outf() and an output callback (fn_out_consume, + * diffstat_consume, checkdiff_consume). xdiff produces a normal diff; the + * filter forwards only the lines inside the requested ranges, collecting + * contiguous in-range lines into a "range hunk" emitted with a synthetic + * @@ header so the callback sees well-formed unified-diff fragments. + * + * A diff describes the change from a pre-image to a post-image. Each + * line is context (' ', in both), a removal ('-', pre-image only), or + * an addition ('+', post-image only). -L tracks ranges in the + * post-image, so a line is in range by its post-image position. + * + * Two 1-based cursors track the next line in each image, named as in + * struct emit_callback and seeded from the xdiff hunk header: + * + * lno_in_postimage advances on '+' and ' ' (lines in the post-image) + * lno_in_preimage advances on '-' and ' ' (lines in the pre-image) + * + * Ranges are 0-based half-open [start, end), so a line is tested at the + * 0-based index idx_in_postimage = lno_in_postimage - 1. + * + * A '-' is not present in the post-image, so it has no post-image line + * number of its own. Since it does not advance lno_in_postimage, it is + * classified at the idx_in_postimage that the following '+'/' ' will + * occupy. xdiff emits a change's removals before its additions, so that + * index is already known when the '-' arrives. + * + * The synthetic "@@ - + @@" header has two sides, old (the + * pre-image) and new (the post-image), matching the xdiff_emit_hunk_fn + * callback; the hunk.old_begin / hunk.new_begin fields below hold those + * begins, and flush_range_hunk() derives the counts from the buffered + * lines. + * + * Example, tracking post-image line 2 (range [1, 2)) of: + * + * pre-image post-image + * 1 a 1 a + * 2 b 2 X (b -> X) + * 3 c 3 c + * + * classify each line by idx_in_postimage. The pre and post columns + * are each cursor's value while that line is classified, i.e. before + * the line advances them (pre = lno_in_preimage, + * post = lno_in_postimage, idx = idx_in_postimage): + * ' a' pre 1 post 1 idx 0 -> before start, skip + * '-b' pre 2 post 2 idx 1 -> keep (removal) + * '+X' pre 3 post 2 idx 1 -> keep (addition) + * ' c' pre 3 post 3 idx 2 -> past end, flush + * + * -b and +X share idx = 1 because -b did not advance lno_in_postimage; + * both land in the range hunk, flushed when ' c' crosses the range end. */ struct line_range_filter { xdiff_emit_line_fn orig_line_fn; @@ -640,20 +680,18 @@ struct line_range_filter { char func[80]; long funclen; - /* The range hunk being accumulated for the current range. */ + /* + * The range hunk being accumulated. At most one is live at a time: + * it is flushed and reset as the cursor leaves each range (and once + * more at end of diff), then reused for the next range. + */ struct { struct strbuf lines; /* buffered in-range diff lines */ - long old_begin, old_count; - long new_begin, new_count; + long old_begin; + long new_begin; int active; - int has_changes; /* any '+' or '-' line? */ } hunk; - /* Removal lines not yet known to be in-range */ - struct strbuf pending_rm; - int pending_rm_count; - long pending_rm_pre_begin; /* pre-image line of first pending */ - int ret; /* latched error from orig_line_fn */ }; @@ -2542,26 +2580,48 @@ static int quick_consume(void *priv, char *line UNUSED, unsigned long len UNUSED return 1; } -static void discard_pending_rm(struct line_range_filter *filter) +/* + * Begin a range hunk at the first in-range line. Its position fixes the + * hunk's begins, taken from the two image cursors before they advance: + * new_begin from the post-image, old_begin from the pre-image. The line + * counts are not tracked here; flush_range_hunk() derives them from the + * buffered lines. + */ +static void begin_range_hunk(struct line_range_filter *filter) { - strbuf_reset(&filter->pending_rm); - filter->pending_rm_count = 0; + filter->hunk.active = 1; + filter->hunk.new_begin = filter->lno_in_postimage; + filter->hunk.old_begin = filter->lno_in_preimage; + strbuf_reset(&filter->hunk.lines); } static void flush_range_hunk(struct line_range_filter *filter) { struct strbuf hdr = STRBUF_INIT; const char *p, *end; + long old_count = 0, new_count = 0; + int has_changes = 0; if (!filter->hunk.active || filter->ret) return; - /* Drain any pending removal lines into the range hunk */ - if (filter->pending_rm_count) { - strbuf_addbuf(&filter->hunk.lines, &filter->pending_rm); - filter->hunk.old_count += filter->pending_rm_count; - filter->hunk.has_changes = 1; - discard_pending_rm(filter); + /* + * Derive the hunk's geometry from the buffered lines: a ' ' + * counts on both sides, a '-' on the old side, a '+' on the new. + * A '-' or '+' marks a real change; the "\ No newline at end of + * file" marker (line[0] == '\\') counts on neither side. + */ + p = filter->hunk.lines.buf; + end = p + filter->hunk.lines.len; + while (p < end) { + const char *eol = memchr(p, '\n', end - p); + if (*p == ' ' || *p == '-') + old_count++; + if (*p == ' ' || *p == '+') + new_count++; + if (*p == '-' || *p == '+') + has_changes = 1; + p = eol ? eol + 1 : end; } /* @@ -2570,15 +2630,15 @@ static void flush_range_hunk(struct line_range_filter *filter) * ctxlen causes xdiff to emit context covering a range that * has no changes in this commit. */ - if (!filter->hunk.has_changes) { + if (!has_changes) { filter->hunk.active = 0; strbuf_reset(&filter->hunk.lines); return; } strbuf_addf(&hdr, "@@ -%ld,%ld +%ld,%ld @@", - filter->hunk.old_begin, filter->hunk.old_count, - filter->hunk.new_begin, filter->hunk.new_count); + filter->hunk.old_begin, old_count, + filter->hunk.new_begin, new_count); if (filter->funclen > 0) { strbuf_addch(&hdr, ' '); strbuf_add(&hdr, filter->func, filter->funclen); @@ -2618,11 +2678,6 @@ static void line_range_hunk_fn(void *data, * When count > 0, begin is 1-based. When count == 0, begin is * adjusted down by 1 by xdl_emit_hunk_hdr(), but no lines of * that type will arrive, so the value is unused. - * - * Any pending removal lines from the previous xdiff hunk are - * intentionally left in pending_rm: the line callback will - * flush or discard them when the next content line reveals - * whether the removals precede in-range content. */ filter->lno_in_postimage = new_begin; filter->lno_in_preimage = old_begin; @@ -2638,88 +2693,56 @@ static void line_range_hunk_fn(void *data, static int line_range_line_fn(void *priv, char *line, unsigned long len) { struct line_range_filter *filter = priv; - const struct range *cur; - long idx_in_postimage, cur_pre; + long idx_in_postimage; + int in_range; if (filter->ret) return filter->ret; - if (line[0] == '-') { - if (!filter->pending_rm_count) - filter->pending_rm_pre_begin = filter->lno_in_preimage; - filter->lno_in_preimage++; - strbuf_add(&filter->pending_rm, line, len); - filter->pending_rm_count++; - return filter->ret; - } - if (line[0] == '\\') { - if (filter->pending_rm_count) - strbuf_add(&filter->pending_rm, line, len); - else if (filter->hunk.active) + if (filter->hunk.active) strbuf_add(&filter->hunk.lines, line, len); - /* otherwise outside tracked range; drop silently */ return filter->ret; } - if (line[0] != '+' && line[0] != ' ') + if (line[0] != '+' && line[0] != ' ' && line[0] != '-') BUG("unexpected diff line type '%c'", line[0]); + /* + * idx_in_postimage is this line's 0-based post-image index (see the model on + * struct line_range_filter). The cursors are advanced only after + * the line is classified, so a '-' is tested at the same idx_in_postimage as + * the '+'/' ' that follows it. + */ idx_in_postimage = filter->lno_in_postimage - 1; - cur_pre = filter->lno_in_preimage; /* save before advancing for context lines */ - filter->lno_in_postimage++; - if (line[0] == ' ') - filter->lno_in_preimage++; - /* Advance past ranges we've passed */ + /* Retire ranges we have passed, flushing the one we leave. */ while (filter->cur_range < filter->ranges->nr && idx_in_postimage >= filter->ranges->ranges[filter->cur_range].end) { if (filter->hunk.active) flush_range_hunk(filter); - discard_pending_rm(filter); filter->cur_range++; } - /* Past all ranges */ - if (filter->cur_range >= filter->ranges->nr) { - discard_pending_rm(filter); - return filter->ret; - } + in_range = filter->cur_range < filter->ranges->nr && + idx_in_postimage >= filter->ranges->ranges[filter->cur_range].start && + idx_in_postimage < filter->ranges->ranges[filter->cur_range].end; - cur = &filter->ranges->ranges[filter->cur_range]; + if (in_range) { + if (!filter->hunk.active) + begin_range_hunk(filter); - /* Before current range */ - if (idx_in_postimage < cur->start) { - discard_pending_rm(filter); - return filter->ret; + strbuf_add(&filter->hunk.lines, line, len); } - /* In range so start a new range hunk if needed */ - if (!filter->hunk.active) { - filter->hunk.active = 1; - filter->hunk.has_changes = 0; - filter->hunk.new_begin = idx_in_postimage + 1; - filter->hunk.old_begin = filter->pending_rm_count - ? filter->pending_rm_pre_begin : cur_pre; - filter->hunk.old_count = 0; - filter->hunk.new_count = 0; - strbuf_reset(&filter->hunk.lines); - } - - /* Flush pending removals into range hunk */ - if (filter->pending_rm_count) { - strbuf_addbuf(&filter->hunk.lines, &filter->pending_rm); - filter->hunk.old_count += filter->pending_rm_count; - filter->hunk.has_changes = 1; - discard_pending_rm(filter); - } - - strbuf_add(&filter->hunk.lines, line, len); - filter->hunk.new_count++; - if (line[0] == '+') - filter->hunk.has_changes = 1; - else - filter->hunk.old_count++; + /* + * Advance each image's cursor: a line present in that image (see + * the model) consumes one of its line numbers. + */ + if (line[0] != '-') + filter->lno_in_postimage++; + if (line[0] != '+') + filter->lno_in_preimage++; return filter->ret; } @@ -4097,7 +4120,6 @@ static void builtin_diff(const char *name_a, lr_state.orig_cb_data = &ecbdata; lr_state.ranges = line_ranges; strbuf_init(&lr_state.hunk.lines, 0); - strbuf_init(&lr_state.pending_rm, 0); /* * Inflate ctxlen so that all changes within @@ -4132,7 +4154,6 @@ static void builtin_diff(const char *name_a, die("unable to generate diff for %s", one->path); strbuf_release(&lr_state.hunk.lines); - strbuf_release(&lr_state.pending_rm); } else if (xdi_diff_outf(&mf1, &mf2, NULL, fn_out_consume, &ecbdata, &xpp, &xecfg)) die("unable to generate diff for %s", one->path); diff --git a/t/t4211-line-log.sh b/t/t4211-line-log.sh index ca4eb7bbc713ef..e9691066deea74 100755 --- a/t/t4211-line-log.sh +++ b/t/t4211-line-log.sh @@ -738,6 +738,131 @@ test_expect_success '-L with -G filters to diff-text matches' ' grep "F2 + 2" actual ' +test_expect_success 'setup for trailing deletion test' ' + git checkout --orphan trailing-del && + git reset --hard && + cat >file.c <<-\EOF && + void tracked() + { + return 1; + } + // trailing comment + EOF + git add file.c && + test_tick && + git commit -m "add file with trailing comment" && + # Modify tracked() AND delete the trailing comment in + # one commit, so the commit touches the tracked range + # and is not filtered out by the revision walker. + cat >file.c <<-\EOF && + void tracked() + { + return 2; + } + EOF + git commit -a -m "modify tracked and delete trailing comment" +' + +test_expect_success '-L does not include deletions past end of tracked range' ' + git log -L:tracked:file.c --format= -1 -p >actual && + # The trailing comment deletion is outside the tracked + # range and should not appear in the patch output. + test_grep "return 2" actual && + test_grep ! "trailing comment" actual +' + +test_expect_success '-L includes leading deletions resolved by in-range line' ' + git checkout --orphan leading-del && + git reset --hard && + cat >file.c <<-\EOF && + // leading comment + void tracked() + { + return 1; + } + EOF + git add file.c && + test_tick && + git commit -m "add file with leading comment" && + cat >file.c <<-\EOF && + void tracked() + { + return 2; + } + EOF + git commit -a -m "modify tracked and delete leading comment" && + git log -L:tracked:file.c --format= -1 -p >actual && + # The leading comment deletion is resolved by the next + # non-removal line (void tracked), which is in range: a + # removal is classified by the position of the following + # line, so it joins the range that line falls in. + test_grep "return 2" actual && + test_grep "leading comment" actual +' + +test_expect_success 'setup for line-range filter edge cases' ' + git checkout --orphan filter-edge && + git reset --hard && + cat >file.c <<-\EOF && + void before() + { + return 0; + } + + void tracked() + { + int a = 1; + int b = 2; + int c = 3; + return a + b + c; + } + + void after() + { + return 9; + } + EOF + git add file.c && + test_tick && + git commit -m "initial" +' + +test_expect_success '-L change at exact first line of range' ' + git checkout filter-edge && + # Change the function signature (first line of range) + sed "s/void tracked/int tracked/" file.c >tmp && + mv tmp file.c && + git commit -a -m "change first line" && + git log -L:tracked:file.c -p --format=%s -1 >actual && + test_grep "change first line" actual && + test_grep "+int tracked" actual && + test_grep "\\-void tracked" actual +' + +test_expect_success '-L change at exact last line of range' ' + git checkout filter-edge && + git reset --hard HEAD~1 && + # Change the closing brace line (last line of range) + sed "s/^}$/} \/\/ end tracked/" file.c >tmp && + mv tmp file.c && + git commit -a -m "change last line" && + git log -L:tracked:file.c -p --format=%s -1 >actual && + test_grep "change last line" actual && + test_grep "end tracked" actual +' + +test_expect_success '-L pure deletion in range (no additions)' ' + git checkout filter-edge && + git reset --hard HEAD~1 && + # Delete a line inside tracked() without adding anything + sed "/int c/d" file.c >tmp && + mv tmp file.c && + git commit -a -m "pure deletion" && + git log -L:tracked:file.c -p --format=%s -1 >actual && + test_grep "pure deletion" actual && + test_grep "\\-.*int c" actual +' + test_expect_success '-L with --diff-filter=M excludes root commit' ' git checkout parent-oids && git log -L:func2:file.c --diff-filter=M --format=%s --no-patch >actual && From 7555510bf32835fae50363b86a8a667c1b681803 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:28:57 +0000 Subject: [PATCH 008/259] diff: emit -L hunk headers via xdiff's formatter The line-range filter builds its own "@@ - + @@" header for each range hunk. For a side with no lines (count 0, such as the old side of a pure insertion), the begin should be the number of the line before the change, per the convention git diff and xdl_emit_hunk_hdr() follow. The hand-rolled code's begin was one too high; in t4211 this produced @@ -25,0 +18,9 @@ an old begin of 25 in a 24-line file, where git diff would give 24. Stop hand-rolling the header. flush_range_hunk() now formats it through xdiff's own emitter: a new xdiff_emit_hunk_header() helper wraps xdl_emit_hunk_hdr(), the function that produces every other diff's hunk headers. The count-0 begin is then correct by construction, and as a side effect -L headers match git diff exactly, including its omission of a count of 1 ("@@ -22 +22 @@" rather than "@@ -22,1 +22,1 @@"). xdiff's hunk callback already hands line_range_hunk_fn() a count-0 begin decremented, so undo that when seeding the cursors and let the formatter re-apply the convention once, at emit time. The off-by-one predates this series, and the two regenerated fixtures reach it from different origins: no-assertion-error has carried it since its test was added in ab60c693a2 (line-log: fix assertion error, 2025-08-18), while vanishes-early acquired it when 86e986f166 (line-log: route -L output through the standard diff pipeline) reshaped its tracked line into a pure insertion. vanishes-early also drops its count-1 counts. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- diff.c | 27 +++++++++++------------- t/t4211/sha1/expect.no-assertion-error | 2 +- t/t4211/sha1/expect.vanishes-early | 6 +++--- t/t4211/sha256/expect.no-assertion-error | 2 +- t/t4211/sha256/expect.vanishes-early | 6 +++--- xdiff-interface.c | 19 +++++++++++++++++ xdiff-interface.h | 15 +++++++++++++ 7 files changed, 54 insertions(+), 23 deletions(-) diff --git a/diff.c b/diff.c index ee765d7ac2183d..9751bb679874b9 100644 --- a/diff.c +++ b/diff.c @@ -2636,14 +2636,9 @@ static void flush_range_hunk(struct line_range_filter *filter) return; } - strbuf_addf(&hdr, "@@ -%ld,%ld +%ld,%ld @@", - filter->hunk.old_begin, old_count, - filter->hunk.new_begin, new_count); - if (filter->funclen > 0) { - strbuf_addch(&hdr, ' '); - strbuf_add(&hdr, filter->func, filter->funclen); - } - strbuf_addch(&hdr, '\n'); + xdiff_emit_hunk_header(&hdr, filter->hunk.old_begin, old_count, + filter->hunk.new_begin, new_count, + filter->func, filter->funclen); filter->ret = filter->orig_line_fn(filter->orig_cb_data, hdr.buf, hdr.len); strbuf_release(&hdr); @@ -2668,19 +2663,21 @@ static void flush_range_hunk(struct line_range_filter *filter) } static void line_range_hunk_fn(void *data, - long old_begin, long old_nr UNUSED, - long new_begin, long new_nr UNUSED, + long old_begin, long old_nr, + long new_begin, long new_nr, const char *func, long funclen) { struct line_range_filter *filter = data; /* - * When count > 0, begin is 1-based. When count == 0, begin is - * adjusted down by 1 by xdl_emit_hunk_hdr(), but no lines of - * that type will arrive, so the value is unused. + * Seed the per-image line cursors from the hunk header's begins. For + * a side with no lines (count 0), xdiff's callback has already moved + * its begin to the line before the change, so add one back to recover + * the true 1-based start. xdiff_emit_hunk_header() reapplies that -1 + * when the clipped hunk is emitted. */ - filter->lno_in_postimage = new_begin; - filter->lno_in_preimage = old_begin; + filter->lno_in_postimage = new_nr ? new_begin : new_begin + 1; + filter->lno_in_preimage = old_nr ? old_begin : old_begin + 1; if (funclen > 0) { if (funclen > (long)sizeof(filter->func)) diff --git a/t/t4211/sha1/expect.no-assertion-error b/t/t4211/sha1/expect.no-assertion-error index 54c568f273a6d2..95faf51a7b46d9 100644 --- a/t/t4211/sha1/expect.no-assertion-error +++ b/t/t4211/sha1/expect.no-assertion-error @@ -8,7 +8,7 @@ diff --git a/b.c b/b.c index bf79c2f..27c829c 100644 --- a/b.c +++ b/b.c -@@ -25,0 +18,9 @@ +@@ -24,0 +18,9 @@ +long f(long x) +{ + int s = 0; diff --git a/t/t4211/sha1/expect.vanishes-early b/t/t4211/sha1/expect.vanishes-early index a413ad36598ddf..e4b1a201d5d254 100644 --- a/t/t4211/sha1/expect.vanishes-early +++ b/t/t4211/sha1/expect.vanishes-early @@ -8,7 +8,7 @@ diff --git a/a.c b/a.c index 0b9cae5..5de3ea4 100644 --- a/a.c +++ b/a.c -@@ -23,0 +24,1 @@ int main () +@@ -22,0 +24 @@ int main () +/* incomplete lines are bad! */ commit 100b61a6f2f720f812620a9d10afb3a960ccb73c @@ -21,7 +21,7 @@ diff --git a/a.c b/a.c index 5e709a1..0b9cae5 100644 --- a/a.c +++ b/a.c -@@ -22,1 +22,1 @@ int main () +@@ -22 +22 @@ int main () -} +} \ No newline at end of file @@ -37,5 +37,5 @@ new file mode 100644 index 0000000..444e415 --- /dev/null +++ b/a.c -@@ -0,0 +20,1 @@ +@@ -0,0 +20 @@ +} diff --git a/t/t4211/sha256/expect.no-assertion-error b/t/t4211/sha256/expect.no-assertion-error index c25f2ce19c05d9..815d27f7f17b7e 100644 --- a/t/t4211/sha256/expect.no-assertion-error +++ b/t/t4211/sha256/expect.no-assertion-error @@ -8,7 +8,7 @@ diff --git a/b.c b/b.c index 69cb69c..a0d566e 100644 --- a/b.c +++ b/b.c -@@ -25,0 +18,9 @@ +@@ -24,0 +18,9 @@ +long f(long x) +{ + int s = 0; diff --git a/t/t4211/sha256/expect.vanishes-early b/t/t4211/sha256/expect.vanishes-early index bc33b963dc8570..263fc9eaace442 100644 --- a/t/t4211/sha256/expect.vanishes-early +++ b/t/t4211/sha256/expect.vanishes-early @@ -8,7 +8,7 @@ diff --git a/a.c b/a.c index e4fa1d8..62c1fc2 100644 --- a/a.c +++ b/a.c -@@ -23,0 +24,1 @@ int main () +@@ -22,0 +24 @@ int main () +/* incomplete lines are bad! */ commit 29f32ac3141c48b22803e5c4127b719917b67d0f8ca8c5248bebfa2a19f7da10 @@ -21,7 +21,7 @@ diff --git a/a.c b/a.c index d325124..e4fa1d8 100644 --- a/a.c +++ b/a.c -@@ -22,1 +22,1 @@ int main () +@@ -22 +22 @@ int main () -} +} \ No newline at end of file @@ -37,5 +37,5 @@ new file mode 100644 index 0000000..9f550c3 --- /dev/null +++ b/a.c -@@ -0,0 +20,1 @@ +@@ -0,0 +20 @@ +} diff --git a/xdiff-interface.c b/xdiff-interface.c index 5ee2b96d0a756f..32e04630ee2ee8 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -91,6 +91,25 @@ static int xdiff_outf(void *priv_, mmbuffer_t *mb, int nbuf) return 0; } +static int strbuf_out_line(void *priv, mmbuffer_t *mb, int nbuf) +{ + struct strbuf *out = priv; + int i; + for (i = 0; i < nbuf; i++) + strbuf_add(out, mb[i].ptr, mb[i].size); + return 0; +} + +void xdiff_emit_hunk_header(struct strbuf *out, + long old_begin, long old_count, + long new_begin, long new_count, + const char *func, long funclen) +{ + xdemitcb_t ecb = { .priv = out, .out_line = strbuf_out_line }; + xdl_emit_hunk_hdr(old_begin, old_count, new_begin, new_count, + func, funclen, &ecb); +} + /* * Trim down common substring at the end of the buffers, * but end on a complete line. diff --git a/xdiff-interface.h b/xdiff-interface.h index ce54e1c0e002f8..51c88296ed5e68 100644 --- a/xdiff-interface.h +++ b/xdiff-interface.h @@ -76,4 +76,19 @@ int xdiff_compare_lines(const char *l1, long s1, */ unsigned long xdiff_hash_string(const char *s, size_t len, long flags); +struct strbuf; + +/* + * Append a unified-diff hunk header to `out`, e.g. + * "@@ - + @@ func\n". The header comes from wrapping xdiff's + * own hunk-header emitter, so it matches what a normal diff would + * produce for these begins and counts. For a side with no lines + * (count 0) the begin is the line before the change, and a count of 1 + * is omitted. + */ +void xdiff_emit_hunk_header(struct strbuf *out, + long old_begin, long old_count, + long new_begin, long new_count, + const char *func, long funclen); + #endif From 56cf30f68c6ab06e34c295c62b081da8904b1c41 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:28:58 +0000 Subject: [PATCH 009/259] diff: extract a line-range diff helper for reuse builtin_diff() open-codes the line-range filter setup and teardown around its xdi_diff_outf() call: zero the struct, point it at the output callback, inflate ctxlen to the largest range span so each range yields a single xdiff hunk, run the diff, flush the trailing range hunk, and release the buffer. The upcoming -L stat and check formats need the same sequence. Extract line_range_filter_init() for the setup and a line_range_filter_diff() helper that prepares the xdiff config the filter needs, runs an initialized filter through xdi_diff_outf(), flushes the final range hunk, and releases it, returning the latched error. The helper inflates ctxlen to the largest range span so each range yields a single xdiff hunk, and clears XDL_EMIT_NO_HUNK_HDR so the hunk headers the filter seeds its position from are always emitted. Folding both into the helper keeps these invariants, which the filter's position tracking relies on, in a single place for every consumer. builtin_diff() now does init + line_range_filter_diff(); the next two patches reuse them in builtin_diffstat() and builtin_checkdiff() instead of repeating the boilerplate. No behavior change: builtin_diff() leaves XDL_EMIT_NO_HUNK_HDR unset, so clearing it is a no-op until the suppressing consumers arrive. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- diff.c | 100 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/diff.c b/diff.c index 9751bb679874b9..6233a96bf02252 100644 --- a/diff.c +++ b/diff.c @@ -2580,6 +2580,18 @@ static int quick_consume(void *priv, char *line UNUSED, unsigned long len UNUSED return 1; } +static void line_range_filter_init(struct line_range_filter *filter, + const struct range_set *ranges, + xdiff_emit_line_fn line_fn, + void *cb_data) +{ + memset(filter, 0, sizeof(*filter)); + filter->orig_line_fn = line_fn; + filter->orig_cb_data = cb_data; + filter->ranges = ranges; + strbuf_init(&filter->hunk.lines, 0); +} + /* * Begin a range hunk at the first in-range line. Its position fixes the * hunk's begins, taken from the two image cursors before they advance: @@ -2744,6 +2756,50 @@ static int line_range_line_fn(void *priv, char *line, unsigned long len) return filter->ret; } +/* + * Run an xdiff pass through an initialized line-range filter, flush the + * final range hunk, and release the filter. Inflates ctxlen to the largest + * range span first, so that every change within a single range lands in one + * xdiff hunk and the inter-change context is emitted; the filter then clips + * back to range boundaries. The optimal ctxlen depends on where changes fall + * within the range, which is only known after xdiff runs, so the max span is + * the upper bound that guarantees correctness in a single pass. Every + * consumer (patch, diffstat, check) relies on one xdiff hunk per range, so + * this lives here rather than at each call site. Also clears + * XDL_EMIT_NO_HUNK_HDR: the filter seeds its per-image position from the hunk + * headers, so a consumer that otherwise suppresses them (diffstat) still gets + * them here. Returns non-zero if xdiff or any forwarded callback failed. + */ +static int line_range_filter_diff(struct line_range_filter *filter, + mmfile_t *mf1, mmfile_t *mf2, + xpparam_t *xpp, xdemitconf_t *xecfg) +{ + const struct range_set *ranges = filter->ranges; + long max_span = 0; + unsigned int i; + int ret; + + for (i = 0; i < ranges->nr; i++) { + long span = ranges->ranges[i].end - ranges->ranges[i].start; + if (span > max_span) + max_span = span; + } + if (max_span > xecfg->ctxlen) + xecfg->ctxlen = max_span; + + /* the filter seeds its per-image position from hunk headers */ + xecfg->flags &= ~XDL_EMIT_NO_HUNK_HDR; + + ret = xdi_diff_outf(mf1, mf2, line_range_hunk_fn, + line_range_line_fn, filter, xpp, xecfg); + if (!ret) { + flush_range_hunk(filter); + ret = filter->ret; + } + strbuf_release(&filter->hunk.lines); + return ret; +} + static void pprint_rename(struct strbuf *name, const char *a, const char *b) { const char *old_name = a; @@ -4108,49 +4164,15 @@ static void builtin_diff(const char *name_a, xdi_diff_outf(&mf1, &mf2, NULL, quick_consume, &ecbdata, &xpp, &xecfg); } else if (line_ranges) { - struct line_range_filter lr_state; - unsigned int i; - long max_span = 0; + struct line_range_filter lr_filter; - memset(&lr_state, 0, sizeof(lr_state)); - lr_state.orig_line_fn = fn_out_consume; - lr_state.orig_cb_data = &ecbdata; - lr_state.ranges = line_ranges; - strbuf_init(&lr_state.hunk.lines, 0); - - /* - * Inflate ctxlen so that all changes within - * any single range are merged into one xdiff - * hunk and the inter-change context is emitted. - * The callback clips back to range boundaries. - * - * The optimal ctxlen depends on where changes - * fall within the range, which is only known - * after xdiff runs; the max range span is the - * upper bound that guarantees correctness in a - * single pass. - */ - for (i = 0; i < line_ranges->nr; i++) { - long span = line_ranges->ranges[i].end - - line_ranges->ranges[i].start; - if (span > max_span) - max_span = span; - } - if (max_span > xecfg.ctxlen) - xecfg.ctxlen = max_span; - - if (xdi_diff_outf(&mf1, &mf2, - line_range_hunk_fn, - line_range_line_fn, - &lr_state, &xpp, &xecfg)) - die("unable to generate diff for %s", - one->path); + line_range_filter_init(&lr_filter, line_ranges, + fn_out_consume, &ecbdata); - flush_range_hunk(&lr_state); - if (lr_state.ret) + if (line_range_filter_diff(&lr_filter, &mf1, &mf2, + &xpp, &xecfg)) die("unable to generate diff for %s", one->path); - strbuf_release(&lr_state.hunk.lines); } else if (xdi_diff_outf(&mf1, &mf2, NULL, fn_out_consume, &ecbdata, &xpp, &xecfg)) die("unable to generate diff for %s", one->path); From 660aae7323bca80e4441479ccd3b83e60dc16477 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:28:59 +0000 Subject: [PATCH 010/259] line-log: support diff stat formats with -L Reuse the line_range_filter in builtin_diffstat() so the stat formats count only the lines within the tracked range. When a filepair carries line_ranges, the filter wraps diffstat_consume() as its output callback, forwarding only the lines inside the range for counting. flush_range_hunk() replays buffered content through diffstat_consume(), which ignores synthetic @@ headers since it only counts '+' and '-' lines. Expand the output format allowlist in setup_revisions() to accept --stat, --numstat, and --shortstat with -L. Leave --dirstat out of the allowlist so it is rejected like any other unsupported format. Its default mode counts each file's whole-file byte damage via diffcore_count_changes(), outside the line-based pipeline that the -L filter scopes, so bare --dirstat cannot honor the tracked range. The --dirstat=lines mode could: it aggregates the same per-file line counts as --numstat, which -L already scopes. But accepting only that sub-mode while bare --dirstat keeps erroring is a confusing split, so the whole format is deferred to a follow-up; --numstat already reports the exact per-file counts within the tracked range. Also drop "yet" from the generic -L rejection message ("does not yet support the requested diff format"). Some rejected formats do not fit a line range at all, so "yet" wrongly implied they are all just awaiting support. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- Documentation/line-range-options.adoc | 12 ++- diff.c | 13 ++- revision.c | 6 +- t/t4211-line-log.sh | 150 ++++++++++++++++++++++---- 4 files changed, 155 insertions(+), 26 deletions(-) diff --git a/Documentation/line-range-options.adoc b/Documentation/line-range-options.adoc index 72f639b5e79ea4..a111a492b49881 100644 --- a/Documentation/line-range-options.adoc +++ b/Documentation/line-range-options.adoc @@ -9,10 +9,14 @@ __ and __ (or __) must exist in the starting revision. You can specify this option more than once. Implies `--patch`. Patch output can be suppressed using `--no-patch`. - Non-patch diff formats `--raw`, `--name-only`, `--name-status`, - and `--summary` are supported. Diff stat formats - (`--stat`, `--numstat`, `--shortstat`, `--dirstat`) are not - currently implemented. + The following non-patch diff formats are supported: `--raw`, + `--name-only`, `--name-status`, `--summary`, + `--stat`, `--numstat`, and `--shortstat`. + The stat formats count only lines within the tracked range. + `--dirstat` is not supported + with `-L`: it summarizes change as each directory's share of + the total churn, not as counts for the tracked lines. Use + `--numstat` for exact per-file counts within the range. + Patch formatting options such as `--word-diff`, `--color-moved`, `--no-prefix`, and whitespace options (`-w`, `-b`) are supported, diff --git a/diff.c b/diff.c index 6233a96bf02252..026fafeb90b6fc 100644 --- a/diff.c +++ b/diff.c @@ -4289,7 +4289,18 @@ static void builtin_diffstat(const char *name_a, const char *name_b, xecfg.ctxlen = o->context; xecfg.interhunkctxlen = o->interhunkcontext; xecfg.flags = XDL_EMIT_NO_HUNK_HDR; - if (xdi_diff_outf(&mf1, &mf2, NULL, + + if (p->line_ranges) { + struct line_range_filter lr_filter; + + line_range_filter_init(&lr_filter, p->line_ranges, + diffstat_consume, diffstat); + + if (line_range_filter_diff(&lr_filter, &mf1, &mf2, + &xpp, &xecfg)) + die("unable to generate diffstat for %s", + one->path); + } else if (xdi_diff_outf(&mf1, &mf2, NULL, diffstat_consume, diffstat, &xpp, &xecfg)) die("unable to generate diffstat for %s", one->path); diff --git a/revision.c b/revision.c index 6a8101e8b7ef5f..2c76e15778de32 100644 --- a/revision.c +++ b/revision.c @@ -3193,8 +3193,10 @@ int setup_revisions(int argc, const char **argv, struct rev_info *revs, struct s (revs->diffopt.output_format & ~(DIFF_FORMAT_PATCH | DIFF_FORMAT_NO_OUTPUT | DIFF_FORMAT_RAW | DIFF_FORMAT_NAME | - DIFF_FORMAT_NAME_STATUS | DIFF_FORMAT_SUMMARY)))) - die(_("-L does not yet support the requested diff format")); + DIFF_FORMAT_NAME_STATUS | DIFF_FORMAT_SUMMARY | + DIFF_FORMAT_NUMSTAT | DIFF_FORMAT_DIFFSTAT | + DIFF_FORMAT_SHORTSTAT)))) + die(_("-L does not support the requested diff format")); if (revs->expand_tabs_in_log < 0) revs->expand_tabs_in_log = revs->expand_tabs_in_log_default; diff --git a/t/t4211-line-log.sh b/t/t4211-line-log.sh index e9691066deea74..b9ca336dbc4f99 100755 --- a/t/t4211-line-log.sh +++ b/t/t4211-line-log.sh @@ -176,24 +176,15 @@ test_expect_success '--name-status shows status and path' ' test_grep ! "^@@" actual ' -test_expect_success '--stat is not yet supported with -L' ' - test_must_fail git log -L1,24:b.c --stat 2>err && - test_grep "does not yet support" err -' - -test_expect_success '--numstat is not yet supported with -L' ' - test_must_fail git log -L1,24:b.c --numstat 2>err && - test_grep "does not yet support" err -' - -test_expect_success '--shortstat is not yet supported with -L' ' - test_must_fail git log -L1,24:b.c --shortstat 2>err && - test_grep "does not yet support" err -' - -test_expect_success '--dirstat is not yet supported with -L' ' +test_expect_success '--dirstat is not supported with -L' ' + # --dirstat is not supported with -L: its default mode measures + # whole-file change, not the tracked lines, and the + # --dirstat=lines variant is deferred too, so both forms are + # rejected like any other unsupported format. test_must_fail git log -L1,24:b.c --dirstat 2>err && - test_grep "does not yet support" err + test_grep "does not support" err && + test_must_fail git log -L1,24:b.c --dirstat=lines 2>err && + test_grep "does not support" err ' test_expect_success 'setup for checking fancy rename following' ' @@ -887,9 +878,9 @@ test_expect_success '-L with -S suppresses non-matching commits' ' test_cmp expect actual ' -test_expect_success '--full-diff is not yet supported with -L' ' +test_expect_success '--full-diff is not supported with -L' ' test_must_fail git log -L1,24:b.c --full-diff 2>err && - test_grep "does not yet support" err + test_grep "does not support" err ' test_expect_success '-L --oneline has no extra blank line before diff' ' @@ -900,6 +891,127 @@ test_expect_success '-L --oneline has no extra blank line before diff' ' test_grep "^diff --git" line2 ' +test_expect_success 'setup for stat range-scoping tests' ' + git checkout --orphan stat-scoping && + git reset --hard && + cat >file.c <<-\EOF && + int func1() + { + return F1; + } + + int func2() + { + return F2; + } + EOF + git add file.c && + test_tick && + git commit -m "Add func1() and func2()" && + + # Modify both functions in a single commit so that + # whole-file stats differ from the counts for the tracked range. + sed -e "s/F1/F1 + 1/" -e "s/F2/F2 + 2/" file.c >tmp && + mv tmp file.c && + git commit -a -m "Modify both functions" +' + +test_expect_success '--numstat counts only lines in tracked range' ' + # "Modify both functions" changes one line in func1 and one in + # func2. Whole-file numstat would show 2 added, 2 deleted. + # numstat for func2 within the tracked range should show only 1 and 1. + git log -L:func2:file.c --numstat --format=%s -1 >actual && + test_grep "Modify both functions" actual && + test_grep "^1 1 file.c$" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--numstat counts only additions for root commit' ' + # Root commit creates both func1 (4 lines) and func2 (4 lines). + # Whole-file numstat would show 9 lines added. numstat for func2 + # within the tracked range should show only 4. + git log -L:func2:file.c --numstat --format=%s >actual && + test_grep "Add func1() and func2()" actual && + test_grep "^4 0 file.c$" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--stat counts only lines in tracked range' ' + git log -L:func2:file.c --stat --format=%s -1 >actual && + test_grep "Modify both functions" actual && + test_grep "file.c |" actual && + test_grep "1 insertion" actual && + test_grep "1 deletion" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--shortstat counts only lines in tracked range' ' + # --shortstat prints only the summary line: no per-file "file.c |" + # line. Counts cover only the tracked range, as for --numstat above. + git log -L:func2:file.c --shortstat --format=%s -1 >actual && + test_grep "Modify both functions" actual && + test_grep "1 insertion" actual && + test_grep "1 deletion" actual && + test_grep ! "file.c |" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--numstat across renames and multiple commits' ' + # parallel-change carries the tracked function f across an a.c -> b.c + # rename and a merge of two parallel histories. With -M, --numstat + # follows the rename and reports added/removed counts for f within + # the tracked range (not whole-file) per commit; the file column flips from + # b.c to a.c at the rename as the walk goes back in time. Commits + # that do not change the range of f emit no row (the merge and the + # pure file-move produce nothing), so there are fewer rows than + # commits. + git checkout parallel-change && + git log -M -L ":f:b.c" --format= --numstat >actual && + cat >expect <<-\EOF && + 1 1 b.c + 1 1 a.c + 1 1 a.c + 1 1 a.c + 1 0 a.c + 13 0 a.c + EOF + test_cmp expect actual +' + +test_expect_success '-L multiple ranges with --numstat excludes untracked change' ' + git checkout --orphan multi-range && + git reset --hard && + cat >m.c <<-\EOF && + int func1() + { + return F1; + } + + int func2() + { + return F2; + } + + int func3() + { + return F3; + } + EOF + git add m.c && + test_tick && + git commit -m "add m.c" && + # Change all three functions but track only func1 and func2. + # Whole-file numstat would be 3 3; a 2 2 result proves the + # untracked func3 change is excluded and the two ranges just sum. + sed -e "s/F1/F1 + 1/" -e "s/F2/F2 + 2/" -e "s/F3/F3 + 3/" m.c >tmp && + mv tmp m.c && + git commit -a -m "Modify all three functions" && + git log -L:func1:m.c -L:func2:m.c --numstat --format=%s -1 >actual && + test_grep "Modify all three functions" actual && + test_grep "^2 2 m.c$" actual && + test_grep ! "^3 3 m.c$" actual +' + test_expect_success '--summary shows new file on root commit' ' git checkout parent-oids && git log -L:func2:file.c --summary --format= >actual && From 54438a56a4edb4c70459180f37af3e43e5d7e6a3 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:29:00 +0000 Subject: [PATCH 011/259] diff: support --check with -L line ranges builtin_checkdiff() runs its own xdiff pass to detect whitespace errors in newly added lines. When -L is active, the check should be scoped to the tracked line ranges rather than the whole file. Reuse the line_range_filter to wrap checkdiff_consume(), the same pattern already used for patch output and diffstat. The filter forwards only in-range lines for whitespace checking. checkdiff reports the file line number of each error, which it normally learns from the hunk header via checkdiff_consume_hunk(). The filter synthesizes its own hunk headers, so give it an optional hunk callback and route checkdiff_consume_hunk() through it; this sets the post-image position before the in-range lines are replayed. Without it the reported line numbers would count from the start of the range hunk rather than the start of the file. The trailing blank-at-eof check is a second pass that scans the whole file via check_blank_at_eof(), so gate its report on the tracked ranges as well; otherwise a blank line added at end of file is reported even when it lies outside the range. Add DIFF_FORMAT_CHECKDIFF to the -L output format allowlist in setup_revisions() so that -L --check is accepted, and list --check among the supported formats in the documentation. Add tests covering that whitespace errors are reported, scoped to the tracked range, and labeled with the correct file line number, including when two errors in one range are separated by a gap that would otherwise split into multiple xdiff hunks. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- Documentation/line-range-options.adoc | 2 +- diff.c | 65 ++++++++++++++++++- revision.c | 2 +- t/t4211-line-log.sh | 92 +++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 5 deletions(-) diff --git a/Documentation/line-range-options.adoc b/Documentation/line-range-options.adoc index a111a492b49881..33b4e948815aac 100644 --- a/Documentation/line-range-options.adoc +++ b/Documentation/line-range-options.adoc @@ -10,7 +10,7 @@ You can specify this option more than once. Implies `--patch`. Patch output can be suppressed using `--no-patch`. The following non-patch diff formats are supported: `--raw`, - `--name-only`, `--name-status`, `--summary`, + `--name-only`, `--name-status`, `--summary`, `--check`, `--stat`, `--numstat`, and `--shortstat`. The stat formats count only lines within the tracked range. `--dirstat` is not supported diff --git a/diff.c b/diff.c index 026fafeb90b6fc..519c5133566032 100644 --- a/diff.c +++ b/diff.c @@ -665,6 +665,12 @@ struct emit_callback { */ struct line_range_filter { xdiff_emit_line_fn orig_line_fn; + /* + * Optional; consumers that report file line numbers (e.g. + * checkdiff) need the synthetic hunk header to set their + * post-image position before in-range lines are replayed. + */ + xdiff_emit_hunk_fn orig_hunk_fn; void *orig_cb_data; const struct range_set *ranges; /* 0-based [start, end) */ unsigned int cur_range; /* index into the range_set */ @@ -2652,6 +2658,17 @@ static void flush_range_hunk(struct line_range_filter *filter) filter->hunk.new_begin, new_count, filter->func, filter->funclen); + /* + * Inform a line-numbering consumer of the post-image position + * before replaying lines, mirroring the hunk callback xdiff + * would have issued for a non-scoped diff. + */ + if (filter->orig_hunk_fn) + filter->orig_hunk_fn(filter->orig_cb_data, + filter->hunk.old_begin, old_count, + filter->hunk.new_begin, new_count, + filter->func, filter->funclen); + filter->ret = filter->orig_line_fn(filter->orig_cb_data, hdr.buf, hdr.len); strbuf_release(&hdr); @@ -4330,11 +4347,29 @@ static void builtin_diffstat(const char *name_a, const char *name_b, diff_free_filespec_data(two); } +/* + * Is the 0-based line index within any of the tracked ranges? + * (range_set ranges are 0-based, half-open [start, end).) This is a + * one-shot query for a single line and scans; the streaming filter + * (line_range_line_fn) uses a forward cursor instead. + */ +static int idx_in_ranges(const struct range_set *ranges, long idx) +{ + unsigned int i; + + for (i = 0; i < ranges->nr; i++) + if (idx >= ranges->ranges[i].start && + idx < ranges->ranges[i].end) + return 1; + return 0; +} + static void builtin_checkdiff(const char *name_a, const char *name_b, const char *attr_path, struct diff_filespec *one, struct diff_filespec *two, - struct diff_options *o) + struct diff_options *o, + const struct range_set *line_ranges) { mmfile_t mf1, mf2; struct checkdiff_t data; @@ -4374,7 +4409,19 @@ static void builtin_checkdiff(const char *name_a, const char *name_b, memset(&xecfg, 0, sizeof(xecfg)); xecfg.ctxlen = 1; /* at least one context line */ xpp.flags = 0; - if (xdi_diff_outf(&mf1, &mf2, checkdiff_consume_hunk, + + if (line_ranges) { + struct line_range_filter lr_filter; + + line_range_filter_init(&lr_filter, line_ranges, + checkdiff_consume, &data); + lr_filter.orig_hunk_fn = checkdiff_consume_hunk; + + if (line_range_filter_diff(&lr_filter, &mf1, &mf2, + &xpp, &xecfg)) + die("unable to generate checkdiff for %s", + one->path); + } else if (xdi_diff_outf(&mf1, &mf2, checkdiff_consume_hunk, checkdiff_consume, &data, &xpp, &xecfg)) die("unable to generate checkdiff for %s", one->path); @@ -4387,6 +4434,17 @@ static void builtin_checkdiff(const char *name_a, const char *name_b, check_blank_at_eof(&mf1, &mf2, &ecbdata); blank_at_eof = ecbdata.blank_at_eof_in_postimage; + /* + * check_blank_at_eof() scans the whole file; with -L, + * keep the report only when its line is in a tracked + * range. The error's location is the first trailing + * blank line (blank_at_eof, 1-based; ranges 0-based), so + * we scope by that line. + */ + if (blank_at_eof && line_ranges && + !idx_in_ranges(line_ranges, blank_at_eof - 1)) + blank_at_eof = 0; + if (blank_at_eof) { static char *err; if (!err) @@ -5179,7 +5237,8 @@ static void run_checkdiff(struct diff_filepair *p, struct diff_options *o) diff_fill_oid_info(p->one, o->repo->index); diff_fill_oid_info(p->two, o->repo->index); - builtin_checkdiff(name, other, attr_path, p->one, p->two, o); + builtin_checkdiff(name, other, attr_path, p->one, p->two, o, + p->line_ranges); } void repo_diff_setup(struct repository *r, struct diff_options *options) diff --git a/revision.c b/revision.c index 2c76e15778de32..7abb287451ba14 100644 --- a/revision.c +++ b/revision.c @@ -3195,7 +3195,7 @@ int setup_revisions(int argc, const char **argv, struct rev_info *revs, struct s DIFF_FORMAT_RAW | DIFF_FORMAT_NAME | DIFF_FORMAT_NAME_STATUS | DIFF_FORMAT_SUMMARY | DIFF_FORMAT_NUMSTAT | DIFF_FORMAT_DIFFSTAT | - DIFF_FORMAT_SHORTSTAT)))) + DIFF_FORMAT_SHORTSTAT | DIFF_FORMAT_CHECKDIFF)))) die(_("-L does not support the requested diff format")); if (revs->expand_tabs_in_log < 0) diff --git a/t/t4211-line-log.sh b/t/t4211-line-log.sh index b9ca336dbc4f99..68576418f41895 100755 --- a/t/t4211-line-log.sh +++ b/t/t4211-line-log.sh @@ -1018,4 +1018,96 @@ test_expect_success '--summary shows new file on root commit' ' test_grep "create mode 100644 file.c" actual ' +test_expect_success 'setup for --check test' ' + git checkout --orphan check-test && + git reset --hard && + cat >check.c <<-\EOF && + void tracked() + { + return; + } + + void other() + { + return; + } + EOF + git add check.c && + test_tick && + git commit -m "add check.c" && + # Introduce trailing whitespace errors in both functions + sed "s/return;/return; /" check.c >check.c.tmp && + mv check.c.tmp check.c && + git commit -a -m "introduce trailing whitespace" +' + +test_expect_success '--check scoped to tracked range with correct file line' ' + # tracked() trailing whitespace is at check.c:3; report it with the + # real file line number, not a count from the start of the range + # hunk. other() at check.c:8 is outside the range and is excluded. + test_must_fail git log -L:tracked:check.c --check --format= >actual && + test_grep "check.c:3: trailing whitespace" actual && + test_grep ! "check.c:8:" actual +' + +test_expect_success '--check reports each of several tracked ranges' ' + # Track both functions as separate ranges. Each range is flushed + # as its own hunk, so the second error must report its real file + # line (check.c:8), not continue the numbering from the first + # range (check.c:3). + test_must_fail git log -L:tracked:check.c -L:other:check.c \ + --check --format= >actual && + test_grep "check.c:3: trailing whitespace" actual && + test_grep "check.c:8: trailing whitespace" actual +' + +test_expect_success '--check line numbers stay correct across a gap in one range' ' + git checkout --orphan check-gap && + git reset --hard && + cat >gap.c <<-\EOF && + void tracked() + { + int a = 1; + int b = 2; + int c = 3; + int d = 4; + int e = 5; + int g = 7; + return; + } + EOF + git add gap.c && + test_tick && + git commit -m "add gap.c" && + # Two trailing-whitespace errors within one tracked range, + # separated by clean lines. ctxlen is inflated to the range span, + # so they land in a single xdiff hunk with the gap as context; + # both must report their real file line number, with the context + # lines between them counted. + sed -e "s/int a = 1;/int a = 1; /" -e "s/int g = 7;/int g = 7; /" gap.c >tmp && + mv tmp gap.c && + git commit -a -m "ws errors with a gap" && + test_must_fail git log -L:tracked:gap.c --check --format= >actual && + test_grep "gap.c:3: trailing whitespace" actual && + test_grep "gap.c:8: trailing whitespace" actual +' + +test_expect_success '--check does not report blank-at-eof outside the range' ' + git checkout --orphan check-eof && + git reset --hard && + printf "void tracked()\n{\n return;\n}\n\nint tail = 1;\n" >eof.c && + git add eof.c && + test_tick && + git commit -m "add eof.c" && + # One commit introduces a trailing-whitespace error inside tracked() + # (line 3) and a blank line at end of file (line 7, outside the + # range). The blank-at-eof check scans the whole file, so it must be + # scoped: report the in-range error, not the out-of-range EOF blank. + printf "void tracked()\n{\n return; \n}\n\nint tail = 1;\n\n" >eof.c && + git commit -a -m "ws in range, blank at eof out of range" && + test_must_fail git log -L:tracked:eof.c --check --format= >actual && + test_grep "eof.c:3: trailing whitespace" actual && + test_grep ! "blank line at EOF" actual +' + test_done From f67c51df064d2b64b257bd8c17d757cc0ce1b7fc Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 27 Jun 2026 17:29:01 +0000 Subject: [PATCH 012/259] diffcore-pickaxe: scope -G to the -L tracked range git log -L scopes its diff output to the tracked range, but pickaxe (-S, -G) still runs in diffcore over the whole-file change, so -L -G selects a commit whenever the pattern appears in any added or removed line of the file, even outside the tracked range. Teach -G to honor the range. diff_grep() already runs an xdiff pass and greps the +/- lines; route that pass through the line-range filter so only the tracked range's lines are grepped. Expose the filter as diff_emit_line_ranges(), an xdi_diff_outf() that emits only the tracked range's lines, thread the filepair's line_ranges through the pickaxe callback, and pass it from pickaxe_match(). Skip scoping under textconv, whose output is not in the original file's line coordinates. -G needs only a hit/no-hit answer, so the line-number concerns the filter handles for patch and check output do not apply here. -S is left matching the whole file: it counts needle occurrences per blob rather than grepping the diff, so scoping it needs a different approach, left to a follow-up. has_changes() takes the range parameter but ignores it for now. Document the resulting -L pickaxe scoping: -G is scoped to the tracked range, while -S still matches the whole file. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- Documentation/line-range-options.adoc | 5 +- diff.c | 15 ++++++ diffcore-pickaxe.c | 37 +++++++++++--- t/t4211-line-log.sh | 72 +++++++++++++++++++++++++-- xdiff-interface.h | 13 +++++ 5 files changed, 130 insertions(+), 12 deletions(-) diff --git a/Documentation/line-range-options.adoc b/Documentation/line-range-options.adoc index 33b4e948815aac..d619ffa6336513 100644 --- a/Documentation/line-range-options.adoc +++ b/Documentation/line-range-options.adoc @@ -20,6 +20,9 @@ + Patch formatting options such as `--word-diff`, `--color-moved`, `--no-prefix`, and whitespace options (`-w`, `-b`) are supported, -as are pickaxe options (`-S`, `-G`) and `--diff-filter`. +as are pickaxe options (`-S`, `-G`) and `--diff-filter`. `-G` is +scoped to the tracked range; `-S` is still evaluated over the whole +file, so an `-S` query may select a commit for a change outside the +range. + include::line-range-format.adoc[] diff --git a/diff.c b/diff.c index 519c5133566032..a8f346621b9fb5 100644 --- a/diff.c +++ b/diff.c @@ -2817,6 +2817,21 @@ static int line_range_filter_diff(struct line_range_filter *filter, return ret; } +/* + * Expose the in-file line-range filter to callers outside diff.c (e.g. + * pickaxe -G); see xdiff-interface.h for the contract. + */ +int diff_emit_line_ranges(mmfile_t *one, mmfile_t *two, + const struct range_set *ranges, + xdiff_emit_line_fn line_fn, void *cb_data, + xpparam_t *xpp, xdemitconf_t *xecfg) +{ + struct line_range_filter filter; + + line_range_filter_init(&filter, ranges, line_fn, cb_data); + return line_range_filter_diff(&filter, one, two, xpp, xecfg); +} + static void pprint_rename(struct strbuf *name, const char *a, const char *b) { const char *old_name = a; diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index a52d569911c48e..047b2bf7ac65d5 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -16,7 +16,8 @@ typedef int (*pickaxe_fn)(mmfile_t *one, mmfile_t *two, struct diff_options *o, - regex_t *regexp, kwset_t kws); + regex_t *regexp, kwset_t kws, + const struct range_set *ranges); struct diffgrep_cb { regex_t *regexp; @@ -42,7 +43,8 @@ static int diffgrep_consume(void *priv, char *line, unsigned long len) static int diff_grep(mmfile_t *one, mmfile_t *two, struct diff_options *o, - regex_t *regexp, kwset_t kws UNUSED) + regex_t *regexp, kwset_t kws UNUSED, + const struct range_set *ranges) { struct diffgrep_cb ecbdata; xpparam_t xpp; @@ -50,8 +52,11 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, int ret; /* - * We have both sides; need to run textual diff and see if - * the pattern appears on added/deleted lines. + * We have both sides; need to run textual diff and see if the + * pattern appears on added/deleted lines. Under -L (ranges set), + * forward only the tracked range's lines so the match is scoped. + * -G needs only a hit/no-hit answer, so the line-number bookkeeping + * the filter does for -L patch and check output is irrelevant here. */ memset(&xpp, 0, sizeof(xpp)); memset(&xecfg, 0, sizeof(xecfg)); @@ -65,8 +70,12 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, * An xdiff error might be our "data->hit" from above. See the * comment for xdiff_emit_line_fn in xdiff-interface.h */ - ret = xdi_diff_outf(one, two, NULL, diffgrep_consume, - &ecbdata, &xpp, &xecfg); + if (ranges) + ret = diff_emit_line_ranges(one, two, ranges, diffgrep_consume, + &ecbdata, &xpp, &xecfg); + else + ret = xdi_diff_outf(one, two, NULL, diffgrep_consume, + &ecbdata, &xpp, &xecfg); if (ecbdata.hit) return 1; if (ret) @@ -119,8 +128,13 @@ static unsigned int contains(mmfile_t *mf, regex_t *regexp, kwset_t kws, static int has_changes(mmfile_t *one, mmfile_t *two, struct diff_options *o UNUSED, - regex_t *regexp, kwset_t kws) + regex_t *regexp, kwset_t kws, + const struct range_set *ranges UNUSED) { + /* + * -S counts needle occurrences in each whole blob. Scoping this to + * a -L range is left to a follow-up; for now -S ignores the range. + */ unsigned int c1 = one ? contains(one, regexp, kws, 0) : 0; unsigned int c2 = two ? contains(two, regexp, kws, c1 + 1) : 0; return c1 != c2; @@ -132,6 +146,7 @@ static int pickaxe_match(struct diff_filepair *p, struct diff_options *o, struct userdiff_driver *textconv_one = NULL; struct userdiff_driver *textconv_two = NULL; mmfile_t mf1, mf2; + const struct range_set *ranges; int ret; /* ignore unmerged */ @@ -169,7 +184,13 @@ static int pickaxe_match(struct diff_filepair *p, struct diff_options *o, mf1.size = fill_textconv(o->repo, textconv_one, p->one, &mf1.ptr); mf2.size = fill_textconv(o->repo, textconv_two, p->two, &mf2.ptr); - ret = fn(&mf1, &mf2, o, regexp, kws); + /* + * -L scopes the search to the tracked range, but the range is in + * original-file line coordinates that do not map onto textconv + * output, so search the whole file when textconv is in play. + */ + ranges = (textconv_one || textconv_two) ? NULL : p->line_ranges; + ret = fn(&mf1, &mf2, o, regexp, kws, ranges); if (textconv_one) free(mf1.ptr); diff --git a/t/t4211-line-log.sh b/t/t4211-line-log.sh index 68576418f41895..bac74d44794778 100755 --- a/t/t4211-line-log.sh +++ b/t/t4211-line-log.sh @@ -722,9 +722,9 @@ test_expect_success '-L with -S filters to string-count changes' ' test_expect_success '-L with -G filters to diff-text matches' ' git checkout parent-oids && git log -L:func2:file.c -G "F2 [+] 2" --format= >actual && - # -G greps the whole-file diff text, not just the tracked range; - # combined with -L, this selects commits that both touch func2 - # and have "F2 + 2" in their diff. + # -G greps the diff text, and under -L only the lines in the + # tracked range (unlike -S above, which searches the whole file); + # this selects commits whose change to func2 contains "F2 + 2". test $(grep -c "^diff --git" actual) = 1 && grep "F2 + 2" actual ' @@ -1110,4 +1110,70 @@ test_expect_success '--check does not report blank-at-eof outside the range' ' test_grep ! "blank line at EOF" actual ' +test_expect_success '-L -G is scoped to the tracked range' ' + git checkout --orphan grep-scope && + git reset --hard && + cat >gp.c <<-\EOF && + int func1() + { + return ALPHA; + } + + int func2() + { + return BETA; + } + EOF + git add gp.c && + test_tick && + git commit -m "add gp.c" && + sed -e "s/ALPHA/ALPHA2/" -e "s/BETA/BETA2/" gp.c >tmp && + mv tmp gp.c && + git commit -a -m "touch both functions" && + # The commit changes ALPHA (func1) and BETA (func2). Tracking func2, + # -G BETA matches its in-range change; -G ALPHA must not, since ALPHA + # changes only outside the tracked range. + git log -L:func2:gp.c -G BETA --format=%s >actual && + test_grep "touch both functions" actual && + git log -L:func2:gp.c -G ALPHA --format=%s >actual && + test_grep ! "touch both functions" actual +' + +test_expect_success '-L -G searches the whole file under textconv' ' + git checkout --orphan grep-textconv && + git reset --hard && + cat >tc.c <<-\EOF && + int func1() + { + return F1; + } + + int func2() + { + return F2; + } + EOF + git add tc.c && + test_tick && + git commit -m "add tc.c" && + # One commit changes func1 and func2; MAGIC lands only in the + # func2 change, outside func1. + sed -e "s/F1/F1 + 1/" -e "s/return F2/return MAGIC/" tc.c >tmp && + mv tmp tc.c && + git commit -a -m "change both funcs" && + echo "tc.c diff=tc" >.gitattributes && + + # Without a textconv driver, -G is scoped to func1, so MAGIC (only + # in the func2 change) does not select the commit. + git log -L:func1:tc.c -G MAGIC --format=%s --no-patch >actual && + test_must_be_empty actual && + + # A textconv driver makes the range (original-file line numbers) + # meaningless against the driver output, so -G falls back to the + # whole file and MAGIC now selects the commit. + git config diff.tc.textconv cat && + git log -L:func1:tc.c -G MAGIC --format=%s --no-patch >actual && + test_grep "change both funcs" actual +' + test_done diff --git a/xdiff-interface.h b/xdiff-interface.h index 51c88296ed5e68..71e5dffefb8ded 100644 --- a/xdiff-interface.h +++ b/xdiff-interface.h @@ -46,6 +46,19 @@ int xdi_diff_outf(mmfile_t *mf1, mmfile_t *mf2, xdiff_emit_line_fn line_fn, void *consume_callback_data, xpparam_t const *xpp, xdemitconf_t const *xecfg); + +struct range_set; +/* + * Like xdi_diff_outf(), but forwards only the lines within the given + * (post-image) line ranges to line_fn, as "git log -L" scopes its output. + * Returns line_fn's latched return value (so a consumer can signal a hit + * with a non-zero return), or non-zero on xdiff failure. Defined in + * diff.c (it reuses the line-range filter there). + */ +int diff_emit_line_ranges(mmfile_t *mf1, mmfile_t *mf2, + const struct range_set *ranges, + xdiff_emit_line_fn line_fn, void *cb_data, + xpparam_t *xpp, xdemitconf_t *xecfg); int read_mmfile(mmfile_t *ptr, const char *filename); void read_mmblob(mmfile_t *ptr, struct object_database *odb, const struct object_id *oid); From ffe5c33a4423b78e893e340db7387aec8befada7 Mon Sep 17 00:00:00 2001 From: Zephyr Yao Date: Thu, 2 Jul 2026 00:17:59 -0400 Subject: [PATCH 013/259] apply: avoid leaking abandoned git-header state When find_header() sees a "diff --git" line, it calls parse_git_diff_header() to parse the git-style extended header. That parser updates the caller's struct patch as it goes, filling in the default name, old/new names, and new/delete state. But not every "diff --git" line found while scanning is ultimately accepted as the patch header. If parse_git_diff_header() returns a length that covers only the "diff --git" line, find_header() continues scanning for another header. In that case the partially parsed git-header state must not interfere with the later traditional "---" / "+++" header. Leaving that state behind can combine incompatible metadata from the abandoned git header and the later traditional header. For example, after: diff --git a/foo b/foo --- /dev/null +++ b/foo @@ -0,0 +1 @@ +x the abandoned git header can leave an old name in the patch, while the traditional header marks the patch as creating a new file. That impossible state later trips the check_preimage() assertion that a creation patch should not have a preimage. Parse a candidate git header into a temporary patch and line number. Commit that temporary state to the real patch only when the git header is actually accepted; otherwise release it and keep scanning with the original patch state unchanged. Also reject an empty parsed default name from the "diff --git" line. An empty patch->def_name is not a valid pathname, and should not be used later as a fallback when old_name and new_name are missing. Add regression tests for both the empty default-name case and the non-empty abandoned-header case above. Co-authored-by: Mahya SamDaliri Signed-off-by: Mahya SamDaliri Co-authored-by: Haotian Zhang Signed-off-by: Haotian Zhang Co-authored-by: Martin Kellogg Signed-off-by: Martin Kellogg Signed-off-by: Zephyr Yao Signed-off-by: Junio C Hamano --- apply.c | 29 ++++++++++++++++++++++------- t/t4100-apply-stat.sh | 25 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/apply.c b/apply.c index 4aa1694cfaa2f0..e01850255808fd 100644 --- a/apply.c +++ b/apply.c @@ -1362,6 +1362,9 @@ int parse_git_diff_header(struct strbuf *root, * the default name from the header. */ patch->def_name = git_header_name(p_value, line, len); + if (patch->def_name && !*patch->def_name) + FREE_AND_NULL(patch->def_name); + if (patch->def_name && root->len) { char *s = xstrfmt("%s%s", root->buf, patch->def_name); free(patch->def_name); @@ -1632,15 +1635,27 @@ static int find_header(struct apply_state *state, * or mode change, so we handle that specially */ if (!memcmp("diff --git ", line, 11)) { - int git_hdr_len = parse_git_diff_header(&state->root, - state->patch_input_file, - &state->linenr, - state->p_value, line, len, - size, patch); - if (git_hdr_len < 0) + struct patch git_patch = { 0 }; + int git_linenr = state->linenr; + int git_hdr_len; + + git_patch.inaccurate_eof = patch->inaccurate_eof; + git_patch.recount = patch->recount; + git_hdr_len = parse_git_diff_header(&state->root, + state->patch_input_file, + &git_linenr, + state->p_value, line, len, + size, &git_patch); + if (git_hdr_len < 0) { + release_patch(&git_patch); return -128; - if (git_hdr_len <= len) + } + if (git_hdr_len <= len) { + release_patch(&git_patch); continue; + } + *patch = git_patch; + state->linenr = git_linenr; *hdrsize = git_hdr_len; return offset; } diff --git a/t/t4100-apply-stat.sh b/t/t4100-apply-stat.sh index 8393076469e083..d3406edaddbe11 100755 --- a/t/t4100-apply-stat.sh +++ b/t/t4100-apply-stat.sh @@ -113,6 +113,31 @@ test_expect_success 'applying a patch with a missing filename reports the input' test_cmp expect err ' +test_expect_success 'empty default filename reports the input' ' + cat >empty-name.patch <<-\EOF && + diff --git "a/""b/" + + --- /dev/null + +++ " + @@ -0,0 +1 @@ + + + EOF + test_must_fail git apply empty-name.patch 2>err && + test_grep "git diff header lacks filename information" err +' + +test_expect_success 'abandoned git header does not reuse names' ' + cat >abandoned-git-header.patch <<-\EOF && + diff --git a/foo b/foo + + --- /dev/null + +++ b/foo + @@ -0,0 +1 @@ + +x + EOF + git apply --check abandoned-git-header.patch +' + test_expect_success 'applying a patch with an invalid mode reports the input' ' cat >mode.patch <<-\EOF && diff --git a/f b/f From cd6ad979ab5a81800c40e32e7e92bdc6f0ec0090 Mon Sep 17 00:00:00 2001 From: Ian Jackson Date: Mon, 6 Jul 2026 12:58:15 +0100 Subject: [PATCH 014/259] git-subtree: Bail out if we find output from Rust rewrite This is going to be forward compatible, but not backward compatible: projects are expected to adopt the new tool, but not go back to this old one. CC: Colin Stagner CC: Johannes Schindelin Signed-off-by: Ian Jackson Signed-off-by: Junio C Hamano --- contrib/subtree/git-subtree.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/contrib/subtree/git-subtree.sh b/contrib/subtree/git-subtree.sh index 791fd8260c4703..e9c7ca7cf513b1 100755 --- a/contrib/subtree/git-subtree.sh +++ b/contrib/subtree/git-subtree.sh @@ -278,6 +278,20 @@ main () { "cmd_$arg_command" "$@" } +# Usage: reject_if_v2_config REV +# +# Bails if we find .git-subtree/config. This file is used by the RIIR +# git-subtree, which can read data from this script, but which generates +# data that this script cannot cope with. So if we find that the user's +# project has already been processed with the new tool, we stop, to +# avoid generating broken output. +reject_if_v2_config () { + local config=.git-subtree/config + if git rev-parse --verify -q "$rev:$config"; then + die "fatal: tree contains $config: has been processed with new standalone (Rust) git-subtree; use that tool instead of this one. See https://codeberg.org/diziet/git-subtree https://crates.io/crates/git-subtree" + fi +} + # Usage: cache_setup cache_setup () { assert test $# = 0 @@ -846,6 +860,7 @@ process_split_commit () { # Or: cmd_add REPOSITORY REF cmd_add () { + reject_if_v2_config HEAD ensure_clean if test $# -eq 1 @@ -934,6 +949,8 @@ cmd_split () { die "fatal: you must provide exactly one revision, and optionally a repository. Got: '$*'" fi + reject_if_v2_config "$rev" + # Now validate prefix against the commit, not the working tree if ! git cat-file -e "$rev:$dir" 2>/dev/null then @@ -1034,6 +1051,7 @@ cmd_merge () { then repository="$2" fi + reject_if_v2_config HEAD ensure_clean if test -n "$arg_addmerge_squash" From 20b117c1eb19c0e08ede3c7285c991eac4c9345f Mon Sep 17 00:00:00 2001 From: Ian Jackson Date: Mon, 6 Jul 2026 12:58:16 +0100 Subject: [PATCH 015/259] git-subtree: Bail out if we find output from Rust rewrite (test) Signed-off-by: Ian Jackson Signed-off-by: Junio C Hamano --- contrib/subtree/t/t7900-subtree.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/contrib/subtree/t/t7900-subtree.sh b/contrib/subtree/t/t7900-subtree.sh index 4194687cfbb9b5..e8fa6401668991 100755 --- a/contrib/subtree/t/t7900-subtree.sh +++ b/contrib/subtree/t/t7900-subtree.sh @@ -439,6 +439,24 @@ test_expect_success 'split sub dir/ with --rejoin' ' ) ' +test_expect_success 'split fail on RIIR git subtree data' ' + subtree_test_create_repo "$test_count" && + subtree_test_create_repo "$test_count/sub proj" && + test_create_commit "$test_count" main1 && + test_create_commit "$test_count/sub proj" sub1 && + ( + cd "$test_count" && + git fetch ./"sub proj" HEAD && + git subtree add --prefix="sub dir" FETCH_HEAD && + # simulate RIIR git-subtree generated data + mkdir .git-subtree && + echo "# sabotage" >.git-subtree/config && + git add .git-subtree/config && + git commit -m sabotage && + test_must_fail git subtree split -P "sub dir" HEAD + ) +' + # Tests that commits from other subtrees are not processed as # part of a split. # From d0ee845a718a447cacabecf9610d2e1da6d83330 Mon Sep 17 00:00:00 2001 From: Gatla Vishweshwar Reddy Date: Sat, 11 Jul 2026 11:36:15 +0530 Subject: [PATCH 016/259] builtin/add.c: replace run_command() with direct apply_all_patches() call When the user runs "git add -e", the diff of the working tree changes is written to a temporary file, opened in an editor, and then applied back to the index. The application step is done by spawning a child process running "git apply --recount --cached ", which is an unnecessary subprocess since the apply machinery is available as a native C API. Replace the run_command() call with a direct call to apply_all_patches() using an initialized apply_state with the cached and recount options set appropriately. This avoids the overhead of forking a subprocess, keeps the operation within the same process, and makes the intent of the code clearer to the reader. Remove the now-unused includes of "run-command.h" and "strvec.h" since no other code in this file requires them after this change. Signed-off-by: Gatla Vishweshwar Reddy Signed-off-by: Junio C Hamano --- builtin/add.c | 19 ++++++++++++------- t/t3702-add-edit.sh | 10 ++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/builtin/add.c b/builtin/add.c index c859f665199efa..20a86a16114191 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -13,7 +13,6 @@ #include "dir.h" #include "gettext.h" #include "pathspec.h" -#include "run-command.h" #include "object-file.h" #include "odb.h" #include "odb/transaction.h" @@ -23,9 +22,9 @@ #include "diff.h" #include "read-cache.h" #include "revision.h" -#include "strvec.h" #include "submodule.h" #include "add-interactive.h" +#include "apply.h" static const char * const builtin_add_usage[] = { N_("git add [] [--] ..."), @@ -187,7 +186,8 @@ static int edit_patch(struct repository *repo, const char *prefix) { char *file = repo_git_path(repo, "ADD_EDIT.patch"); - struct child_process child = CHILD_PROCESS_INIT; + struct apply_state state; + const char *apply_argv[2]; struct rev_info rev; int out; struct stat st; @@ -217,11 +217,16 @@ static int edit_patch(struct repository *repo, if (!st.st_size) die(_("empty patch. aborted")); - child.git_cmd = 1; - strvec_pushl(&child.args, "apply", "--recount", "--cached", file, - NULL); - if (run_command(&child)) + apply_argv[0] = file; + apply_argv[1] = NULL; + if (init_apply_state(&state, repo, NULL)) + die(_("could not initialize apply state")); + state.cached = 1; + if (check_apply_state(&state, 0)) + die(_("could not check apply state")); + if (apply_all_patches(&state, 1, apply_argv, APPLY_OPT_RECOUNT)) die(_("could not apply '%s'"), file); + clear_apply_state(&state); unlink(file); free(file); diff --git a/t/t3702-add-edit.sh b/t/t3702-add-edit.sh index 8bacacbac6807c..f6285640051e59 100755 --- a/t/t3702-add-edit.sh +++ b/t/t3702-add-edit.sh @@ -124,5 +124,15 @@ test_expect_success 'add -e notices editor failure' ' test_must_fail env GIT_EDITOR=false git add -e && test_expect_code 1 git diff --exit-code ' +test_expect_success 'add -e works from a subdirectory' ' + git reset --hard && + echo change >>file && + mkdir -p subdir && + ( + cd subdir && + GIT_EDITOR=cat git add -e ../file + ) && + git diff --cached | grep -q "^+change" +' test_done From 8e6737046c30da3afab0bb9aeefbf3ffffac9652 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 15 Jul 2026 16:12:11 +0000 Subject: [PATCH 017/259] trace2: tolerate failed timestamp formatting Some users reported issues of repeated messages: fatal: recursion detected in die handler This wasn't happening every time, but we eventually captured a GIT_TRACE2_PERF log file with this issue and revealed an interesting internal detail, failing with this message: unable to format message: %4d-%02d-%02dT%02d:%02d:%02d.%06ldZ This specific format string tracks to tr2_tbuf_utc_datetime_extended() in trace2/tr2_tbuf.c. This logic began as tr2_tbuf_utc_time() in ee4512ed481 (trace2: create new combined trace facility, 2019-02-22) but was later split in bad229aef23 (trace2: clarify UTC datetime formatting, 2019-04-15). This use of xsnprintf() is writing a very specific datetime format into a 32-character buffer. The format requires that the input data will not overflow the format digits or the buffer will not hold the result. Since we are using xsnprintf() here, those failures turn into die() events. This method and its siblings, tr2_tbuf_local_time() and tr2_tbuf_utc_datetime(), are used in the tracing library. The extended form is used only for the 'event' format, which these users were using via a config setting for use in client-side telemetry. The non-extended form is used to help generate the 'SID' that defines the process in the traces. Not only are these inappropriate times for a failure, but the extended method is called specifially during the 'atexit' event, which was triggering this problem in a loop as the 'atexit' event would be retriggered by the die(). I could not determine the exact cause of why these errors started occuring in a bunch. My best guess is that these users are dogfooding an early operating system version that is more likely to fail in the gettimeofday() function and thus leaves the structures uninitialized and potentially violating the expected values. However, for full defense-in-depth I made several modifications: 1. Both 'tv' and 'tm' structs are initialized with zero values, allowing an erroring gettimeofday() or gmtime_r() method to leave them zero-valued. A zero-valued date is better than a die() here. 2. Replace the use of xsnprintf() with snprintf() to avoid the possibility of calling die() here. Instead, check the response to see if there was a failure. On failure, put a blank value into the buffer instead of possibly allowing a value that would not format correctly for a trace2 consumer. This value should be seen as obviously wrong and therefore signals a problem. As the core issue in this code seems to require a system method returning an error, no test accompanies this change. This change removes all uses of xsnprintf() from the trace2/ directory. There are two uses of xstrdup() that could be considered for removal, but they only die() on out-of-memory errors instead of formatting issues. I chose to leave those in place for now. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- trace2/tr2_tbuf.c | 49 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/trace2/tr2_tbuf.c b/trace2/tr2_tbuf.c index c3b3822ed7e4af..ef57376f3c3e24 100644 --- a/trace2/tr2_tbuf.c +++ b/trace2/tr2_tbuf.c @@ -3,45 +3,64 @@ void tr2_tbuf_local_time(struct tr2_tbuf *tb) { - struct timeval tv; - struct tm tm; + struct timeval tv = { 0 }; + struct tm tm = { 0 }; time_t secs; + int len; gettimeofday(&tv, NULL); secs = tv.tv_sec; localtime_r(&secs, &tm); - xsnprintf(tb->buf, sizeof(tb->buf), "%02d:%02d:%02d.%06ld", tm.tm_hour, - tm.tm_min, tm.tm_sec, (long)tv.tv_usec); + len = snprintf(tb->buf, sizeof(tb->buf), "%02d:%02d:%02d.%06ld", + tm.tm_hour, tm.tm_min, tm.tm_sec, (long)tv.tv_usec); + + if (len < 0 || (size_t)len >= sizeof(tb->buf)) { + const char *blank = "00:00:00.000000"; + strlcpy(tb->buf, blank, sizeof(tb->buf)); + } } void tr2_tbuf_utc_datetime_extended(struct tr2_tbuf *tb) { - struct timeval tv; - struct tm tm; + struct timeval tv = { 0 }; + struct tm tm = { 0 }; time_t secs; + int len; gettimeofday(&tv, NULL); secs = tv.tv_sec; gmtime_r(&secs, &tm); - xsnprintf(tb->buf, sizeof(tb->buf), - "%4d-%02d-%02dT%02d:%02d:%02d.%06ldZ", tm.tm_year + 1900, - tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, - (long)tv.tv_usec); + len = snprintf(tb->buf, sizeof(tb->buf), + "%4d-%02d-%02dT%02d:%02d:%02d.%06ldZ", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec, (long)tv.tv_usec); + + if (len < 0 || (size_t)len >= sizeof(tb->buf)) { + const char *blank = "1900-00-00T00:00:00.000000Z"; + strlcpy(tb->buf, blank, sizeof(tb->buf)); + } } void tr2_tbuf_utc_datetime(struct tr2_tbuf *tb) { - struct timeval tv; - struct tm tm; + struct timeval tv = { 0 }; + struct tm tm = { 0 }; time_t secs; + int len; gettimeofday(&tv, NULL); secs = tv.tv_sec; gmtime_r(&secs, &tm); - xsnprintf(tb->buf, sizeof(tb->buf), "%4d%02d%02dT%02d%02d%02d.%06ldZ", - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, - tm.tm_min, tm.tm_sec, (long)tv.tv_usec); + len = snprintf(tb->buf, sizeof(tb->buf), + "%4d%02d%02dT%02d%02d%02d.%06ldZ", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec, (long)tv.tv_usec); + + if (len < 0 || (size_t)len >= sizeof(tb->buf)) { + const char *blank = "19000000T000000.000000Z"; + strlcpy(tb->buf, blank, sizeof(tb->buf)); + } } From d1e198ff9ebd2f3af5fb281f618f5e088489971e Mon Sep 17 00:00:00 2001 From: Paulius Zaleckas Date: Thu, 16 Jul 2026 17:09:53 +0300 Subject: [PATCH 018/259] submodule: fix premature failure in recursive submodule fetch When git fetch --recurse-submodules encounters a failure fetching a submodule's refs (phase 1), it immediately marks the overall operation as failed, even though a subsequent OID-based fetch (phase 2) is about to be attempted for any missing commits. If phase 2 succeeds, the overall result should be success, but the prematurely set failure flag makes it look like an error. Restructure fetch_finish() so that a phase-1 failure does not record an error immediately. Instead, the decision is deferred: - If missing commits trigger a phase-2 (OID-based) retry and that retry succeeds, no error is recorded. - If the phase-2 retry also fails, the error is recorded then. - If the submodule was fetched unconditionally (RECURSE_SUBMODULES_ON) and is not in the changed list, a phase-1 failure is recorded right away since there is no OID retry to fall back on. - If phase 1 fails but all required commits are already present locally, there is no retry to defer to; the failure is still recorded, since the fetch itself went wrong (e.g. a transport error) even though the wanted commits happen to be available. This resolves the NEEDSWORK comment added by bd5e567dc7 (submodule: explain first attempt failure clearly, 2019-03-13). Extract the common error-recording logic into a helper record_fetch_error() and use it in fetch_start_failure() and for the "Could not access submodule" error in get_fetch_task_from_index() as well; the latter now also lists the submodule in the final error summary. Add a test ensuring a failed submodule fetch is still reported when the gitlinked commits happen to be present locally. Helped-by: Ramsay Jones Signed-off-by: Paulius Zaleckas Signed-off-by: Junio C Hamano --- submodule.c | 52 +++++++++++++++++++-------- t/t5526-fetch-submodules.sh | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 14 deletions(-) diff --git a/submodule.c b/submodule.c index fd91201a92d7b0..8bcef68a42480d 100644 --- a/submodule.c +++ b/submodule.c @@ -1562,6 +1562,13 @@ static struct fetch_task *fetch_task_create(struct submodule_parallel_fetch *spf return NULL; } +static void record_fetch_error(struct submodule_parallel_fetch *spf, + const char *name) +{ + spf->result = 1; + strbuf_addf(&spf->submodules_with_errors, "\t%s\n", name); +} + static struct fetch_task * get_fetch_task_from_index(struct submodule_parallel_fetch *spf, struct strbuf *err) @@ -1599,7 +1606,7 @@ get_fetch_task_from_index(struct submodule_parallel_fetch *spf, ce->name); if (S_ISGITLINK(ce->ce_mode) && !is_empty_dir(empty_submodule_path.buf)) { - spf->result = 1; + record_fetch_error(spf, ce->name); strbuf_addf(err, _("Could not access submodule '%s'\n"), ce->name); @@ -1753,7 +1760,7 @@ static int fetch_start_failure(struct strbuf *err UNUSED, struct submodule_parallel_fetch *spf = cb; struct fetch_task *task = task_cb; - spf->result = 1; + record_fetch_error(spf, task->sub->name); fetch_task_free(task); return 0; @@ -1779,18 +1786,12 @@ static int fetch_finish(int retvalue, struct strbuf *err UNUSED, if (!task || !task->sub) BUG("callback cookie bogus"); - if (retvalue) { + if (retvalue && task->commits) { /* - * NEEDSWORK: This indicates that the overall fetch - * failed, even though there may be a subsequent fetch - * by commit hash that might work. It may be a good - * idea to not indicate failure in this case, and only - * indicate failure if the subsequent fetch fails. + * This is the second pass (OID-based fetch) and it failed. + * The commits are genuinely unavailable from the remote. */ - spf->result = 1; - - strbuf_addf(&spf->submodules_with_errors, "\t%s\n", - task->sub->name); + record_fetch_error(spf, task->sub->name); } /* Is this the second time we process this submodule? */ @@ -1798,9 +1799,17 @@ static int fetch_finish(int retvalue, struct strbuf *err UNUSED, goto out; it = string_list_lookup(&spf->changed_submodule_names, task->sub->name); - if (!it) - /* Could be an unchanged submodule, not contained in the list */ + if (!it) { + /* + * This submodule is not in the changed list (e.g. it was + * fetched because RECURSE_SUBMODULES_ON fetches all populated + * submodules). A phase 1 failure here has no OID-based retry + * to fall back on, so it is a genuine error. + */ + if (retvalue) + record_fetch_error(spf, task->sub->name); goto out; + } cs_data = it->util; oid_array_filter(&cs_data->new_commits, @@ -1809,6 +1818,11 @@ static int fetch_finish(int retvalue, struct strbuf *err UNUSED, /* Are there commits we want, but do not exist? */ if (cs_data->new_commits.nr) { + /* + * Schedule an OID-based phase 2 fetch to retrieve the missing + * commits directly. Defer any error from phase 1: if phase 2 + * succeeds, the overall operation should still succeed. + */ task->commits = &cs_data->new_commits; ALLOC_GROW(spf->oid_fetch_tasks, spf->oid_fetch_tasks_nr + 1, @@ -1818,6 +1832,16 @@ static int fetch_finish(int retvalue, struct strbuf *err UNUSED, return 0; } + /* + * All required commits are already present locally (they were either + * fetched by phase 1 or existed beforehand), so there is no phase 2 + * retry to defer to. If phase 1 failed, the fetch itself went wrong + * (e.g. a transport error) and must still be reported, even though + * the gitlinked commits are available. + */ + if (retvalue) + record_fetch_error(spf, task->sub->name); + out: fetch_task_free(task); return 0; diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh index 1242ee918526ea..7ad274ce040afd 100755 --- a/t/t5526-fetch-submodules.sh +++ b/t/t5526-fetch-submodules.sh @@ -1262,4 +1262,76 @@ test_expect_success "fetch --all with --no-recurse-submodules only fetches super ! grep "Fetching submodule" fetch-log ' +# Create an isolated environment for submodule fetch error tests. +# +# Sets up sub_bare (the submodule upstream), super_bare (the superproject +# upstream), super_work (a working clone of super_bare with an initialized +# submodule), and clone (a clone of super_bare with an initialized submodule +# at a reachable commit). The caller can then create an unreachable commit +# and push the superproject to put the clone one commit behind a state it +# cannot fully fetch. +# +# Usage: create_err_env +create_err_env () { + local envdir="$1" && + mkdir "$envdir" && + + git init --bare "$envdir/sub_bare" && + git clone "$envdir/sub_bare" "$envdir/sub_work" && + test_commit -C "$envdir/sub_work" "${envdir}_base" && + git -C "$envdir/sub_work" push && + + git init --bare "$envdir/super_bare" && + git clone "$envdir/super_bare" "$envdir/super_work" && + git -C "$envdir/super_work" submodule add \ + "$pwd/$envdir/sub_bare" sub && + git -C "$envdir/super_work" commit -m "add submodule" && + git -C "$envdir/super_work" push && + + git clone "$envdir/super_bare" "$envdir/clone" && + git -C "$envdir/clone" submodule update --init +} + +# Push a commit to /super_bare that records a submodule SHA that is +# present locally in super_work/sub but NOT pushed to sub_bare, making the +# submodule commit unreachable from clone's sub remote. +push_unreachable_commit () { + local envdir="$1" && + git -C "$envdir/super_work/sub" commit --allow-empty -m "unreachable" && + git -C "$envdir/super_work" add sub && + git -C "$envdir/super_work" commit -m "point sub to unreachable commit" && + git -C "$envdir/super_work" push +} + +test_expect_success 'setup for submodule fetch error tests' ' + git config --global protocol.file.allow always +' + +test_expect_success 'failed submodule fetch is fatal even when its commits are present locally' ' + # Create the same commit (unreferenced, via commit-tree with fixed + # dates) in both super_work/sub and clone/sub, point the gitlink at + # it, and break clone/sub'\''s remote. The commit exists in clone/sub + # but is unreachable, so the submodule stays in the changed list; the + # fetch failure must still be reported even though there is nothing + # left to fetch by commit hash. + test_when_finished "rm -fr env_phase1" && + create_err_env env_phase1 && + commit=$(GIT_AUTHOR_DATE="1234567890 +0000" \ + GIT_COMMITTER_DATE="1234567890 +0000" \ + git -C env_phase1/super_work/sub commit-tree \ + "HEAD^{tree}" -p HEAD -m present) && + present=$(GIT_AUTHOR_DATE="1234567890 +0000" \ + GIT_COMMITTER_DATE="1234567890 +0000" \ + git -C env_phase1/clone/sub commit-tree \ + "HEAD^{tree}" -p HEAD -m present) && + test "$commit" = "$present" && + git -C env_phase1/super_work/sub checkout "$commit" && + git -C env_phase1/super_work add sub && + git -C env_phase1/super_work commit -m "gitlink to locally-present commit" && + git -C env_phase1/super_work push && + git -C env_phase1/clone/sub remote set-url origin "$pwd/env_phase1/missing" && + test_must_fail git -C env_phase1/clone fetch --recurse-submodules 2>err && + test_grep "Errors during submodule fetch" err +' + test_done From 0b977622b8434e6753425fb66024e2fd93bba0c8 Mon Sep 17 00:00:00 2001 From: Paulius Zaleckas Date: Thu, 16 Jul 2026 17:09:54 +0300 Subject: [PATCH 019/259] fetch: add fetch.submoduleErrors to make submodule fetch errors non-fatal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When fetching with --recurse-submodules, a submodule commit that is not yet reachable from any of the submodule's remote refs causes the entire fetch to fail. This is overly strict when the missing commit belongs to an upstream branch that is still being prepared (e.g. an in-progress merge topic): the local branch does not need that commit, so there is no reason to treat its absence as fatal. Add a new config key fetch.submoduleErrors (values: fail/warn) and a corresponding --submodule-errors=(fail|warn) command-line option that control this behaviour. The default remains fail (existing behaviour); setting the value to warn causes submodule fetch failures to be reported on stderr without affecting the overall exit status of git fetch / git pull. Forward the option to child fetches in add_options_to_argv() so that it also takes effect for `git fetch --all` / `--multiple` (where per-remote child processes handle the submodule recursion themselves) and for nested submodule recursion. The resolved value is forwarded whenever it was set explicitly, in either direction: the per-remote children re-read the repository configuration, so a command-line --submodule-errors=fail must be passed down to them to override fetch.submoduleErrors=warn from the configuration. When neither the configuration nor the command line sets a value, nothing is forwarded and the child processes fall back to their own configuration. Helped-by: Jean-Noël Avila Helped-by: Ramsay Jones Helped-by: Junio C Hamano Signed-off-by: Paulius Zaleckas Signed-off-by: Junio C Hamano --- Documentation/config/fetch.adoc | 14 +++++ Documentation/fetch-options.adoc | 8 +++ builtin/fetch.c | 70 ++++++++++++++++++++++++- submodule.c | 8 ++- submodule.h | 7 ++- t/t5526-fetch-submodules.sh | 89 ++++++++++++++++++++++++++++++++ 6 files changed, 192 insertions(+), 4 deletions(-) diff --git a/Documentation/config/fetch.adoc b/Documentation/config/fetch.adoc index 04ac90912d3a58..5c9c942a704cf3 100644 --- a/Documentation/config/fetch.adoc +++ b/Documentation/config/fetch.adoc @@ -10,6 +10,20 @@ reference. Defaults to `on-demand`, or to the value of `submodule.recurse` if set. +`fetch.submoduleErrors`:: + Controls how errors from submodule fetches are handled when + `--recurse-submodules` is in effect. When set to `fail` (the default), + any submodule fetch error causes the overall `git fetch` or `git pull` + to exit with a non-zero status. When set to `warn`, submodule fetch + errors are reported to standard error but do not affect the exit + status of the command. This is useful when working in repositories + where some branches reference submodule commits that are not yet + available on the submodule remote, but those commits are not needed + for the currently checked-out branch. ++ +The value of this option can be overridden by the `--submodule-errors` +option of linkgit:git-fetch[1]. + `fetch.fsckObjects`:: If it is set to true, git-fetch-pack will check all fetched objects. See `transfer.fsckObjects` for what's diff --git a/Documentation/fetch-options.adoc b/Documentation/fetch-options.adoc index 035f780e583cee..78525f6848056f 100644 --- a/Documentation/fetch-options.adoc +++ b/Documentation/fetch-options.adoc @@ -294,6 +294,14 @@ ifndef::git-pull[] `--no-recurse-submodules`:: Disable recursive fetching of submodules (this has the same effect as using the `--recurse-submodules=no` option). + +`--submodule-errors=(fail|warn)`:: + Control how errors from submodule fetches are handled when + `--recurse-submodules` is in effect. When set to `fail` (the default), + any submodule fetch error causes the overall `git fetch` to exit with a + non-zero status. When set to `warn`, submodule fetch errors are reported + to standard error but do not affect the exit status of the command. Can + also be configured via `fetch.submoduleErrors`. See linkgit:git-config[1]. endif::git-pull[] `--set-upstream`:: diff --git a/builtin/fetch.c b/builtin/fetch.c index c1d7c672f4e0d8..2c583ed0cc8508 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -110,8 +110,30 @@ struct fetch_config { int recurse_submodules; int parallel; int submodule_fetch_jobs; + int submodule_errors; }; +/* really private - use accessors below to parse and format */ +static const char *submodule_error_name[] = { + [SUBMODULE_ERRORS_FAIL] = "fail", + [SUBMODULE_ERRORS_WARN] = "warn", +}; + +static const char *submodule_error(unsigned num) +{ + if (ARRAY_SIZE(submodule_error_name) <= num) + BUG("invalid submodule errors mode %u", num); + return submodule_error_name[num]; +} + +static int parse_submodule_error(const char *name) +{ + for (unsigned num = 0; num < ARRAY_SIZE(submodule_error_name); num++) + if (!strcmp(submodule_error_name[num], name)) + return num; + return -1; +} + static int git_fetch_config(const char *k, const char *v, const struct config_context *ctx, void *cb) { @@ -152,6 +174,19 @@ static int git_fetch_config(const char *k, const char *v, return 0; } + if (!strcmp(k, "fetch.submoduleerrors")) { + int mode; + + if (!v) + return config_error_nonbool(k); + mode = parse_submodule_error(v); + if (mode < 0) + die(_("invalid value for '%s': '%s'"), + "fetch.submoduleErrors", v); + fetch_config->submodule_errors = mode; + return 0; + } + if (!strcmp(k, "fetch.parallel")) { fetch_config->parallel = git_config_int(k, v, ctx->kvi); if (fetch_config->parallel < 0) @@ -2205,6 +2240,9 @@ static void add_options_to_argv(struct strvec *argv, strvec_push(argv, "--no-recurse-submodules"); else if (config->recurse_submodules == RECURSE_SUBMODULES_ON_DEMAND) strvec_push(argv, "--recurse-submodules=on-demand"); + if (config->submodule_errors != -1) + strvec_pushf(argv, "--submodule-errors=%s", + submodule_error(config->submodule_errors)); if (tags == TAGS_SET) strvec_push(argv, "--tags"); else if (tags == TAGS_UNSET) @@ -2464,6 +2502,23 @@ static int fetch_one(struct remote *remote, int argc, const char **argv, return exit_code; } +static int option_parse_submodule_errors(const struct option *opt, + const char *arg, int unset) +{ + int *v = opt->value; + int mode; + + if (unset) { + *v = SUBMODULE_ERRORS_FAIL; + return 0; + } + mode = parse_submodule_error(arg); + if (mode < 0) + die(_("invalid value for '%s': '%s'"), "--submodule-errors", arg); + *v = mode; + return 0; +} + int cmd_fetch(int argc, const char **argv, const char *prefix, @@ -2477,6 +2532,7 @@ int cmd_fetch(int argc, .recurse_submodules = RECURSE_SUBMODULES_DEFAULT, .parallel = 1, .submodule_fetch_jobs = -1, + .submodule_errors = -1, /* unset */ }; const char *submodule_prefix = ""; const char *bundle_uri; @@ -2491,6 +2547,7 @@ int cmd_fetch(int argc, int max_jobs = -1; int recurse_submodules_cli = RECURSE_SUBMODULES_DEFAULT; int recurse_submodules_default = RECURSE_SUBMODULES_ON_DEMAND; + int submodule_errors_cli = -1; /* -1: not set on command line */ int fetch_write_commit_graph = -1; int stdin_refspecs = 0; int negotiate_only = 0; @@ -2527,6 +2584,10 @@ int cmd_fetch(int argc, OPT_CALLBACK_F(0, "recurse-submodules", &recurse_submodules_cli, N_("on-demand"), N_("control recursive fetching of submodules"), PARSE_OPT_OPTARG, option_fetch_parse_recurse_submodules), + OPT_CALLBACK_F(0, "submodule-errors", &submodule_errors_cli, + N_("(fail|warn)"), + N_("control how submodule fetch errors are handled"), + 0, option_parse_submodule_errors), OPT_BOOL(0, "dry-run", &dry_run, N_("dry run")), OPT_BOOL(0, "porcelain", &porcelain, N_("machine-readable output")), @@ -2616,6 +2677,9 @@ int cmd_fetch(int argc, if (recurse_submodules_cli != RECURSE_SUBMODULES_DEFAULT) config.recurse_submodules = recurse_submodules_cli; + if (submodule_errors_cli != -1) + config.submodule_errors = submodule_errors_cli; + if (negotiate_only) { switch (recurse_submodules_cli) { case RECURSE_SUBMODULES_OFF: @@ -2819,11 +2883,14 @@ int cmd_fetch(int argc, if (!result && remote && (config.recurse_submodules != RECURSE_SUBMODULES_OFF)) { struct strvec options = STRVEC_INIT; int max_children = max_jobs; + int submodule_errors = config.submodule_errors; if (max_children < 0) max_children = config.submodule_fetch_jobs; if (max_children < 0) max_children = config.parallel; + if (submodule_errors < 0) + submodule_errors = SUBMODULE_ERRORS_FAIL; add_options_to_argv(&options, &config); trace2_region_enter_printf("fetch", "recurse-submodule", the_repository, "%s", submodule_prefix); @@ -2833,7 +2900,8 @@ int cmd_fetch(int argc, config.recurse_submodules, recurse_submodules_default, verbosity < 0, - max_children); + max_children, + submodule_errors); trace2_region_leave_printf("fetch", "recurse-submodule", the_repository, "%s", submodule_prefix); strvec_clear(&options); } diff --git a/submodule.c b/submodule.c index 8bcef68a42480d..da4ace751fc2f7 100644 --- a/submodule.c +++ b/submodule.c @@ -1409,6 +1409,7 @@ struct submodule_parallel_fetch { int oid_fetch_tasks_nr, oid_fetch_tasks_alloc; struct strbuf submodules_with_errors; + int submodule_errors; }; #define SPF_INIT { \ .args = STRVEC_INIT, \ @@ -1565,7 +1566,8 @@ static struct fetch_task *fetch_task_create(struct submodule_parallel_fetch *spf static void record_fetch_error(struct submodule_parallel_fetch *spf, const char *name) { - spf->result = 1; + if (spf->submodule_errors == SUBMODULE_ERRORS_FAIL) + spf->result = 1; strbuf_addf(&spf->submodules_with_errors, "\t%s\n", name); } @@ -1851,7 +1853,8 @@ int fetch_submodules(struct repository *r, const struct strvec *options, const char *prefix, int command_line_option, int default_option, - int quiet, int max_parallel_jobs) + int quiet, int max_parallel_jobs, + int submodule_errors) { struct submodule_parallel_fetch spf = SPF_INIT; const struct run_process_parallel_opts opts = { @@ -1871,6 +1874,7 @@ int fetch_submodules(struct repository *r, spf.default_option = default_option; spf.quiet = quiet; spf.prefix = prefix; + spf.submodule_errors = submodule_errors; if (!r->worktree) goto out; diff --git a/submodule.h b/submodule.h index b10e16e6c063d2..c80b687d2a7c67 100644 --- a/submodule.h +++ b/submodule.h @@ -90,12 +90,17 @@ int should_update_submodules(void); */ const struct submodule *submodule_from_ce(const struct cache_entry *ce); void check_for_new_submodule_commits(struct object_id *oid); +/* Values for the submodule_errors parameter of fetch_submodules(). */ +#define SUBMODULE_ERRORS_FAIL 0 /* submodule fetch errors are fatal (default) */ +#define SUBMODULE_ERRORS_WARN 1 /* submodule fetch errors are non-fatal warnings */ + int fetch_submodules(struct repository *r, const struct strvec *options, const char *prefix, int command_line_option, int default_option, - int quiet, int max_parallel_jobs); + int quiet, int max_parallel_jobs, + int submodule_errors); unsigned is_submodule_modified(const char *path, int ignore_untracked); int submodule_uses_gitfile(const char *path); diff --git a/t/t5526-fetch-submodules.sh b/t/t5526-fetch-submodules.sh index 7ad274ce040afd..19d17440cf2220 100755 --- a/t/t5526-fetch-submodules.sh +++ b/t/t5526-fetch-submodules.sh @@ -1307,6 +1307,57 @@ test_expect_success 'setup for submodule fetch error tests' ' git config --global protocol.file.allow always ' +test_expect_success 'fetch --recurse-submodules fails when submodule commit is unreachable (default)' ' + test_when_finished "rm -fr env_default" && + create_err_env env_default && + push_unreachable_commit env_default && + test_must_fail git -C env_default/clone fetch --recurse-submodules 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success 'fetch.submoduleErrors=warn: unreachable submodule commit is non-fatal' ' + test_when_finished "rm -fr env_warn_cfg" && + create_err_env env_warn_cfg && + push_unreachable_commit env_warn_cfg && + git -C env_warn_cfg/clone -c fetch.submoduleErrors=warn \ + fetch --recurse-submodules 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success '--submodule-errors=warn: unreachable submodule commit is non-fatal' ' + test_when_finished "rm -fr env_warn_cli" && + create_err_env env_warn_cli && + push_unreachable_commit env_warn_cli && + git -C env_warn_cli/clone fetch --recurse-submodules \ + --submodule-errors=warn 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success '--submodule-errors=fail: unreachable submodule commit is fatal' ' + test_when_finished "rm -fr env_fail_cli" && + create_err_env env_fail_cli && + push_unreachable_commit env_fail_cli && + test_must_fail git -C env_fail_cli/clone fetch --recurse-submodules \ + --submodule-errors=fail 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success 'fetch.submoduleErrors=warn does not suppress successful fetch' ' + # A new reachable submodule commit (pushed to sub_bare) should be + # fetched without any error summary. + test_when_finished "rm -fr env_ok" && + create_err_env env_ok && + test_commit -C env_ok/sub_work reachable_ok && + git -C env_ok/sub_work push && + git -C env_ok/super_work submodule update --remote && + git -C env_ok/super_work add sub && + git -C env_ok/super_work commit -m "point sub to reachable commit" && + git -C env_ok/super_work push && + git -C env_ok/clone -c fetch.submoduleErrors=warn \ + fetch --recurse-submodules 2>err && + test_grep ! "Errors during submodule fetch" err +' + test_expect_success 'failed submodule fetch is fatal even when its commits are present locally' ' # Create the same commit (unreferenced, via commit-tree with fixed # dates) in both super_work/sub and clone/sub, point the gitlink at @@ -1334,4 +1385,42 @@ test_expect_success 'failed submodule fetch is fatal even when its commits are p test_grep "Errors during submodule fetch" err ' +test_expect_success '--submodule-errors=warn is honored by fetch --all' ' + # A second remote forces fetch_multiple(), which hands the submodule + # recursion off to per-remote child processes; the option must be + # forwarded to them. + test_when_finished "rm -fr env_all" && + create_err_env env_all && + push_unreachable_commit env_all && + git -C env_all/clone remote add second "$pwd/env_all/super_bare" && + git -C env_all/clone fetch --all --recurse-submodules \ + --submodule-errors=warn 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success '--submodule-errors=fail overrides warn config for fetch --all' ' + # The per-remote child processes re-read the repository config, so + # the command-line override must be forwarded to them explicitly. + test_when_finished "rm -fr env_override" && + create_err_env env_override && + push_unreachable_commit env_override && + git -C env_override/clone remote add second "$pwd/env_override/super_bare" && + git -C env_override/clone config fetch.submoduleErrors warn && + test_must_fail git -C env_override/clone fetch --all --recurse-submodules \ + --submodule-errors=fail 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success 'fetch.submoduleErrors=warn: inaccessible submodule is non-fatal' ' + test_when_finished "rm -fr env_access" && + create_err_env env_access && + rm env_access/clone/sub/.git && + rm -r env_access/clone/.git/modules/sub && + git -C env_access/clone -c fetch.submoduleErrors=warn \ + fetch --recurse-submodules 2>err && + test_grep "Could not access submodule" err && + test_must_fail git -C env_access/clone fetch --recurse-submodules 2>err && + test_grep "Could not access submodule" err +' + test_done From 179eccf0d01729c19a3238905b951b1880aa4ba1 Mon Sep 17 00:00:00 2001 From: Hugo Sales Date: Tue, 21 Jul 2026 15:04:42 +0100 Subject: [PATCH 020/259] rebase: add --[no-]edit to --continue Allow skipping the editor when continuing after resolving conflicts, via --no-edit or the rebase.noEdit configuration variable. The --edit option overrides rebase.noEdit when both are set. Signed-off-by: Hugo Sales Signed-off-by: Junio C Hamano --- Documentation/config/rebase.adoc | 6 ++++ Documentation/git-rebase.adoc | 17 +++++++++-- builtin/rebase.c | 29 ++++++++++++++++-- sequencer.c | 29 +++++++++++++++++- t/t3436-rebase-more-options.sh | 52 ++++++++++++++++++++++++++++++++ 5 files changed, 126 insertions(+), 7 deletions(-) diff --git a/Documentation/config/rebase.adoc b/Documentation/config/rebase.adoc index c6187ab28b2fb7..321ab8b529ca3f 100644 --- a/Documentation/config/rebase.adoc +++ b/Documentation/config/rebase.adoc @@ -62,6 +62,12 @@ instead of: + Defaults to false. +rebase.noEdit:: + When set to true, `git rebase --continue` uses the commit message + without launching $EDITOR, as if `--no-edit` were given. The + `--edit` option to `git rebase --continue` overrides this setting. + Defaults to false. + rebase.rescheduleFailedExec:: Automatically reschedule `exec` commands that failed. This only makes sense in interactive mode (or when an `--exec` option was provided). diff --git a/Documentation/git-rebase.adoc b/Documentation/git-rebase.adoc index f6c22d1598978a..cc0a69b5a599a3 100644 --- a/Documentation/git-rebase.adoc +++ b/Documentation/git-rebase.adoc @@ -181,6 +181,16 @@ including not with each other: --continue:: Restart the rebasing process after having resolved a merge conflict. ++ +-e:: +--edit:: +--no-edit:: + With `--continue`, edit or do not edit the commit message, + respectively. By default, the configured $EDITOR is opened so you + can update the commit message after resolving conflicts. + `--no-edit` reuses the existing message without launching an + editor. The `rebase.noEdit` configuration variable can be used to + enable `--no-edit` by default; `--edit` overrides that setting. --skip:: Restart the rebasing process by skipping the current patch. @@ -783,9 +793,10 @@ Commit Rewording When a conflict occurs while rebasing, rebase stops and asks the user to resolve. Since the user may need to make notable changes while resolving conflicts, after conflicts are resolved and the user has run -`git rebase --continue`, the rebase should open an editor and ask the -user to update the commit message. The 'merge' backend does this, while -the 'apply' backend blindly applies the original commit message. +`git rebase --continue`, the rebase opens an editor and asks the +user to update the commit message, unless `rebase.noEdit` is set or +`--no-edit` is passed to `--continue`. The 'merge' backend does this, +while the 'apply' backend blindly applies the original commit message. Miscellaneous differences ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/builtin/rebase.c b/builtin/rebase.c index fa4f5d9306b856..58c1d999c9ef25 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -43,7 +43,7 @@ static char const * const builtin_rebase_usage[] = { "[--onto | --keep-base] [ []]"), N_("git rebase [-i] [options] [--exec ] [--onto ] " "--root []"), - "git rebase --continue | --abort | --skip | --edit-todo", + "git rebase --continue [--[no-]edit] | --abort | --skip | --edit-todo", NULL }; @@ -135,6 +135,8 @@ struct rebase_options { int config_autosquash; int config_rebase_merges; int config_update_refs; + int config_no_edit; + int edit; }; #define REBASE_OPTIONS_INIT { \ @@ -156,6 +158,8 @@ struct rebase_options { .update_refs = -1, \ .config_update_refs = -1, \ .strategy_opts = STRING_LIST_INIT_NODUP,\ + .config_no_edit = -1, \ + .edit = -1, \ } static void rebase_options_release(struct rebase_options *opts) @@ -215,6 +219,13 @@ static struct replay_opts get_replay_opts(const struct rebase_options *opts) replay.have_squash_onto = 1; } + if (opts->action == ACTION_CONTINUE) { + if (opts->edit >= 0) + replay.edit = opts->edit; + else if (opts->config_no_edit > 0) + replay.edit = 0; + } + return replay; } @@ -839,6 +850,11 @@ static int rebase_config(const char *var, const char *value, return 0; } + if (!strcmp(var, "rebase.noedit")) { + opts->config_no_edit = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "rebase.forkpoint")) { opts->fork_point = git_config_bool(var, value) ? -1 : 0; return 0; @@ -1168,6 +1184,8 @@ int cmd_rebase(int argc, ACTION_CONTINUE), OPT_CMDMODE(0, "skip", &options.action, N_("skip current patch and continue"), ACTION_SKIP), + OPT_BOOL('e', "edit", &options.edit, + N_("edit the commit message")), OPT_CMDMODE(0, "abort", &options.action, N_("abort and check out the original branch"), ACTION_ABORT), @@ -1308,10 +1326,15 @@ int cmd_rebase(int argc, "which is no longer supported; use 'merges' instead")); if (options.action != ACTION_NONE && total_argc != 2) { - usage_with_options(builtin_rebase_usage, - builtin_rebase_options); + if (options.action != ACTION_CONTINUE || + options.edit < 0 || total_argc != 3) + usage_with_options(builtin_rebase_usage, + builtin_rebase_options); } + if (options.edit >= 0 && options.action != ACTION_CONTINUE) + die(_("--edit and --no-edit can only be used with --continue")); + if (argc > 2) usage_with_options(builtin_rebase_usage, builtin_rebase_options); diff --git a/sequencer.c b/sequencer.c index 57855b0066ac98..e724605e8431ad 100644 --- a/sequencer.c +++ b/sequencer.c @@ -2211,6 +2211,25 @@ static int should_edit(struct replay_opts *opts) { return opts->edit; } +static int should_edit_rebase_continue(struct replay_opts *opts) +{ + if (opts->edit < 0) + return 1; + return opts->edit; +} + +static void finalize_continue_edit_flags(struct replay_opts *opts, + unsigned int *flags) +{ + if (*flags & CLEANUP_MSG) + return; + + if (should_edit_rebase_continue(opts)) + *flags |= EDIT_MSG; + else + *flags &= ~EDIT_MSG; +} + static void refer_to_commit(struct repository *r, struct strbuf *msgbuf, const struct commit *commit, bool use_commit_reference) @@ -5261,7 +5280,7 @@ static int commit_staged_changes(struct repository *r, struct todo_list *todo_list) { struct replay_ctx *ctx = opts->ctx; - unsigned int flags = ALLOW_EMPTY | EDIT_MSG; + unsigned int flags = ALLOW_EMPTY; unsigned int final_fixup = 0, is_clean; struct strbuf rev = STRBUF_INIT; const char *reflog_action = reflog_message(opts, "continue", NULL); @@ -5426,6 +5445,8 @@ static int commit_staged_changes(struct repository *r, } } + finalize_continue_edit_flags(opts, &flags); + if (run_git_commit(final_fixup ? NULL : rebase_path_message(), reflog_action, opts, flags)) { ret = error(_("could not commit staged changes.")); @@ -5483,6 +5504,12 @@ int sequencer_continue(struct repository *r, struct replay_opts *opts) res = -1; goto release_todo_list; } + + /* + * Command-line --[no-]edit applies only to this + * --continue invocation, not to subsequent picks. + */ + opts->edit = -1; } else if (!file_exists(get_todo_path(opts))) return continue_single_pick(r, opts); else if ((res = read_populate_todo(r, &todo_list, opts))) diff --git a/t/t3436-rebase-more-options.sh b/t/t3436-rebase-more-options.sh index 94671d3c465046..c84c6717ab41f8 100755 --- a/t/t3436-rebase-more-options.sh +++ b/t/t3436-rebase-more-options.sh @@ -201,6 +201,58 @@ test_expect_success '--ignore-date is an alias for --reset-author-date' ' test_atime_is_ignored -2 ' +test_expect_success '--no-edit on continue uses existing commit message' ' + git checkout commit2 && + test_must_fail git rebase -m --onto commit2^^ commit2^ && + echo resolved >foo && + git add foo && + write_script fail-if-editor-invoked <<-\EOF && + echo editor invoked >&2 + exit 1 + EOF + GIT_EDITOR=./fail-if-editor-invoked git rebase --continue --no-edit && + git log --format=%s -1 >actual && + echo commit2 >expect && + test_cmp expect actual +' + +test_expect_success '--no-edit cannot be used when starting a rebase' ' + test_must_fail git rebase --no-edit -m main side 2>err && + test_grep "only be used with --continue" err +' + +test_expect_success 'rebase.noEdit skips editor on continue' ' + git config rebase.noEdit true && + git checkout commit2 && + test_must_fail git rebase -m --onto commit2^^ commit2^ && + echo resolved >foo && + git add foo && + write_script fail-if-editor-invoked <<-\EOF && + echo editor invoked >&2 + exit 1 + EOF + GIT_EDITOR=./fail-if-editor-invoked git rebase --continue && + git log --format=%s -1 >actual && + echo commit2 >expect && + test_cmp expect actual +' + +test_expect_success '--edit on continue overrides rebase.noEdit' ' + git config rebase.noEdit true && + git checkout commit2 && + test_must_fail git rebase -m --onto commit2^^ commit2^ && + echo resolved >foo && + git add foo && + ( + set_fake_editor && + FAKE_COMMIT_MESSAGE="edited on continue" \ + git rebase --continue --edit + ) && + test_write_lines "edited on continue" "" >expect && + git log --format=%B -1 >actual && + test_cmp expect actual +' + # This must be the last test in this file test_expect_success '$EDITOR and friends are unchanged' ' test_editor_unchanged From 9c82ba86a0c5e630d5defbe8118612fd5c5e4612 Mon Sep 17 00:00:00 2001 From: Son Luong Ngoc Date: Wed, 22 Jul 2026 08:15:05 +0000 Subject: [PATCH 021/259] rebase: skip branch symref aliases git rebase --update-refs can finish rewriting the current branch and then fail while updating a local branch that is a symbolic ref. This can happen during a default-branch rename where refs/heads/main points at refs/heads/master while users migrate. The problem is a partially applied ref update: the main rebase has already succeeded when the later ref update fails. The sequencer queues updates from local branch decorations. Commit 106b6885c7 (rebase: ignore non-branch update-refs) filters out decorations such as HEAD and tags. A branch symref is still a local branch decoration, but refs_update_ref() dereferences it, so an alias to another branch duplicates the concrete branch update. Resolve local branch decorations before queuing them. Skip symrefs whose targets are under refs/heads/ so that only the concrete branch update is queued. Keep an owned copy of the resolved HEAD and skip the current branch before checked-out handling so later ref resolution cannot change the comparison. This prevents a successful rebase from being followed by a failed, partially applied ref update while preserving each alias as a symref. Signed-off-by: Son Luong Ngoc Signed-off-by: Junio C Hamano --- sequencer.c | 44 +++++++++++++++++++++++++---------- t/t3400-rebase.sh | 2 +- t/t3404-rebase-interactive.sh | 16 +++++++++++++ 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/sequencer.c b/sequencer.c index 1ee4b2875b25a0..17f5baab62ff70 100644 --- a/sequencer.c +++ b/sequencer.c @@ -6445,32 +6445,50 @@ static int add_decorations_to_list(const struct commit *commit, struct todo_add_branch_context *ctx) { const struct name_decoration *decoration = get_name_decoration(&commit->object); - const char *head_ref = refs_resolve_ref_unsafe(get_main_ref_store(the_repository), - "HEAD", - RESOLVE_REF_READING, - NULL, - NULL); + struct ref_store *refs = get_main_ref_store(the_repository); + char *head_ref = refs_resolve_refdup(refs, "HEAD", + RESOLVE_REF_READING, + NULL, NULL); while (decoration) { struct todo_item *item; const char *path; + char *resolved_ref; + int flags = 0; size_t base_offset = ctx->buf->len; /* - * If the branch is the current HEAD, then it will be - * updated by the default rebase behavior. - * Exclude it from the list of refs to update, - * as well as any non-branch decorations. * Non-branch decorations may be present if the pretty format * includes "%d", which would have loaded all refs * into the global decoration table. */ - if ((head_ref && !strcmp(head_ref, decoration->name)) || - (decoration->type != DECORATION_REF_LOCAL)) { + if (decoration->type != DECORATION_REF_LOCAL) { + decoration = decoration->next; + continue; + } + + resolved_ref = refs_resolve_refdup(refs, decoration->name, + RESOLVE_REF_READING, + NULL, &flags); + if (resolved_ref && (flags & REF_ISSYMREF) && + starts_with(resolved_ref, "refs/heads/")) { + free(resolved_ref); + decoration = decoration->next; + continue; + } + + /* + * If the branch is the current HEAD, then it will be + * updated by the default rebase behavior. + */ + if (head_ref && !strcmp(head_ref, decoration->name)) { + free(resolved_ref); decoration = decoration->next; continue; } + path = branch_checked_out(decoration->name); + ALLOC_GROW(ctx->items, ctx->items_nr + 1, ctx->items_alloc); @@ -6478,7 +6496,7 @@ static int add_decorations_to_list(const struct commit *commit, memset(item, 0, sizeof(*item)); /* If the branch is checked out, then leave a comment instead. */ - if ((path = branch_checked_out(decoration->name))) { + if (path) { item->command = TODO_COMMENT; strbuf_commented_addf(ctx->buf, comment_line_str, "Ref %s checked out at '%s'\n", @@ -6498,9 +6516,11 @@ static int add_decorations_to_list(const struct commit *commit, item->arg_len = ctx->buf->len - base_offset; ctx->items_nr++; + free(resolved_ref); decoration = decoration->next; } + free(head_ref); return 0; } diff --git a/t/t3400-rebase.sh b/t/t3400-rebase.sh index c0c00fbb7b1e4e..be120bdcd1a41d 100755 --- a/t/t3400-rebase.sh +++ b/t/t3400-rebase.sh @@ -471,7 +471,7 @@ test_expect_success 'git rebase --update-ref with core.commentChar and branch on GIT_SEQUENCE_EDITOR="cat >actual" git -c core.commentChar=% \ rebase -i --update-refs base && test_grep "% Ref refs/heads/wt-topic checked out at" actual && - test_grep "% Ref refs/heads/topic2 checked out at" actual + test_grep ! "% Ref refs/heads/topic2 checked out at" actual ' test_done diff --git a/t/t3404-rebase-interactive.sh b/t/t3404-rebase-interactive.sh index 58b3bb0c271aae..b05d93884668b5 100755 --- a/t/t3404-rebase-interactive.sh +++ b/t/t3404-rebase-interactive.sh @@ -1975,15 +1975,23 @@ test_expect_success '--update-refs ignores non-branch decorations' ' ) && grep ^update-ref todo >actual && test_write_lines "update-ref refs/heads/no-conflict-branch" >expect && + test_grep ! "^# Ref refs/heads/update-refs checked out" todo && test_cmp expect actual ' test_expect_success '--update-refs updates refs correctly' ' + test_when_finished " + test_might_fail git symbolic-ref -d refs/heads/no-conflict-branch-alias && + test_might_fail git symbolic-ref -d refs/heads/second-alias + " && git checkout -B update-refs no-conflict-branch && git branch -f base HEAD~4 && git branch -f first HEAD~3 && git branch -f second HEAD~3 && git branch -f third HEAD~1 && + git symbolic-ref refs/heads/no-conflict-branch-alias \ + refs/heads/no-conflict-branch && + git symbolic-ref refs/heads/second-alias refs/heads/second && test_commit extra2 fileX && git commit --amend --fixup=L && @@ -1991,8 +1999,16 @@ test_expect_success '--update-refs updates refs correctly' ' test_cmp_rev HEAD~3 refs/heads/first && test_cmp_rev HEAD~3 refs/heads/second && + test_cmp_rev HEAD~3 refs/heads/second-alias && test_cmp_rev HEAD~1 refs/heads/third && test_cmp_rev HEAD refs/heads/no-conflict-branch && + test_cmp_rev HEAD refs/heads/no-conflict-branch-alias && + test_write_lines refs/heads/no-conflict-branch >expect && + git symbolic-ref refs/heads/no-conflict-branch-alias >actual && + test_cmp expect actual && + test_write_lines refs/heads/second >expect && + git symbolic-ref refs/heads/second-alias >actual && + test_cmp expect actual && q_to_tab >expect <<-\EOF && Successfully rebased and updated refs/heads/update-refs. From 056eed4ba7c60dfbb220e2ecc49313c919a22221 Mon Sep 17 00:00:00 2001 From: Son Luong Ngoc Date: Wed, 22 Jul 2026 08:15:06 +0000 Subject: [PATCH 022/259] rebase: guard non-branch symref targets A local branch symbolic ref may point outside refs/heads/. Such an alias cannot be skipped like a branch-to-branch alias because its concrete target ref is absent from the local branch decoration list. However, queuing each alias independently can update the same target ref more than once and make the second compare-and-swap fail. A reservation from another worktree can also name either an alias or its resolved target ref, so checking only one form can miss an in-progress update. Fix these cases by checking both the literal alias and its resolved target ref against checked-out reservations. Deduplicate updates by target ref. Also reserve both forms when loading another worktree's update-refs state. This makes different aliases honor the same in-progress update. This keeps non-branch symrefs supported without allowing duplicate or cross-worktree ref updates. Signed-off-by: Son Luong Ngoc Signed-off-by: Junio C Hamano --- branch.c | 15 ++++++++ sequencer.c | 19 +++++++++ t/t3404-rebase-interactive.sh | 72 +++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+) diff --git a/branch.c b/branch.c index 243db7d0fc0226..98a50d83680201 100644 --- a/branch.c +++ b/branch.c @@ -442,10 +442,25 @@ static void prepare_checked_out_branches(void) &update_refs)) { struct string_list_item *item; for_each_string_list_item(item, &update_refs) { + char *resolved_ref; + int flags = 0; + old = strmap_put(¤t_checked_out_branches, item->string, xstrdup(wt->path)); free(old); + + resolved_ref = refs_resolve_refdup( + get_main_ref_store(the_repository), + item->string, RESOLVE_REF_READING, + NULL, &flags); + if (resolved_ref && (flags & REF_ISSYMREF)) { + old = strmap_put( + ¤t_checked_out_branches, + resolved_ref, xstrdup(wt->path)); + free(old); + } + free(resolved_ref); } string_list_clear(&update_refs, 1); } diff --git a/sequencer.c b/sequencer.c index 17f5baab62ff70..73c7d7f0c7860c 100644 --- a/sequencer.c +++ b/sequencer.c @@ -6439,6 +6439,7 @@ struct todo_add_branch_context { size_t items_alloc; struct strbuf *buf; struct string_list refs_to_oids; + struct string_list symref_update_targets; }; static int add_decorations_to_list(const struct commit *commit, @@ -6453,6 +6454,7 @@ static int add_decorations_to_list(const struct commit *commit, while (decoration) { struct todo_item *item; const char *path; + const char *checked_ref; char *resolved_ref; int flags = 0; size_t base_offset = ctx->buf->len; @@ -6488,6 +6490,17 @@ static int add_decorations_to_list(const struct commit *commit, } path = branch_checked_out(decoration->name); + if (!path && resolved_ref && (flags & REF_ISSYMREF)) { + checked_ref = resolved_ref; + path = branch_checked_out(checked_ref); + } + if (!path && resolved_ref && (flags & REF_ISSYMREF) && + string_list_has_string(&ctx->symref_update_targets, + resolved_ref)) { + free(resolved_ref); + decoration = decoration->next; + continue; + } ALLOC_GROW(ctx->items, ctx->items_nr + 1, @@ -6503,6 +6516,10 @@ static int add_decorations_to_list(const struct commit *commit, decoration->name, path); } else { struct string_list_item *sti; + + if (resolved_ref && (flags & REF_ISSYMREF)) + string_list_insert(&ctx->symref_update_targets, + resolved_ref); item->command = TODO_UPDATE_REF; strbuf_addf(ctx->buf, "%s\n", decoration->name); @@ -6534,6 +6551,7 @@ static int todo_list_add_update_ref_commands(struct todo_list *todo_list) struct todo_add_branch_context ctx = { .buf = &todo_list->buf, .refs_to_oids = STRING_LIST_INIT_DUP, + .symref_update_targets = STRING_LIST_INIT_DUP, }; ctx.items_alloc = 2 * todo_list->nr + 1; @@ -6559,6 +6577,7 @@ static int todo_list_add_update_ref_commands(struct todo_list *todo_list) res = write_update_refs_state(&ctx.refs_to_oids); string_list_clear(&ctx.refs_to_oids, 1); + string_list_clear(&ctx.symref_update_targets, 0); if (res) { /* we failed, so clean up the new list. */ diff --git a/t/t3404-rebase-interactive.sh b/t/t3404-rebase-interactive.sh index b05d93884668b5..54981c909bb9b6 100755 --- a/t/t3404-rebase-interactive.sh +++ b/t/t3404-rebase-interactive.sh @@ -2024,6 +2024,78 @@ test_expect_success '--update-refs updates refs correctly' ' test_cmp expect err.trimmed ' +test_expect_success '--update-refs checks resolved non-branch symref target' ' + test_when_finished " + git worktree remove --force checked-out-target-wt && + git symbolic-ref -d refs/heads/non-branch-alias && + git tag -d checked-out-target + " && + git tag checked-out-target HEAD~1 && + git symbolic-ref refs/heads/non-branch-alias refs/tags/checked-out-target && + git worktree add --detach checked-out-target-wt checked-out-target && + git -C checked-out-target-wt symbolic-ref HEAD refs/tags/checked-out-target && + + GIT_SEQUENCE_EDITOR="cat >todo" git rebase -i --update-refs HEAD~2 && + + test_grep "^# Ref refs/heads/non-branch-alias checked out at" todo && + test_write_lines refs/tags/checked-out-target >expect && + git symbolic-ref refs/heads/non-branch-alias >actual && + test_cmp expect actual +' + +test_expect_success '--update-refs deduplicates non-branch symref targets' ' + test_when_finished " + git symbolic-ref -d refs/heads/non-branch-alias-one && + git symbolic-ref -d refs/heads/non-branch-alias-two && + git tag -d shared-non-branch-target + " && + git tag shared-non-branch-target HEAD~1 && + git symbolic-ref refs/heads/non-branch-alias-one \ + refs/tags/shared-non-branch-target && + git symbolic-ref refs/heads/non-branch-alias-two \ + refs/tags/shared-non-branch-target && + + GIT_SEQUENCE_EDITOR=: git rebase -i --force-rebase --update-refs HEAD~2 && + + test_cmp_rev HEAD~1 refs/heads/non-branch-alias-one && + test_cmp_rev HEAD~1 refs/heads/non-branch-alias-two && + test_write_lines refs/tags/shared-non-branch-target >expect && + git symbolic-ref refs/heads/non-branch-alias-one >actual && + test_cmp expect actual && + git symbolic-ref refs/heads/non-branch-alias-two >actual && + test_cmp expect actual +' + +test_expect_success '--update-refs honors non-branch symref reservations' ' + test_when_finished " + test_might_fail git worktree remove --force reserved-target-wt && + test_might_fail git symbolic-ref -d \ + refs/heads/reserved-non-branch-alias-one && + test_might_fail git symbolic-ref -d \ + refs/heads/reserved-non-branch-alias-two && + test_might_fail git tag -d reserved-non-branch-target + " && + git tag reserved-non-branch-target HEAD~1 && + git symbolic-ref refs/heads/reserved-non-branch-alias-one \ + refs/tags/reserved-non-branch-target && + git symbolic-ref refs/heads/reserved-non-branch-alias-two \ + refs/tags/reserved-non-branch-target && + git worktree add --detach reserved-target-wt HEAD && + wt_gitdir=$(git -C reserved-target-wt rev-parse --absolute-git-dir) && + mkdir -p "$wt_gitdir/rebase-merge" && + old_oid=$(git rev-parse refs/heads/reserved-non-branch-alias-one) && + test_write_lines refs/heads/reserved-non-branch-alias-one \ + "$old_oid" "$old_oid" >"$wt_gitdir/rebase-merge/update-refs" && + + GIT_SEQUENCE_EDITOR="cat >todo" git rebase -i --update-refs HEAD~2 && + + test_grep "^# Ref refs/heads/reserved-non-branch-alias-one checked out at" \ + todo && + test_grep "^# Ref refs/heads/reserved-non-branch-alias-two checked out at" \ + todo && + test_grep ! "^update-ref refs/heads/reserved-non-branch-alias" todo +' + test_expect_success 'respect user edits to update-ref steps' ' git checkout -B update-refs-break no-conflict-branch && git branch -f base HEAD~4 && From 82114627614c22b05c7c0919ad59ac8f225fe7d5 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 16:05:50 -0500 Subject: [PATCH 023/259] t5308: test reverse indexes with duplicate objects A non-strict .idx has one entry for each object in the pack, even when multiple entries have the same object ID. Thus a pack ordered A, B, A, C has two .idx entries for A at distinct offsets. The corresponding per-pack reverse index must represent both entries and map them back to physical pack order. Existing reverse-index tests do not cover packs with duplicate objects. Add one and check that %(objectsize:disk) for B stops at the second A, rather than extending through it to C. Exercise both the on-disk and in-memory reverse-index implementations. As part of validating Git's handling of packs containing duplicate objects, cover their per-pack reverse indexes. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- t/t5308-pack-detect-duplicates.sh | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/t/t5308-pack-detect-duplicates.sh b/t/t5308-pack-detect-duplicates.sh index 0f841378677165..4ff8f5b44953f4 100755 --- a/t/t5308-pack-detect-duplicates.sh +++ b/t/t5308-pack-detect-duplicates.sh @@ -27,6 +27,11 @@ HI_SHA1=$EMPTY_BLOB # duplicate runs). MISSING_SHA1=$(test_oid missing_oid) +# Three distinct objects for tests where physical pack order matters. +A=$(test_oid packlib_7_0) +B=$LO_SHA1 +C=$HI_SHA1 + # git will never intentionally create packfiles with # duplicate objects, so we have to construct them by hand. # @@ -72,6 +77,40 @@ test_expect_success 'lookup in duplicated pack' ' test_cmp expect actual ' +test_expect_success 'duplicate entries remain in pack reverse index' ' + clear_packs && + { + pack_header 4 && + pack_obj $A && + pack_obj $B && + pack_obj $A && + pack_obj $C + } >physical-order.pack && + pack_trailer physical-order.pack && + + test_must_fail git index-pack --rev-index --stdin --strict \ + err && + test_grep "appears twice in the pack" err && + + git index-pack --rev-index --stdin offsets.raw && + + sort -n offsets.raw | grep -A1 "$B" | cut -d" " -f1 >adjacent && + echo $(($(tail -n1 adjacent) - $(head -n1 adjacent))) >expect && + echo "$B" >in && + + GIT_TEST_REV_INDEX_DIE_IN_MEMORY=1 \ + git cat-file --batch-check="%(objectsize:disk)" \ + actual.disk && + GIT_TEST_REV_INDEX_DIE_ON_DISK=1 \ + git -c pack.readReverseIndex=false \ + cat-file --batch-check="%(objectsize:disk)" \ + actual.mem && + + test_cmp expect actual.disk && + test_cmp expect actual.mem +' + test_expect_success 'index-pack can reject packs with duplicates' ' clear_packs && create_pack dups.pack 2 && From f4037afe80c7957ad6a071bfb0cd206eccfc7126 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 16:06:00 -0500 Subject: [PATCH 024/259] packfile: recover delta cycles through duplicate entries 98f8854c94 (index-pack: allow revisiting REF_DELTA chains, 2025-04-28) changed t5309's recoverable-cycle case to expect index-pack to accept the pack, but did not read the resulting objects. The .idx for a pack with duplicate OIDs retains every physical entry, with equal OIDs in a contiguous run. Ordinary REF_DELTA lookup selects one representation from such a run. If that choice closes a cycle, both type and content readers follow the same physical offsets indefinitely even though another representation is usable. Keep the ordinary walk, and recognize a cycle only when its existing stack shows an exact repeated pack offset. At that point, restart at the requested object and search duplicate representations depth-first. Treat each OID as a node, try each entry in its .idx run, and translate OFS_DELTA bases back to OIDs. A bitmap marks each OID group already visited. Reaching a full object records the exact offsets along the acyclic path so unpack_entry() can replay it; packed_to_object_type() needs only the resulting type. Thus, acyclic lookups continue to use the existing single-entry lookup. Duplicate-run scans, the visited bitmap, and OFS-to-OID translation remain confined to recovery after a proven cycle. Extend t5309 to read both type and content after indexing. Cover a root-level full duplicate, a mixed REF/OFS cycle, and a tail into a three-object cycle which must backtrack before taking an alternate REF_DELTA/OFS_DELTA path to a full base. Keep the fixtures hash-independent so the same cases run under SHA-1 and SHA-256. This adds the reader validation missing from the earlier acceptance test. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- packfile.c | 199 +++++++++++++++++++++++++++++++++++ t/t5309-pack-delta-cycles.sh | 148 ++++++++++++++++++++++++-- 2 files changed, 341 insertions(+), 6 deletions(-) diff --git a/packfile.c b/packfile.c index 0eee45055f833e..6049d1fd8c6881 100644 --- a/packfile.c +++ b/packfile.c @@ -10,6 +10,7 @@ #include "dir.h" #include "packfile.h" #include "delta.h" +#include "ewah/ewok.h" #include "hash-lookup.h" #include "commit.h" #include "object.h" @@ -1077,6 +1078,160 @@ static int get_delta_base_oid(struct packed_git *p, return -1; } +/* + * Search duplicate representations for a chain ending in a full object. + * Representations of the same OID are interchangeable as delta bases. + * + * Each path entry is the search frame for one OID. It walks that OID's + * contiguous .idx entries and retains the chosen entry's .pack offset + * for replay. + */ +struct delta_path_entry { + off_t selected_offset; /* zero until a candidate is selected */ + uint32_t next; /* index position of the next candidate */ + uint32_t remaining; /* total number of candidates remaining */ +}; + +struct delta_path { + struct delta_path_entry *entries; + size_t nr, alloc; +}; + +static int push_oid_group(struct packed_git *p, + const struct object_id *oid, + struct bitmap *visited, struct delta_path *path) +{ + struct object_id candidate; + struct delta_path_entry *entry; + uint32_t first_index_pos, group_index_pos, index_pos; + + /* + * Determine the range of index positions referring to duplicate + * copies of the given object. + * + * Any position within that range is OK, since we will determine + * the exact range below. + */ + if (!bsearch_pack(oid, p, &group_index_pos)) + return 0; + if (bitmap_get(visited, group_index_pos)) + return 0; + + first_index_pos = group_index_pos; + while (first_index_pos > 0) { + if (nth_packed_object_id(&candidate, p, first_index_pos - 1) < 0) + return -1; + if (!oideq(&candidate, oid)) + break; + first_index_pos--; + } + + for (index_pos = first_index_pos; index_pos < p->num_objects; index_pos++) { + if (nth_packed_object_id(&candidate, p, index_pos) < 0) + return -1; + if (!oideq(&candidate, oid)) + break; + } + + bitmap_set(visited, group_index_pos); + + ALLOC_GROW(path->entries, path->nr + 1, path->alloc); + entry = &path->entries[path->nr++]; + entry->selected_offset = 0; + entry->next = first_index_pos; + entry->remaining = index_pos - first_index_pos; + + return 1; +} + +static enum object_type find_delta_path(struct packed_git *p, + struct pack_window **w_curs, + off_t offset, + struct delta_path *path) +{ + struct object_id oid; + uint32_t pack_pos; + struct bitmap *visited; + enum object_type result = OBJ_BAD; + + if (offset_to_pack_pos(p, offset, &pack_pos) < 0) + return OBJ_BAD; + if (nth_packed_object_id(&oid, p, pack_pos_to_index(p, pack_pos)) < 0) + return OBJ_BAD; + + visited = bitmap_new(); + if (push_oid_group(p, &oid, visited, path) != 1) + goto done; + + /* + * Search depth-first for a chain ending in a full object. Each frame + * tries every representation of one OID; a delta pushes its base OID, + * while exhausting a frame backtracks to its parent. + */ + while (path->nr) { + struct delta_path_entry *entry = &path->entries[path->nr - 1]; + enum object_type candidate_type; + off_t curpos; + size_t size; + + /* + * This OID has no path to a full object. Let its parent try + * another representation; exhausting the root fails the search. + */ + if (!entry->remaining) { + path->nr--; + continue; + } + + entry->selected_offset = + nth_packed_object_offset(p, entry->next++); + entry->remaining--; + curpos = entry->selected_offset; + candidate_type = unpack_object_header(p, w_curs, &curpos, &size); + + /* + * A full object terminates the chain, and its type is + * inherited by every delta above it. A delta continues + * at its base; any other type rejects only this + * representation. + */ + switch (candidate_type) { + case OBJ_COMMIT: + case OBJ_TREE: + case OBJ_BLOB: + case OBJ_TAG: + result = candidate_type; + goto done; + case OBJ_OFS_DELTA: + case OBJ_REF_DELTA: + break; + default: + /* + * A bad or unknown type rejects only this copy; + * another representation of the same OID may + * still work. + */ + continue; + } + + /* + * Descend to this delta's base. A malformed reference + * or a missing or already-visited base rejects this + * copy. A newly pushed base is examined next; an index + * error aborts the search. + */ + if (get_delta_base_oid(p, w_curs, curpos, &oid, candidate_type, + entry->selected_offset)) + continue; + if (push_oid_group(p, &oid, visited, path) < 0) + goto done; + } + +done: + bitmap_free(visited); + return result; +} + static int retry_bad_packed_offset(struct repository *r, struct packed_git *p, off_t obj_offset) @@ -1105,11 +1260,27 @@ static enum object_type packed_to_object_type(struct repository *r, { off_t small_poi_stack[POI_STACK_PREALLOC]; off_t *poi_stack = small_poi_stack; + off_t root_offset = obj_offset; int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC; while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { off_t base_offset; size_t size; + + if (poi_stack_nr > 0 && poi_stack_nr % 2 == 0 && + obj_offset == poi_stack[poi_stack_nr / 2]) { + struct delta_path path = { 0 }; + /* + * Normal lookup returned to the same pack + * entry. Restart from the requested object + * using alternate representations. + */ + type = find_delta_path(p, w_curs, root_offset, &path); + free(path.entries); + if (type == OBJ_BAD) + goto unwind; + break; + } /* Push the object we're going to leave behind */ if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) { poi_stack_alloc = alloc_nr(poi_stack_nr); @@ -1525,9 +1696,11 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, { struct pack_window *w_curs = NULL; off_t curpos = obj_offset; + off_t root_offset = obj_offset; void *data = NULL; size_t size; enum object_type type; + struct delta_path path = { 0 }; struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC]; struct unpack_entry_stack_ent *delta_stack = small_delta_stack; int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC; @@ -1553,6 +1726,22 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, break; } + if (!path.nr && + delta_stack_nr > 0 && delta_stack_nr % 2 == 0 && + obj_offset == delta_stack[delta_stack_nr / 2].obj_offset) { + /* + * Normal lookup returned to the same pack + * entry. Find an acyclic path if one exists, + * discard this walk, and replay that path. + */ + if (find_delta_path(p, &w_curs, root_offset, + &path) == OBJ_BAD) + break; + delta_stack_nr = 0; + curpos = obj_offset = path.entries[0].selected_offset; + continue; + } + if (do_check_packed_object_crc && p->index_version > 1) { uint32_t pack_pos, index_pos; off_t len; @@ -1591,6 +1780,15 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, break; } + /* Use the base chosen by recovery, not the one from normal lookup. */ + if (path.nr) { + size_t path_pos = (size_t)delta_stack_nr + 1; + + if (path_pos >= path.nr) + BUG("alternate delta path ends in a delta"); + base_offset = path.entries[path_pos].selected_offset; + } + /* push object, proceed to base */ if (delta_stack_nr >= delta_stack_alloc && delta_stack == small_delta_stack) { @@ -1731,6 +1929,7 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, out: unuse_pack(&w_curs); + free(path.entries); if (delta_stack != small_delta_stack) free(delta_stack); diff --git a/t/t5309-pack-delta-cycles.sh b/t/t5309-pack-delta-cycles.sh index 6b03675d91b5e1..f613950e38f262 100755 --- a/t/t5309-pack-delta-cycles.sh +++ b/t/t5309-pack-delta-cycles.sh @@ -9,6 +9,83 @@ test_description='test index-pack handling of delta cycles in packfiles' A=$(test_oid packlib_7_0) B=$(test_oid packlib_7_76) +# Copy the entries from a complete pack without its header or trailer. +pack_entries () { + entry_size=$(wc -c <"$1") && + dd if="$1" bs=1 skip=12 \ + count=$((entry_size - 12 - $(test_oid rawsz))) 2>/dev/null +} + +# B as an OFS_DELTA against A at the given one-byte distance. +pack_obj_b_ofs_a () { + pack_obj "$B" "$A" >b-ref.tmp && + printf "\145" && + printf "\\$(printf "%03o" "$1")" && + dd if=b-ref.tmp bs=1 skip=$((1 + $(test_oid rawsz))) 2>/dev/null +} + +# Return the base of the first one-byte-header REF_DELTA for the given OID. +first_ref_base () { + idx=$(echo .git/objects/pack/*.idx) && + offset=$(git show-index <"$idx" | + awk -v oid="$1" '$2 == oid { print $1; exit }') && + dd if="${idx%.idx}.pack" bs=1 skip=$((offset + 1)) \ + count=$(test_oid rawsz) 2>/dev/null | + test-tool hexdump | + tr -d " \n" +} + +# The order of equal-OID entries in the .idx is unspecified. Retain a pack +# which selects $1 as a delta against $2. Unless $3 is "-", also require the +# first copy searched during recovery to be a REF_DELTA against $3. +install_cycle () { + cycle_oid=$1 && + cycle_base=$2 && + first_base=$3 && + shift 3 && + for pack + do + clear_packs && + git index-pack --fix-thin --stdin <"$pack" && + selected_base=$(echo "$cycle_oid" | + git cat-file --batch-check="%(deltabase)") || + return 1 + if test "$selected_base" = "$cycle_base" && + { test "$first_base" = "-" || + test "$(first_ref_base "$cycle_oid")" = "$first_base"; } + then + return 0 + fi + done + return 1 +} + +make_cycle_pack () { + cycle_pack=$1 && + shift && + test-tool -C alt-source pack-deltas --num-objects=6 >refs.tmp <<-EOF && + REF_DELTA $T $X + REF_DELTA $X $1 + REF_DELTA $Y $Z + REF_DELTA $Z $X + REF_DELTA $X $2 + REF_DELTA $X $3 + EOF + { + pack_header 8 && + pack_entries refs.tmp && + cat a-full && + pack_obj_b_ofs_a "$a_full_size" + } >"$cycle_pack" && + pack_trailer "$cycle_pack" +} + +check_blob () { + test "$(git cat-file -t "$1")" = blob && + git cat-file blob "$1" >actual && + test_cmp_bin "$2" actual +} + # double-check our hand-constucted packs test_expect_success 'index-pack works with a single delta (A->B)' ' clear_packs && @@ -67,18 +144,77 @@ test_expect_success 'failover to an object in another pack' ' ' test_expect_success 'failover to a duplicate object in the same pack' ' - clear_packs && + { + pack_header 3 && + pack_obj $A && + pack_obj $B $A && + pack_obj $A $B + } >recoverable-1.pack && + pack_trailer recoverable-1.pack && { pack_header 3 && pack_obj $A $B && pack_obj $B $A && pack_obj $A - } >recoverable.pack && - pack_trailer recoverable.pack && + } >recoverable-2.pack && + pack_trailer recoverable-2.pack && + + # The selected copy of A is part of the cycle, but the full copy + # lets both type and content lookups resolve it. + install_cycle "$A" "$B" - recoverable-1.pack recoverable-2.pack && + printf "\7\0" >expect && + check_blob "$A" expect +' + +test_expect_success 'failover from a mixed REF/OFS cycle' ' + pack_obj "$A" "$B" >a-ref && + pack_obj "$B" >b-full && + a_ref_size=$(wc -c mixed-1.pack && + pack_trailer mixed-1.pack && + { + pack_header 3 && + cat a-ref && + pack_obj_b_ofs_a "$a_ref_size" && + cat b-full + } >mixed-2.pack && + pack_trailer mixed-2.pack && + + # The REF_DELTA for A selects the OFS_DELTA copy of B; the + # full B is its escape. + install_cycle "$B" "$A" - mixed-1.pack mixed-2.pack && + printf "\7\0" >expect && + check_blob "$A" expect +' - # This cycle does not fail since the existence of a full copy - # of A in the pack allows us to resolve the cycle. - git index-pack --fix-thin --stdin /dev/null && + X=$(printf x | git -C alt-source hash-object -w --stdin) && + Y=$(printf y | git -C alt-source hash-object -w --stdin) && + Z=$(printf z | git -C alt-source hash-object -w --stdin) && + printf "tail T\n" >tail && + T=$(git -C alt-source hash-object -w --stdin a-full && + a_full_size=$(wc -c X->Y->Z->X. Recovery must exhaust that + # branch, then use X->B->A, whose final edge is an OFS_DELTA. + install_cycle "$X" "$Y" "$Y" \ + alternate-1.pack alternate-2.pack alternate-3.pack && + check_blob "$T" tail ' test_expect_success 'index-pack works with thin pack A->B->C with B on disk' ' From e3ffc236a8c7780d22442c7ebf1fca757bdccdf4 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 16:06:09 -0500 Subject: [PATCH 025/259] midx: verify duplicate pack entries by OID and offset A MIDX retains one entry per OID, while a non-strict pack index can contain the same OID at several offsets. verify_midx_file() compares the recorded offset with find_pack_entry_one(), which may return a different member of that duplicate run and falsely report a valid MIDX as corrupt. Check instead that the exact OID/offset pair exists anywhere in the contiguous duplicate run in the source pack. That is the relevant invariant: same-pack duplicates have no canonical representation, and every matching physical copy is valid, including those recorded by existing MIDXs. This matches reader behavior. The OOFF chunk records the selected (pack, offset), and midx_to_pack_pos() reconstructs its RIDX position from that pair. If midx_pair_to_pack_pos() is given an unselected duplicate offset, the lookup misses and its sole caller falls back from partial reuse to normal packing. Readers therefore remain consistent with whichever representation the MIDX records. Write a MIDX over the existing duplicate-pack fixture, assert that its selected offset differs from find_pack_entry_one(), and verify it. Then run fsck as an application-level check. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- midx.c | 59 +++++++++++++++++++++++++++---- t/helper/test-find-pack.c | 18 +++++++--- t/t5308-pack-detect-duplicates.sh | 14 ++++++++ 3 files changed, 80 insertions(+), 11 deletions(-) diff --git a/midx.c b/midx.c index 76c3f92cc374b8..05fe99f8cad4bd 100644 --- a/midx.c +++ b/midx.c @@ -906,6 +906,48 @@ static int compare_pair_pos_vs_id(const void *_a, const void *_b) return b->pack_int_id - a->pack_int_id; } +/* + * Return whether the pack index contains an entry with both "oid" and + * "offset". A pack index may contain duplicate OIDs, so an arbitrary + * OID lookup is not enough to validate a particular offset. + * + * Do not use offset_to_pack_pos() here: it may consult an optional '.rev' + * file, which is verified separately, or build an in-memory reverse index + * that remains attached to the pack. Verify the MIDX directly against its + * source pack index instead. + */ +static int pack_index_has_oid_at_offset(struct packed_git *p, + const struct object_id *oid, + off_t offset) +{ + struct object_id candidate; + uint32_t pos, i; + + if (!bsearch_pack(oid, p, &pos)) + return 0; + + if (nth_packed_object_offset(p, pos) == offset) + return 1; + + for (i = pos; i > 0; i--) { + if (nth_packed_object_id(&candidate, p, i - 1) || + !oideq(&candidate, oid)) + break; + if (nth_packed_object_offset(p, i - 1) == offset) + return 1; + } + + for (i = pos + 1; i < p->num_objects; i++) { + if (nth_packed_object_id(&candidate, p, i) || + !oideq(&candidate, oid)) + break; + if (nth_packed_object_offset(p, i) == offset) + return 1; + } + + return 0; +} + /* * Limit calls to display_progress() for performance reasons. * The interval here was arbitrarily chosen. @@ -1015,7 +1057,6 @@ int verify_midx_file(struct odb_source_packed *source, unsigned flags) for (i = 0; i < m->num_objects + m->num_objects_in_base; i++) { struct object_id oid; struct pack_entry e; - off_t m_offset, p_offset; if (i > 0 && pairs[i-1].pack_int_id != pairs[i].pack_int_id && nth_midxed_pack(m, pairs[i-1].pack_int_id)) { @@ -1040,12 +1081,16 @@ int verify_midx_file(struct odb_source_packed *source, unsigned flags) break; } - m_offset = e.offset; - p_offset = find_pack_entry_one(&oid, e.p); - - if (m_offset != p_offset) - midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64), - pairs[i].pos, oid_to_hex(&oid), m_offset, p_offset); + /* + * Check that the exact offset recorded in the MIDX + * belongs to this OID. A pack index may contain + * duplicate OIDs, in which case an arbitrary OID lookup + * can return a different, equally valid copy than the + * one selected by the MIDX writer. + */ + if (!pack_index_has_oid_at_offset(e.p, &oid, e.offset)) + midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64), + pairs[i].pos, oid_to_hex(&oid), e.offset); midx_display_sparse_progress(progress, i + 1); } diff --git a/t/helper/test-find-pack.c b/t/helper/test-find-pack.c index 28d5b1fe094345..51093a7030c89a 100644 --- a/t/helper/test-find-pack.c +++ b/t/helper/test-find-pack.c @@ -11,12 +11,15 @@ * Display the path(s), one per line, of the packfile(s) containing * the given object. * + * With '--show-offset', display the offset selected by + * find_pack_entry_one() instead of the packfile path. + * * If '--check-count ' is passed, then error out if the number of * packfiles containing the object is not . */ static const char *const find_pack_usage[] = { - "test-tool find-pack [--check-count ] ", + "test-tool find-pack [--check-count ] [--show-offset] ", NULL }; @@ -24,11 +27,13 @@ int cmd__find_pack(int argc, const char **argv) { struct object_id oid; struct packed_git *p; - int count = -1, actual_count = 0; + int count = -1, actual_count = 0, show_offset = 0; const char *prefix = setup_git_directory(the_repository); struct option options[] = { OPT_INTEGER('c', "check-count", &count, "expected number of packs"), + OPT_BOOL(0, "show-offset", &show_offset, + "show matching pack offsets"), OPT_END(), }; @@ -40,8 +45,13 @@ int cmd__find_pack(int argc, const char **argv) die("cannot parse %s as an object name", argv[0]); repo_for_each_pack(the_repository, p) { - if (find_pack_entry_one(&oid, p)) { - printf("%s\n", p->pack_name); + off_t offset = find_pack_entry_one(&oid, p); + + if (offset) { + if (show_offset) + printf("%"PRIuMAX"\n", (uintmax_t)offset); + else + printf("%s\n", p->pack_name); actual_count++; } } diff --git a/t/t5308-pack-detect-duplicates.sh b/t/t5308-pack-detect-duplicates.sh index 4ff8f5b44953f4..493ebbc4af47d3 100755 --- a/t/t5308-pack-detect-duplicates.sh +++ b/t/t5308-pack-detect-duplicates.sh @@ -77,6 +77,20 @@ test_expect_success 'lookup in duplicated pack' ' test_cmp expect actual ' +test_expect_success 'verify MIDX containing duplicated pack objects' ' + git multi-pack-index write && + test-tool read-midx --show-objects .git/objects >midx-objects && + midx_offset=$( + awk -v oid="$LO_SHA1" "\$1 == oid { print \$2 }" Date: Fri, 24 Jul 2026 16:06:20 -0500 Subject: [PATCH 026/259] test-tool bitmap: reject packs with duplicate objects The bitmap writer builds its object-to-position map in packing_data, whose hash table permits one entry per OID. The bitmap test helper accepts an arbitrary existing pack, so a pack with duplicate entries calls packlist_alloc() twice for the same OID and trips its internal BUG(). Detect duplicate OIDs while ingesting the pack and die before calling packlist_alloc(). This preserves the internal uniqueness check for other callers while making unsupported helper input fail gracefully. Reuse t5308's existing duplicate pack to exercise the fatal diagnostic. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- t/helper/test-bitmap.c | 3 +++ t/t5308-pack-detect-duplicates.sh | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/t/helper/test-bitmap.c b/t/helper/test-bitmap.c index 8547ef67e243eb..6f851e04210198 100644 --- a/t/helper/test-bitmap.c +++ b/t/helper/test-bitmap.c @@ -50,6 +50,9 @@ static int add_packed_object(const struct object_id *oid, oi.typep = &type; + if (packlist_find(packed, oid)) + die("pack contains duplicate object %s", oid_to_hex(oid)); + entry = packlist_alloc(packed, oid); entry->idx.offset = nth_packed_object_offset(pack, pos); if (packed_object_info(NULL, pack, entry->idx.offset, &oi) < 0) diff --git a/t/t5308-pack-detect-duplicates.sh b/t/t5308-pack-detect-duplicates.sh index 493ebbc4af47d3..c6273a1aeb2973 100755 --- a/t/t5308-pack-detect-duplicates.sh +++ b/t/t5308-pack-detect-duplicates.sh @@ -59,6 +59,13 @@ test_expect_success 'index-pack will allow duplicate objects by default' ' git index-pack --stdin err && + test_grep "fatal: pack contains duplicate object" err +' + test_expect_success 'create batch-check test vectors' ' cat >input <<-EOF && $LO_SHA1 From fd2739b159d075cd4c6fa69b2cd876ba1caa5c88 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 16:06:24 -0500 Subject: [PATCH 027/259] pack-bitmap: handle duplicate pack entries during MIDX reuse A MIDX pseudo-pack assigns one bit position to each selected OID. Its preferred-pack fast paths handle duplicate OIDs across packs in the MIDX, where the MIDX chooses one copy, but assume that each individual pack contains one entry per OID. Consider a preferred pack ordered [A, B, A, C]. The MIDX retains one copy of A, leaving three pseudo-pack positions for four physical entries. This creates two problems: - In MIDX-backed single-pack reuse, bitmap_nr came from pack->num_objects and therefore counted both copies of A. Read the selected count from the root MIDX layer BTMP chunk instead. If that layer has no BTMP chunk, skip pack reuse and let the ordinary packing path handle the bitmap-selected objects. MIDXs without BTMP lose preferred-pack reuse until rewritten, rather than requiring a scan of the entire pseudo-pack to retain an optional optimization. - Correcting the range length still does not align the two orderings. Once one copy of A is omitted, MIDX position N need not name physical pack entry N. That mismatch matters during both selection and output. Whole-word selection bypasses the per-object check that the exact physical base of a delta is present. Verbatim reuse copies the first N pack entries for the first N bits. The per-object writer likewise used each bit as a physical pack position. Use the direct mapping only when the range begins at zero and its selected count equals the physical entry count. Since selected entries remain in pack-offset order, equal counts mean that none was omitted and the two positions coincide. For every other MIDX range, let whole-word selection fall through to the existing per-bit path, which resolves the selected MIDX offset to a physical pack position before checking the delta base. Disable verbatim prefix reuse, and perform the same translation in the per-object writer. Leave classic single-pack bitmap handling unchanged. The production writer creates those bitmaps only for the pack it just wrote, which has one entry per OID. The test helper can target an existing pack, but rejects duplicate OIDs. Cover the A, B, A, C case in a MIDX-backed preferred pack. Check reuse with BTMP, then hide BTMP and check that the optional reuse path is skipped. Also make B a delta against the omitted copy of A and require normal packing to handle it. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- builtin/pack-objects.c | 24 +++++----- pack-bitmap.c | 27 ++++++++++- t/t5332-multi-pack-reuse.sh | 89 +++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 15 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 3673b14b89b275..c33392296107e1 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1182,12 +1182,12 @@ static size_t write_reused_pack_verbatim(struct bitmapped_pack *reuse_packfile, size_t pos = 0; size_t end; - if (reuse_packfile->bitmap_pos) { + if (reuse_packfile->bitmap_pos || + reuse_packfile->bitmap_nr != reuse_packfile->p->num_objects) { /* - * We can't reuse whole chunks verbatim out of - * non-preferred packs since we can't guarantee that - * all duplicate objects were resolved in favor of - * that pack. + * We can't reuse whole chunks verbatim from non-preferred + * packs or packs with entries missing from the bitmap because + * bitmap and pack positions may differ. * * Even if we have a whole eword_t worth of bits that * could be reused, there may be objects between the @@ -1196,7 +1196,7 @@ static size_t write_reused_pack_verbatim(struct bitmapped_pack *reuse_packfile, * pack, causing us to send duplicate or unwanted * objects. * - * Handle non-preferred packs from within + * Handle these packs from within * write_reused_pack(), which inspects and reuses * individual bits. */ @@ -1263,20 +1263,18 @@ static void write_reused_pack(struct bitmapped_pack *reuse_packfile, if (pos + offset >= reuse_packfile->bitmap_pos + reuse_packfile->bitmap_nr) goto done; - if (reuse_packfile->bitmap_pos) { + if (reuse_packfile->bitmap_pos || + reuse_packfile->bitmap_nr != reuse_packfile->p->num_objects) { /* - * When doing multi-pack reuse on a - * non-preferred pack, translate bit positions - * from the MIDX pseudo-pack order back to their - * pack-relative positions before attempting - * reuse. + * Translate MIDX bitmap positions which do not + * correspond directly to physical pack positions. */ struct multi_pack_index *m = reuse_packfile->from_midx; uint32_t midx_pos; off_t pack_ofs; if (!m) - BUG("non-zero bitmap position without MIDX"); + BUG("cannot translate bitmap position without MIDX"); midx_pos = pack_pos_to_midx(m, pos + offset); pack_ofs = nth_midxed_offset(m, midx_pos); diff --git a/pack-bitmap.c b/pack-bitmap.c index d8dc4ae8d1633c..81412904365f40 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -2383,7 +2383,8 @@ static void reuse_partial_packfile_from_bitmap_1(struct bitmap_index *bitmap_git struct pack_window *w_curs = NULL; size_t pos = pack->bitmap_pos / BITS_IN_EWORD; - if (!pack->bitmap_pos) { + if (!pack->bitmap_pos && + pack->bitmap_nr == pack->p->num_objects) { /* * If we're processing the first (in the case of a MIDX, the * preferred pack) or the only (in the case of single-pack @@ -2399,6 +2400,9 @@ static void reuse_partial_packfile_from_bitmap_1(struct bitmap_index *bitmap_git * all ties are broken in favor of that pack (i.e. the one * we're currently processing). So any duplicate bases will be * resolved in favor of the pack we're processing. + * + * The range must also contain every physical pack entry so that + * bitmap and pack positions correspond. */ while (pos < result->word_alloc && pos < pack->bitmap_nr / BITS_IN_EWORD && @@ -2527,14 +2531,26 @@ void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, } else { struct packed_git *pack; uint32_t pack_int_id; + uint32_t bitmap_nr; if (bitmap_is_midx(bitmap_git)) { + struct bitmapped_pack bitmapped_pack; struct multi_pack_index *m = bitmap_git->midx; uint32_t preferred_pack_pos; while (m->base_midx) m = m->base_midx; + if (!m->chunk_bitmapped_packs) { + /* + * Without BTMP, determining the preferred + * pack's range requires scanning the pseudo-pack. + * Skip reuse and leave the bitmap result for + * normal packing. + */ + return; + } + if (midx_preferred_pack(m, &preferred_pack_pos) < 0) { warning(_("unable to compute preferred pack, disabling pack-reuse")); return; @@ -2542,6 +2558,12 @@ void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, pack = nth_midxed_pack(m, preferred_pack_pos); pack_int_id = preferred_pack_pos; + + if (nth_bitmapped_pack(m, &bitmapped_pack, + pack_int_id) < 0) + return; + pack = bitmapped_pack.p; + bitmap_nr = bitmapped_pack.bitmap_nr; } else { pack = bitmap_git->pack; /* @@ -2554,13 +2576,14 @@ void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, * that we do not expect to read this field. */ pack_int_id = -1; + bitmap_nr = pack->num_objects; } if (is_pack_valid(pack)) { ALLOC_GROW(packs, packs_nr + 1, packs_alloc); packs[packs_nr].p = pack; packs[packs_nr].pack_int_id = pack_int_id; - packs[packs_nr].bitmap_nr = pack->num_objects; + packs[packs_nr].bitmap_nr = bitmap_nr; packs[packs_nr].bitmap_pos = 0; packs[packs_nr].from_midx = bitmap_git->midx; packs_nr++; diff --git a/t/t5332-multi-pack-reuse.sh b/t/t5332-multi-pack-reuse.sh index 881ce668e1d14e..126ab9df991ce4 100755 --- a/t/t5332-multi-pack-reuse.sh +++ b/t/t5332-multi-pack-reuse.sh @@ -4,6 +4,7 @@ test_description='pack-objects multi-pack reuse' . ./test-lib.sh . "$TEST_DIRECTORY"/lib-bitmap.sh +. "$TEST_DIRECTORY"/lib-pack.sh GIT_TEST_MULTI_PACK_INDEX=0 GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0 @@ -32,6 +33,14 @@ pack_position () { grep "$1" objects | cut -d" " -f1 } +# B as an OFS_DELTA against A at the given one-byte distance. +pack_obj_b_ofs_a () { + pack_obj "$B" "$A" >b-ref.tmp && + printf "\145" && + printf "\\$(printf "%03o" "$1")" && + dd if=b-ref.tmp bs=1 skip=$((1 + $(test_oid rawsz))) 2>/dev/null +} + # test_pack_objects_reused_all test_pack_objects_reused_all () { : >trace2.txt && @@ -288,4 +297,84 @@ test_expect_success 'duplicate objects with verbatim reuse' ' ) ' +test_expect_success 'reuse with intra-pack duplicate objects' ' + git init intra-pack-duplicate-objects && + ( + cd intra-pack-duplicate-objects && + + # Make enough objects to exercise whole-word reuse. + test_commit_bulk 20 && + test_commit --printf A a "\7\0" && + test_commit --printf B b "\7\76" && + + objects_nr=$(git rev-list --count --objects --all) && + git rev-list --objects --all | + cut -d" " -f1 >objects && + A=$(test_oid packlib_7_0) && + B=$(test_oid packlib_7_76) && + grep -v -e "^$A$" -e "^$B$" objects >rest && + pack_obj "$A" >a-full && + pack_obj "$B" >b-full && + while read oid + do + pack_obj "$oid" || exit 1 + done rest.entries && + { + # Arrange the pack as A, B, A, C..., so that physical + # positions diverge from MIDX pseudo-pack order. + pack_header $((objects_nr + 1)) && + cat a-full b-full a-full rest.entries + } >duplicate.pack && + pack_trailer duplicate.pack && + clear_packs && + git index-pack --stdin b-delta && + { + pack_header "$((objects_nr + 1))" && + cat a-full a-full b-delta rest.entries + } >candidate.pack && + pack_trailer candidate.pack && + clear_packs && + git index-pack --stdin Date: Tue, 28 Jul 2026 17:45:51 +0200 Subject: [PATCH 028/259] replay: add helper to put entry into replayed_commits The function replay_revisions() in replay.c is rather lengthy. Extract the logic to put a commit entry into a `struct mapped_commits` into a helper function put_mapped_commit(). While at it, rename mapped_commit() to get_mapped_commit() to pair with this new function. Signed-off-by: Toon Claes Signed-off-by: Junio C Hamano --- replay.c | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/replay.c b/replay.c index 463c900d6c7c56..860e194ba064c9 100644 --- a/replay.c +++ b/replay.c @@ -254,9 +254,9 @@ static void set_up_replay_mode(struct repository *repo, strset_clear(&rinfo.positive_refs); } -static struct commit *mapped_commit(kh_oid_map_t *replayed_commits, - struct commit *commit, - struct commit *fallback) +static struct commit *get_mapped_commit(kh_oid_map_t *replayed_commits, + struct commit *commit, + struct commit *fallback) { khint_t pos; if (!commit) @@ -267,6 +267,21 @@ static struct commit *mapped_commit(kh_oid_map_t *replayed_commits, return kh_value(replayed_commits, pos); } +static void put_mapped_commit(kh_oid_map_t *replayed_commits, + struct commit *commit, + struct commit *new_commit) +{ + khint_t pos; + int ret; + + pos = kh_put_oid_map(replayed_commits, commit->object.oid, &ret); + if (ret == 0) + BUG("Duplicate rewritten commit: %s", + oid_to_hex(&commit->object.oid)); + + kh_value(replayed_commits, pos) = new_commit; +} + static struct commit *pick_regular_commit(struct repository *repo, struct commit *pickme, kh_oid_map_t *replayed_commits, @@ -287,7 +302,7 @@ static struct commit *pick_regular_commit(struct repository *repo, base_tree = lookup_tree(repo, repo->hash_algo->empty_tree); } - replayed_base = mapped_commit(replayed_commits, base, onto); + replayed_base = get_mapped_commit(replayed_commits, base, onto); replayed_base_tree = repo_get_commit_tree(repo, replayed_base); pickme_tree = repo_get_commit_tree(repo, pickme); @@ -427,8 +442,6 @@ int replay_revisions(struct rev_info *revs, replayed_commits = kh_init_oid_map(); while ((commit = get_revision(revs))) { const struct name_decoration *decoration; - khint_t pos; - int hr; if (commit->parents && commit->parents->next) die(_("replaying merge commits is not supported yet!")); @@ -440,11 +453,7 @@ int replay_revisions(struct rev_info *revs, break; /* Record commit -> last_commit mapping */ - pos = kh_put_oid_map(replayed_commits, commit->object.oid, &hr); - if (hr == 0) - BUG("Duplicate rewritten commit: %s\n", - oid_to_hex(&commit->object.oid)); - kh_value(replayed_commits, pos) = last_commit; + put_mapped_commit(replayed_commits, commit, last_commit); /* Update any necessary branches */ if (ref) From 959a273edd9cd9ca48b6c1a6f0922ffb56d37835 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Tue, 28 Jul 2026 17:45:52 +0200 Subject: [PATCH 029/259] replay: resolve the replay base outside pick_regular_commit() Depending on what gets passed into the function pick_regular_commit(), it decides the new base for the replayed commit. It first tries to find the replayed results of `pickme`'s parent in the `replayed_commits` map. If not found, it falls back to `onto`. When using git-replay(1) with --onto, the fallback is the revision passed in with this option, but when using --revert, the fallback is `last_commit`. It's rather confusing the base is decided partly inside pick_regular_commit() and partly by its caller. Move the base selection completely into the caller: replay_revisions(). This bundles all the logic of deciding on the base together. Also, this reduces the number of parameters of pick_regular_commit(), making its interface cleaner. This refactoring doesn't bring any behavior changes. Signed-off-by: Toon Claes Signed-off-by: Junio C Hamano --- replay.c | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/replay.c b/replay.c index 860e194ba064c9..7e35f40d379e84 100644 --- a/replay.c +++ b/replay.c @@ -284,25 +284,19 @@ static void put_mapped_commit(kh_oid_map_t *replayed_commits, static struct commit *pick_regular_commit(struct repository *repo, struct commit *pickme, - kh_oid_map_t *replayed_commits, - struct commit *onto, + struct commit *replayed_base, struct merge_options *merge_opt, struct merge_result *result, enum replay_mode mode, enum replay_empty_commit_action empty) { - struct commit *base, *replayed_base; struct tree *pickme_tree, *base_tree, *replayed_base_tree; - if (pickme->parents) { - base = pickme->parents->item; - base_tree = repo_get_commit_tree(repo, base); - } else { - base = NULL; + if (pickme->parents) + base_tree = repo_get_commit_tree(repo, pickme->parents->item); + else base_tree = lookup_tree(repo, repo->hash_algo->empty_tree); - } - replayed_base = get_mapped_commit(replayed_commits, base, onto); replayed_base_tree = repo_get_commit_tree(repo, replayed_base); pickme_tree = repo_get_commit_tree(repo, pickme); @@ -443,12 +437,26 @@ int replay_revisions(struct rev_info *revs, while ((commit = get_revision(revs))) { const struct name_decoration *decoration; + /* + * Decide where to replay this commit on. + * If the parent commit was replayed already, the replayed result + * can be found in `replayed_commits`. Otherwise fall back to `onto`. + * When reverting, commits are replayed in reverse order and thus + * its parent isn't replayed yet. Therefore revert commits are + * always replayed onto `last_commit`. + */ + struct commit *parent = commit->parents ? commit->parents->item : NULL; + struct commit *base = get_mapped_commit(replayed_commits, parent, onto); + + if (mode == REPLAY_MODE_REVERT) + base = last_commit; + if (commit->parents && commit->parents->next) die(_("replaying merge commits is not supported yet!")); - last_commit = pick_regular_commit(revs->repo, commit, replayed_commits, - mode == REPLAY_MODE_REVERT ? last_commit : onto, - &merge_opt, &result, mode, opts->empty); + last_commit = pick_regular_commit(revs->repo, commit, base, + &merge_opt, &result, + mode, opts->empty); if (!last_commit) break; From 6af34ada9641387947a0aa025029d6461283fe3c Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Tue, 28 Jul 2026 17:45:53 +0200 Subject: [PATCH 030/259] replay: offer an option to linearize the commit topology One of the stated goals of git-replay(1) is to allow implementing the git-rebase(1) functionality on the server side. The default mode of git-rebase(1) is to act as if `--no-rebase-merges` was given. This mode drops merge commits instead of replaying them, and linearizes the history into a sequence of regular (single-parent) commits. Add option `--linearize` to git-replay(1) to do the same. Each replayed commit is stacked on top of the previously replayed one. When a merge is encountered, the commits reachable from all of its sides are replayed into the single line and the merge itself is dropped. If a ref was pointing to a merge commit, that ref is updated to the merge's last replayed ancestor. git-replay(1) accepts multiple revision ranges, for example: $ git replay --onto main topic1 topic2 Without `--linearize` this replays 'topic1' and 'topic2' onto 'main' independently and updates both refs. For now this is disallowed with option `--linearize`. Linearizing more than one branch at once would concatenate unrelated histories into a single line, and update each branch to some point in that line. That won't be the result most users want, especially because the order depends on the order of the revision walk, not the order of the branch names on the command line. For the same reason disallow the use of `--contained` with `--linearize`. Users who want to linearize multiple branches are advised to do this in separate git-replay(1) invocations. Linearizing multiple branches at once might be added later. Note that `--linearize` is not modeled after git-rebase(1)'s `--rebase-merges[=]` interface. Recreating merges, by preserving their topology, is a distinct operation that would be a separate mode. `--linearize` only drops merges and replays commits linearly. So git-replay(1) uses its own option rather than reusing that interface. Based-on-patches-by: Johannes Schindelin Signed-off-by: Toon Claes Signed-off-by: Junio C Hamano --- Documentation/git-replay.adoc | 19 +++++- builtin/replay.c | 6 +- replay.c | 60 ++++++++++++------- replay.h | 5 ++ t/t3650-replay-basics.sh | 109 +++++++++++++++++++++++++++++++++- 5 files changed, 176 insertions(+), 23 deletions(-) diff --git a/Documentation/git-replay.adoc b/Documentation/git-replay.adoc index a32f72aead3750..656a6924d9bddd 100644 --- a/Documentation/git-replay.adoc +++ b/Documentation/git-replay.adoc @@ -10,7 +10,7 @@ SYNOPSIS -------- [verse] (EXPERIMENTAL!) 'git replay' ([--contained] --onto= | --advance= | --revert=) - [--ref=] [--ref-action=] + [--ref=] [--ref-action=] [--linearize] DESCRIPTION ----------- @@ -88,6 +88,23 @@ incompatible with `--contained` (which is a modifier for `--onto` only). + The default mode can be configured via the `replay.refAction` configuration variable. +--linearize:: + In this mode, each replayed commit is stacked on top of the + previously replayed one, so all replayed commits are flattened into + a single linear history. ++ +When a merge commit is encountered, the behavior of git-rebase(1)'s +option `--no-rebase-merges` is imitated. All commits in the range +reachable from the merge commit are replayed into a linear history, and +the merge commit itself is dropped. A ref that pointed to a merge commit +is updated to the merge's last replayed ancestor. ++ +Only a single branch can be linearized at a time: `--linearize` cannot +be combined with multiple positive revisions or with `--contained`, +because that would concatenate otherwise unrelated histories into one +line. To linearize several branches, replay them in separate `git +replay` invocations. + :: Range of commits to replay; see "Specifying Ranges" in linkgit:git-rev-parse[1]. In `--advance=` or diff --git a/builtin/replay.c b/builtin/replay.c index 39e3a86f6c10ab..d39626a37d055c 100644 --- a/builtin/replay.c +++ b/builtin/replay.c @@ -85,7 +85,7 @@ int cmd_replay(int argc, const char *const replay_usage[] = { N_("(EXPERIMENTAL!) git replay " "([--contained] --onto= | --advance= | --revert=)\n" - "[--ref=] [--ref-action=] "), + "[--ref=] [--ref-action=] [--linearize] "), NULL }; struct option replay_options[] = { @@ -111,6 +111,8 @@ int cmd_replay(int argc, N_("mode"), N_("control ref update behavior (update|print)"), PARSE_OPT_NONEG), + OPT_BOOL(0, "linearize", &opts.linearize, + N_("drop merge commits, replaying only non-merge commits")), OPT_END() }; @@ -132,6 +134,8 @@ int cmd_replay(int argc, opts.contained, "--contained"); die_for_incompatible_opt2(!!opts.ref, "--ref", !!opts.contained, "--contained"); + die_for_incompatible_opt2(opts.linearize, "--linearize", + !!opts.contained, "--contained"); /* Parse ref action mode from command line or config */ ref_mode = get_ref_action_mode(repo, ref_action); diff --git a/replay.c b/replay.c index 7e35f40d379e84..1e1bc7c10a8ffe 100644 --- a/replay.c +++ b/replay.c @@ -404,6 +404,12 @@ int replay_revisions(struct rev_info *revs, set_up_replay_mode(revs->repo, &revs->cmdline, opts->onto, &detached_head, &advance, &revert, &onto, &update_refs); + if (opts->linearize && + update_refs && strset_get_size(update_refs) > 1) { + ret = error(_("'--linearize' cannot be used with multiple revision ranges")); + goto out; + } + if (opts->ref) { struct object_id oid; @@ -437,26 +443,40 @@ int replay_revisions(struct rev_info *revs, while ((commit = get_revision(revs))) { const struct name_decoration *decoration; - /* - * Decide where to replay this commit on. - * If the parent commit was replayed already, the replayed result - * can be found in `replayed_commits`. Otherwise fall back to `onto`. - * When reverting, commits are replayed in reverse order and thus - * its parent isn't replayed yet. Therefore revert commits are - * always replayed onto `last_commit`. - */ - struct commit *parent = commit->parents ? commit->parents->item : NULL; - struct commit *base = get_mapped_commit(replayed_commits, parent, onto); - - if (mode == REPLAY_MODE_REVERT) - base = last_commit; - - if (commit->parents && commit->parents->next) - die(_("replaying merge commits is not supported yet!")); - - last_commit = pick_regular_commit(revs->repo, commit, base, - &merge_opt, &result, - mode, opts->empty); + if (commit->parents && commit->parents->next) { + if (!opts->linearize) + die(_("replaying merge commits is not supported yet!")); + /* + * Drop the merge commit: do not pick it, leave + * `last_commit` unchanged, and fall through to the + * rest of the loop. As a result: + * - refs pointing to the merge commit will be updated + * to `last_commit`. + * - the next replayed commit uses `last_commit` as its + * `base`. + */ + } else { + /* + * Decide where to replay this commit onto. + * If the parent commit was replayed already, the replayed result + * can be found in `replayed_commits`. Otherwise fall back to `onto`. + * When reverting, commits are replayed in reverse order and thus + * its parent isn't replayed yet. Therefore revert commits are + * always replayed onto `last_commit`. + * Also when opts->linearize is true, set the base to + * `last_commit` to create a single linear history. + */ + struct commit *parent = commit->parents ? commit->parents->item : NULL; + struct commit *base = get_mapped_commit(replayed_commits, parent, onto); + + if (opts->linearize || mode == REPLAY_MODE_REVERT) + base = last_commit; + + last_commit = pick_regular_commit(revs->repo, commit, base, + &merge_opt, &result, + mode, opts->empty); + } + if (!last_commit) break; diff --git a/replay.h b/replay.h index 491db145e30151..2c71afbfde05b4 100644 --- a/replay.h +++ b/replay.h @@ -62,6 +62,11 @@ struct replay_revisions_options { * Defaults to REPLAY_EMPTY_COMMIT_DROP. */ enum replay_empty_commit_action empty; + + /* + * Whether to linearize the commits (i.e. drop merge commits). + */ + int linearize; }; /* This struct is used as an out-parameter by `replay_revisions()`. */ diff --git a/t/t3650-replay-basics.sh b/t/t3650-replay-basics.sh index 3353bc4a4dc6ed..255bae58460297 100755 --- a/t/t3650-replay-basics.sh +++ b/t/t3650-replay-basics.sh @@ -52,8 +52,19 @@ test_expect_success 'setup' ' test_merge P O --no-ff && git switch main && + git switch --orphan unrelated && + test_commit unrelated-root && + git switch -c conflict B && - test_commit C.conflict C.t conflict + test_commit C.conflict C.t conflict && + git branch -D unrelated && + + git switch -c divergent-x main && + test_commit X && + git switch -c divergent-y main && + test_commit Y && + git switch divergent-x && + test_merge Z divergent-y --no-ff ' test_expect_success 'setup bare' ' @@ -565,4 +576,100 @@ test_expect_success '--onto with --ref rejects multiple revision ranges' ' test_grep "cannot be used with multiple revision ranges" err ' +test_expect_success 'replay to rebase merge commit with --linearize' ' + git replay --ref-action=print --linearize \ + --onto main I..topic-with-merge >result && + + test_line_count = 1 result && + + git log --format=%s $(cut -f 3 -d " " result) >actual && + test_write_lines O N J M L B A >expect && + test_cmp expect actual +' + +test_expect_success 'replay to rebase merge commit with --linearize down to the root commit' ' + git replay --ref-action=print --linearize \ + --onto unrelated-root topic-with-merge >result && + + test_line_count = 1 result && + + git log --format=%s $(cut -f 3 -d " " result) >actual && + test_write_lines O N J I B A unrelated-root >expect && + test_cmp expect actual +' + +test_expect_success 'replay to cherry-pick merge commit with --linearize' ' + git replay --ref-action=print --linearize \ + --advance main I..topic-with-merge >result && + + test_line_count = 1 result && + + git log --format=%s $(cut -f 3 -d " " result) >actual && + test_write_lines O N J M L B A >expect && + test_cmp expect actual && + + printf "update refs/heads/main " >expect && + printf "%s " $(cut -f 3 -d " " result) >>expect && + git rev-parse main >>expect && + test_cmp expect result +' + +test_expect_success 'replay --linearize produces the same patches' ' + git replay --ref-action=print --linearize \ + --onto main I..topic-with-merge >result && + + test_line_count = 1 result && + tip=$(cut -f 3 -d " " result) && + + # range-diff does not care about the dropped merge, + # so the original commits (I..topic-with-merge) + # and the replayed chain (main..tip) must produce identical patches. + git range-diff I..topic-with-merge main..$tip >out && + test_file_not_empty out && + test_grep ! -v "=" out && + + git log --oneline main..$tip >out && + test_line_count = 3 out +' + +test_expect_success '--linearize rejects multiple revision ranges' ' + test_must_fail git replay --ref-action=print --linearize \ + --onto main ^B topic2 topic3 topic4 2>err && + test_grep "cannot be used with multiple revision ranges" err +' + +test_expect_success 'replay with --linearize of a divergent merge keeps both sides' ' + git replay --ref-action=print --linearize \ + --onto main main..divergent-x >result && + test_line_count = 1 result && + tip=$(cut -f 3 -d " " result) && + + # The merge Z is dropped, but both X and Y are linearized onto main; + # neither side is lost. + git log --format=%s main..$tip >actual && + test_write_lines Y X >expect && + test_cmp expect actual +' + +test_expect_success '--linearize and --contained cannot be used together' ' + test_must_fail git replay --ref-action=print --linearize --contained \ + --onto main ^B topic-with-merge 2>err && + test_grep "cannot be used together" err +' + +test_expect_success 'replay --revert with --linearize reverts a range containing a merge' ' + git replay --ref-action=print --revert=divergent-x --linearize \ + main..divergent-x >result && + test_line_count = 1 result && + tip=$(cut -f 3 -d " " result) && + + git log --format=%s $tip >actual && + test_write_lines \ + "Revert \"X\"" "Revert \"Y\"" Z Y X M L B A >expect && + test_cmp expect actual && + + test_must_fail git cat-file -e $tip:X.t && + test_must_fail git cat-file -e $tip:Y.t +' + test_done From 64ffd57fb53718894b289d6919ee930000ebb330 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Sat, 25 Jul 2026 15:34:28 +0000 Subject: [PATCH 031/259] sequencer: teach autostash apply to report conflicts Add a conflicted parameter to apply_save_autostash_oid() and apply_save_autostash_ref() so callers can learn whether applying the stash resulted in conflicts. Thread the parameter through apply_autostash_ref() and update existing callers to pass NULL. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- builtin/checkout.c | 3 ++- builtin/commit.c | 2 +- builtin/merge.c | 6 +++--- sequencer.c | 29 +++++++++++++++++++---------- sequencer.h | 3 ++- 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/builtin/checkout.c b/builtin/checkout.c index b78b3a1d16def4..5858d1fc309dd0 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -1239,7 +1239,8 @@ static int switch_branches(const struct checkout_opts *opts, new_branch_info->name, "local", stash_label_base, - autostash_msg.buf); + autostash_msg.buf, + NULL); } if (ret) { branch_info_release(&old_branch_info); diff --git a/builtin/commit.c b/builtin/commit.c index 28f61745034506..d678a81865a6e0 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1980,7 +1980,7 @@ int cmd_commit(int argc, } apply_autostash_ref(the_repository, "MERGE_AUTOSTASH", - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); cleanup: free_commit_extra_headers(extra); diff --git a/builtin/merge.c b/builtin/merge.c index 5b46a596f0bdf4..cecb8fb716eb8f 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -538,7 +538,7 @@ static void finish(struct commit *head_commit, if (new_head) apply_autostash_ref(the_repository, "MERGE_AUTOSTASH", - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); strbuf_release(&reflog_message); } @@ -1680,7 +1680,7 @@ int cmd_merge(int argc, &commit->object.oid, overwrite_ignore)) { apply_autostash_ref(the_repository, "MERGE_AUTOSTASH", - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); ret = 1; goto done; } @@ -1844,7 +1844,7 @@ int cmd_merge(int argc, fprintf(stderr, _("Merge with strategy %s failed.\n"), use_strategies[0]->name); apply_autostash_ref(the_repository, "MERGE_AUTOSTASH", - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); ret = 2; goto done; } else if (best_strategy == wt_strategy) diff --git a/sequencer.c b/sequencer.c index 57855b0066ac98..1f4a9705c27a19 100644 --- a/sequencer.c +++ b/sequencer.c @@ -4730,7 +4730,8 @@ void create_autostash_ref(struct repository *r, const char *refname, static int apply_save_autostash_oid(const char *stash_oid, int attempt_apply, const char *label_ours, const char *label_theirs, const char *label_base, - const char *stash_msg) + const char *stash_msg, + bool *conflicted) { struct child_process child = CHILD_PROCESS_INIT; int ret = 0; @@ -4765,14 +4766,16 @@ static int apply_save_autostash_oid(const char *stash_oid, int attempt_apply, strvec_push(&store.args, stash_oid); if (run_command(&store)) ret = error(_("cannot store %s"), stash_oid); - else if (attempt_apply) + else if (attempt_apply) { + if (conflicted) + *conflicted = true; fprintf(stderr, _("Your local changes are stashed, however applying them\n" "resulted in conflicts. You can either resolve the conflicts\n" "and then discard the stash with \"git stash drop\", or, if you\n" "do not want to resolve them now, run \"git reset --hard\" and\n" "apply the local changes later by running \"git stash pop\".\n")); - else + } else fprintf(stderr, _("Autostash exists; creating a new stash entry.\n" "Your changes are safe in the stash.\n" @@ -4796,7 +4799,7 @@ static int apply_save_autostash(const char *path, int attempt_apply) strbuf_trim(&stash_oid); ret = apply_save_autostash_oid(stash_oid.buf, attempt_apply, - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); unlink(path); strbuf_release(&stash_oid); @@ -4815,19 +4818,24 @@ int apply_autostash(const char *path) int apply_autostash_oid(const char *stash_oid) { - return apply_save_autostash_oid(stash_oid, 1, NULL, NULL, NULL, NULL); + return apply_save_autostash_oid(stash_oid, 1, NULL, NULL, NULL, NULL, + NULL); } static int apply_save_autostash_ref(struct repository *r, const char *refname, int attempt_apply, const char *label_ours, const char *label_theirs, const char *label_base, - const char *stash_msg) + const char *stash_msg, + bool *conflicted) { struct object_id stash_oid; char stash_oid_hex[GIT_MAX_HEXSZ + 1]; int flag, ret; + if (conflicted) + *conflicted = false; + if (!refs_ref_exists(get_main_ref_store(r), refname)) return 0; @@ -4840,7 +4848,7 @@ static int apply_save_autostash_ref(struct repository *r, const char *refname, oid_to_hex_r(stash_oid_hex, &stash_oid); ret = apply_save_autostash_oid(stash_oid_hex, attempt_apply, label_ours, label_theirs, label_base, - stash_msg); + stash_msg, conflicted); refs_delete_ref(get_main_ref_store(r), "", refname, &stash_oid, REF_NO_DEREF); @@ -4851,16 +4859,17 @@ static int apply_save_autostash_ref(struct repository *r, const char *refname, int save_autostash_ref(struct repository *r, const char *refname) { return apply_save_autostash_ref(r, refname, 0, - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); } int apply_autostash_ref(struct repository *r, const char *refname, const char *label_ours, const char *label_theirs, - const char *label_base, const char *stash_msg) + const char *label_base, const char *stash_msg, + bool *conflicted) { return apply_save_autostash_ref(r, refname, 1, label_ours, label_theirs, label_base, - stash_msg); + stash_msg, conflicted); } static int checkout_onto(struct repository *r, struct replay_opts *opts, diff --git a/sequencer.h b/sequencer.h index 3164bd437d6a22..3e9ff093b2d1dd 100644 --- a/sequencer.h +++ b/sequencer.h @@ -237,7 +237,8 @@ int apply_autostash(const char *path); int apply_autostash_oid(const char *stash_oid); int apply_autostash_ref(struct repository *r, const char *refname, const char *label_ours, const char *label_theirs, - const char *label_base, const char *stash_msg); + const char *label_base, const char *stash_msg, + bool *conflicted); #define SUMMARY_INITIAL_COMMIT (1 << 0) #define SUMMARY_SHOW_AUTHOR_DATE (1 << 1) From f4996ed4619836f4108520a4721a0a781298b644 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Sat, 25 Jul 2026 15:34:29 +0000 Subject: [PATCH 032/259] checkout -m: refine autostash fallback When unpack_trees() fails under "git checkout -m", only create an autostash and retry if there are tracked local changes. Without such changes, the fallback cannot help and merely repeats the same failure. Use the conflict result from apply_autostash_ref() to print a blank line before the branch-switch message, visually separating it from the conflict advice. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- builtin/checkout.c | 17 +++++++++++++---- t/t7201-co.sh | 17 ++++++++++++++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/builtin/checkout.c b/builtin/checkout.c index 5858d1fc309dd0..d2ca09b673cee2 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -838,10 +838,11 @@ static void init_topts(struct unpack_trees_options *topts, static int merge_working_tree(const struct checkout_opts *opts, struct branch_info *old_branch_info, struct branch_info *new_branch_info, - bool quiet, + bool allow_autostash, int *writeout_error) { int ret; + bool can_autostash = false; struct lock_file lock_file = LOCK_INIT; struct tree *new_tree; @@ -888,9 +889,13 @@ static int merge_working_tree(const struct checkout_opts *opts, return 1; } + if (allow_autostash) + can_autostash = has_unstaged_changes(the_repository, 1) || + has_uncommitted_changes(the_repository, 1); + /* 2-way merge to the new branch */ init_topts(&topts, opts->show_progress, - opts->overwrite_ignore, quiet); + opts->overwrite_ignore, can_autostash); init_checkout_metadata(&topts.meta, new_branch_info->refname, new_branch_info->commit ? &new_branch_info->commit->object.oid : @@ -917,7 +922,8 @@ static int merge_working_tree(const struct checkout_opts *opts, clear_unpack_trees_porcelain(&topts); if (ret == -1) { rollback_lock_file(&lock_file); - return MERGE_WORKING_TREE_UNPACK_FAILED; + return can_autostash ? + MERGE_WORKING_TREE_UNPACK_FAILED : 1; } } @@ -1163,6 +1169,7 @@ static int switch_branches(const struct checkout_opts *opts, int flag, writeout_error = 0; int do_merge = 1; int created_autostash = 0; + bool autostash_conflicted = false; struct strbuf old_commit_shortname = STRBUF_INIT; struct strbuf autostash_msg = STRBUF_INIT; const char *stash_label_base = NULL; @@ -1240,7 +1247,7 @@ static int switch_branches(const struct checkout_opts *opts, "local", stash_label_base, autostash_msg.buf, - NULL); + &autostash_conflicted); } if (ret) { branch_info_release(&old_branch_info); @@ -1253,6 +1260,8 @@ static int switch_branches(const struct checkout_opts *opts, if (!opts->quiet && !old_branch_info.path && old_branch_info.commit && new_branch_info->commit != old_branch_info.commit) orphaned_commit_warning(old_branch_info.commit, new_branch_info->commit); + if (autostash_conflicted && !opts->quiet) + fputc('\n', stderr); update_refs_for_switch(opts, &old_branch_info, new_branch_info); if (created_autostash) { diff --git a/t/t7201-co.sh b/t/t7201-co.sh index 7613b1d2a446b0..52b2010d73e0d5 100755 --- a/t/t7201-co.sh +++ b/t/t7201-co.sh @@ -240,6 +240,14 @@ test_expect_success 'checkout -m creates a recoverable stash on conflict' ' test_grep "git stash drop" actual && test_grep "git stash pop" actual && test_grep "The following paths have local changes" actual && + sed -n "/apply the local changes later/,/Switched to branch/p" \ + actual >separator.actual && + cat >separator.expect <<-EOF && + apply the local changes later by running "git stash pop". + + Switched to branch ${SQ}side${SQ} + EOF + test_cmp separator.expect separator.actual && git log -p -1 --format="%gs%n%B" -g --diff-merges=1 refs/stash >actual && sed /^index/d actual >actual.trimmed && cat >expect <<-EOF && @@ -262,11 +270,18 @@ test_expect_success 'checkout -m creates a recoverable stash on conflict' ' git reset --hard ' -test_expect_success 'checkout -m which would overwrite untracked file' ' +test_expect_success 'checkout -m only retries untracked-file failure with local changes' ' git checkout -f --detach main && test_commit another-file && git checkout HEAD^ && >another-file.t && + test_must_fail env GIT_TRACE2_EVENT="$(pwd)/trace" \ + git checkout -m @{-1} 2>err && + test_grep "untracked working tree files" err && + grep "\"region_enter\".*\"category\":\"index\",\"label\":\"refresh\"" \ + trace >refresh.events && + test_line_count = 1 refresh.events && + fill 1 2 3 4 5 >one && test_must_fail git checkout -m @{-1} 2>err && q_to_tab >expect <<-\EOF && From cbbcd3395717397bd2f68a70e02c00f62edfbab9 Mon Sep 17 00:00:00 2001 From: Chungmin Lee Date: Mon, 27 Jul 2026 22:25:38 -0700 Subject: [PATCH 033/259] regexec: work around macOS TRE leak on invalid UTF-8 On macOS, the system regex engine leaks an internal buffer when regexec() encounters an invalid multibyte sequence in a UTF-8 locale. The line-by-line path can call regexec_buf() for each pattern on every line, so "git grep" can leak repeatedly on a file containing invalid UTF-8. The total leak grows with the number of calls, and the per-call allocation grows with the pattern's automaton. In one case, grepping a repository containing PDFs exhausted memory and caused the machine to restart. ce025ae4f61e (grep: disable lookahead on error, 2024-10-20) made "git grep" fall back to line-by-line matching when regexec() reports an error on invalid UTF-8. That fallback cannot prevent this leak: the allocation has already leaked when regexec() returns REG_ILLSEQ. Avoid the leaking path by providing a Darwin-specific regexec_buf(). Walk the input with mbrtowc(), split it at bytes that cannot form a complete multibyte character, and search each valid segment separately. This preserves matches in valid text on either side of an invalid byte. Search each segment with REG_STARTEND so match offsets remain relative to the original buffer. Set REG_NOTBOL and REG_NOTEOL for internal segment boundaries so "^" and "$" do not match there. Keep the flags clear at the true beginning and end of the buffer. Use the normal regexec_buf() path in single-byte locales, where no byte can form an invalid multibyte sequence. Use the bundled regex implementation unchanged when NO_REGEX is enabled. Declare the Darwin override in compat/darwin.h and map regexec_buf() to darwin_regexec_buf(). This follows the platform override pattern used by the other compatibility headers and leaves the common inline implementation as the default. There is no reliable way to detect a future macOS version in which the system regex implementation has been fixed. Even after a fix, Git will need the workaround while it supports affected macOS releases, so treat it as an indefinite compatibility workaround. Add tests for matches before, after, and between invalid bytes, including an offset check after an invalid byte. Also check incomplete trailing input and anchors at true and internal line boundaries. Signed-off-by: Chungmin Lee Signed-off-by: Junio C Hamano --- Makefile | 4 ++ compat/darwin.h | 8 +++ compat/darwin/regexec.c | 91 +++++++++++++++++++++++++++++ config.mak.uname | 1 + contrib/buildsystems/CMakeLists.txt | 3 + git-compat-util.h | 5 ++ meson.build | 5 ++ t/t7810-grep.sh | 37 ++++++++++++ 8 files changed, 154 insertions(+) create mode 100644 compat/darwin.h create mode 100644 compat/darwin/regexec.c diff --git a/Makefile b/Makefile index 1cec251f4387cf..81075c38a2af69 100644 --- a/Makefile +++ b/Makefile @@ -2264,6 +2264,10 @@ ifdef USE_ENHANCED_BASIC_REGULAR_EXPRESSIONS COMPAT_CFLAGS += -DUSE_ENHANCED_BASIC_REGULAR_EXPRESSIONS COMPAT_OBJS += compat/regcomp_enhanced.o endif +ifdef DARWIN_REGEXEC + COMPAT_OBJS += compat/darwin/regexec.o + BASIC_CFLAGS += -DDARWIN_REGEXEC +endif endif ifdef NATIVE_CRLF BASIC_CFLAGS += -DNATIVE_CRLF diff --git a/compat/darwin.h b/compat/darwin.h new file mode 100644 index 00000000000000..6fbdc345608ec4 --- /dev/null +++ b/compat/darwin.h @@ -0,0 +1,8 @@ +#ifndef COMPAT_DARWIN_H +#define COMPAT_DARWIN_H + +int darwin_regexec_buf(const regex_t *preg, const char *buf, size_t size, + size_t nmatch, regmatch_t pmatch[], int eflags); +#define regexec_buf darwin_regexec_buf + +#endif diff --git a/compat/darwin/regexec.c b/compat/darwin/regexec.c new file mode 100644 index 00000000000000..13fb7d5946bc8c --- /dev/null +++ b/compat/darwin/regexec.c @@ -0,0 +1,91 @@ +#include "git-compat-util.h" + +#include + +/* + * Darwin's TRE regex engine leaks an internal buffer when it encounters an + * invalid multibyte sequence. Since the leak has already happened when + * regexec() reports REG_ILLSEQ, keep invalid bytes out of regexec() by + * searching each valid segment separately. + */ + +/* + * Search buf[start, end), where size is the full size of buf. REG_STARTEND + * keeps match offsets relative to buf. Do not let an internal segment create + * a false beginning or end of line. + */ +static int regexec_segment(const regex_t *preg, const char *buf, + size_t size, size_t start, size_t end, + size_t nmatch, regmatch_t pmatch[], int eflags) +{ + eflags |= REG_STARTEND; + if (start > 0) + eflags |= REG_NOTBOL; + if (end < size) + eflags |= REG_NOTEOL; + pmatch[0].rm_so = start; + pmatch[0].rm_eo = end; + return regexec(preg, buf, nmatch, pmatch, eflags); +} + +int darwin_regexec_buf(const regex_t *preg, const char *buf, size_t size, + size_t nmatch, regmatch_t pmatch[], int eflags) +{ + size_t seg_start = 0, i = 0; + mbstate_t mbs; + + assert(nmatch > 0 && pmatch); + + /* + * A single-byte locale cannot contain an invalid multibyte sequence, + * so use regexec() directly. + */ + if (MB_CUR_MAX == 1) { + pmatch[0].rm_so = 0; + pmatch[0].rm_eo = size; + return regexec(preg, buf, nmatch, pmatch, eflags | REG_STARTEND); + } + + memset(&mbs, 0, sizeof(mbs)); + while (i < size) { + unsigned char c = (unsigned char)buf[i]; + size_t n; + + if (c < 0x80) { + i++; + continue; + } + + n = mbrtowc(NULL, buf + i, size - i, &mbs); + if (!n) + n = 1; + if (n != (size_t)-1 && n != (size_t)-2) { + i += n; + continue; + } + + /* + * -1 denotes an encoding error; -2 denotes an incomplete + * trailing sequence. In either case, buf[i] cannot begin a + * complete valid character within this buffer. Search an + * empty initial segment to preserve zero-width matches at the + * true beginning. + */ + if (i > seg_start || i == 0) { + int ret = regexec_segment(preg, buf, size, seg_start, i, + nmatch, pmatch, eflags); + if (ret != REG_NOMATCH) + return ret; + } + i++; + seg_start = i; + memset(&mbs, 0, sizeof(mbs)); + } + + /* + * Search the final segment even when it is empty, so an empty buffer + * or a buffer ending in invalid bytes still has its true end. + */ + return regexec_segment(preg, buf, size, seg_start, size, + nmatch, pmatch, eflags); +} diff --git a/config.mak.uname b/config.mak.uname index 9ebd240378ca59..4660ff3e8aadea 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -154,6 +154,7 @@ ifeq ($(uname_S),Darwin) HAVE_DEV_TTY = YesPlease COMPAT_OBJS += compat/precompose_utf8.o BASIC_CFLAGS += -DPRECOMPOSE_UNICODE + DARWIN_REGEXEC = YesPlease BASIC_CFLAGS += -DPROTECT_HFS_DEFAULT=1 HAVE_BSD_SYSCTL = YesPlease FREAD_READS_DIRECTORIES = UnfortunatelyYes diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index a57c4b464fa456..83e8b710ec88a3 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -519,6 +519,9 @@ if(NOT HAVE_REGEX) include_directories(${CMAKE_SOURCE_DIR}/compat/regex) list(APPEND compat_SOURCES compat/regex/regex.c ) add_compile_definitions(NO_REGEX NO_MBSUPPORT GAWK) +elseif(APPLE) + list(APPEND compat_SOURCES compat/darwin/regexec.c) + add_compile_definitions(DARWIN_REGEXEC) endif() diff --git a/git-compat-util.h b/git-compat-util.h index 88097764078538..96995c65d3ed63 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -162,6 +162,9 @@ static inline int is_xplatform_dir_sep(int c) #include "compat/win32/path-utils.h" #include "compat/msvc.h" #endif +#ifdef DARWIN_REGEXEC +#include "compat/darwin.h" +#endif /* used on Mac OS X */ #ifdef PRECOMPOSE_UNICODE @@ -992,6 +995,7 @@ static inline int strtol_i(char const *s, int base, int *result) #error "Git requires REG_STARTEND support. Compile with NO_REGEX=NeedsStartEnd" #endif +#ifndef regexec_buf static inline int regexec_buf(const regex_t *preg, const char *buf, size_t size, size_t nmatch, regmatch_t pmatch[], int eflags) { @@ -1000,6 +1004,7 @@ static inline int regexec_buf(const regex_t *preg, const char *buf, size_t size, pmatch[0].rm_eo = size; return regexec(preg, buf, nmatch, pmatch, eflags | REG_STARTEND); } +#endif #ifdef USE_ENHANCED_BASIC_REGULAR_EXPRESSIONS int git_regcomp(regex_t *preg, const char *pattern, int cflags); diff --git a/meson.build b/meson.build index 3247697f74aae1..53c4816323e83c 100644 --- a/meson.build +++ b/meson.build @@ -1387,6 +1387,11 @@ if not get_option('b_sanitize').contains('address') and get_option('regex').allo libgit_c_args += '-DUSE_ENHANCED_BASIC_REGULAR_EXPRESSIONS' compat_sources += 'compat/regcomp_enhanced.c' endif + + if host_machine.system() == 'darwin' + libgit_c_args += '-DDARWIN_REGEXEC' + compat_sources += 'compat/darwin/regexec.c' + endif elif not get_option('regex').enabled() libgit_c_args += [ '-DNO_REGEX', diff --git a/t/t7810-grep.sh b/t/t7810-grep.sh index d61c4a4d73c390..149654e7f4cb5b 100755 --- a/t/t7810-grep.sh +++ b/t/t7810-grep.sh @@ -89,6 +89,10 @@ test_expect_success setup ' function dummy() {} EOF printf "\200\nASCII\n" >invalid-utf8 && + printf "before\346world\n" >invalid-utf8-embedded && + printf "a\346b\347c\n" >invalid-utf8-multi && + printf "\346world\n" >invalid-utf8-leading && + printf "before\346\n" >invalid-utf8-trailing && if test_have_prereq FUNNYNAMES then echo unusual >"\"unusual\" pathname" && @@ -595,6 +599,39 @@ test_expect_success MB_REGEX 'grep two chars in single-char multibyte file' ' LC_ALL=en_US.UTF-8 test_expect_code 1 git grep ".." reverse-question-mark ' +test_expect_success MACOS,MB_REGEX 'grep matches valid text on both sides of invalid UTF-8' ' + LC_ALL=en_US.UTF-8 git grep -h "befo[r]e" invalid-utf8-embedded >actual && + test_cmp invalid-utf8-embedded actual && + LC_ALL=en_US.UTF-8 git grep -h "worl[d]" invalid-utf8-embedded >actual && + test_cmp invalid-utf8-embedded actual && + LC_ALL=en_US.UTF-8 git grep -h -o "worl[d]" invalid-utf8-embedded >actual && + echo world >expected && + test_cmp expected actual +' + +test_expect_success MACOS,MB_REGEX 'grep matches a run between two invalid sequences' ' + LC_ALL=en_US.UTF-8 git grep -h "[b]" invalid-utf8-multi >actual && + test_cmp invalid-utf8-multi actual +' + +test_expect_success MB_REGEX 'grep does not anchor ^ or $ inside an invalid-byte line' ' + test_expect_code 1 env LC_ALL=en_US.UTF-8 \ + git grep -h "^world" invalid-utf8-embedded && + test_expect_code 1 env LC_ALL=en_US.UTF-8 \ + git grep -h "before\$" invalid-utf8-embedded +' + +test_expect_success MACOS,MB_REGEX 'grep anchors ^ and $ at true line ends past invalid UTF-8' ' + LC_ALL=en_US.UTF-8 git grep -h "^before" invalid-utf8-embedded >actual && + test_cmp invalid-utf8-embedded actual && + LC_ALL=en_US.UTF-8 git grep -h "world\$" invalid-utf8-embedded >actual && + test_cmp invalid-utf8-embedded actual && + LC_ALL=en_US.UTF-8 git grep -h "^" invalid-utf8-leading >actual && + test_cmp invalid-utf8-leading actual && + LC_ALL=en_US.UTF-8 git grep -h "\$" invalid-utf8-trailing >actual && + test_cmp invalid-utf8-trailing actual +' + cat >expected < Date: Tue, 28 Jul 2026 15:32:37 -0700 Subject: [PATCH 034/259] SQUASH??? Signed-off-by: Junio C Hamano --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 81075c38a2af69..ed2868ce10bbc9 100644 --- a/Makefile +++ b/Makefile @@ -110,6 +110,9 @@ include shared.mak # Define USE_HOMEBREW_LIBICONV to link against libiconv installed by # Homebrew, if present. # +# Define DARWIN_REGEXEC if regexec() in your platform regex library +# leaks when fed an invalid UTF-8 sequence. +# # Define NO_APPLE_COMMON_CRYPTO if you are building on Darwin/Mac OS X # and do not want to use Apple's CommonCrypto library. This allows you # to provide your own OpenSSL library, for example from MacPorts. From c6b106fd8cfa13090d9a4b7417d01958ca6d37f1 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Fri, 31 Jul 2026 10:10:41 +0000 Subject: [PATCH 035/259] ci: cancel stale pull request workflow runs The CI workflow groups runs by commit hash, so every push to a pull request starts a separate workflow run. Group pull request runs by pull request number and cancel runs superseded by a newer push, while keeping push events grouped by commit hash for the skip-if-redundant behavior. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cf341d74dbff21..6b56c36996950f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,18 +5,20 @@ on: [push, pull_request] env: DEVELOPER: 1 -# If more than one workflow run is triggered for the very same commit hash -# (which happens when multiple branches pointing to the same commit), only -# the first one is allowed to run, the second will be kept in the "queued" -# state. This allows a successful completion of the first run to be reused -# in the second run via the `skip-if-redundant` logic in the `config` job. +# For pull requests, only the latest workflow run is allowed to proceed. +# Older runs are canceled when a new revision is pushed. # -# The only caveat is that if a workflow run is triggered for the same commit -# hash that another run is already being held, that latter run will be -# canceled. For more details about the `concurrency` attribute, see: +# For pushes, if more than one workflow run is triggered for the very same +# commit hash (which happens when multiple branches point to the same commit), +# only the first one is allowed to run. This allows a successful completion of +# the first run to be reused in the second run via the `skip-if-redundant` +# logic in the `config` job. +# +# For more details about the `concurrency` attribute, see: # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency concurrency: - group: ${{ github.sha }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: ci-config: From 240662ef2da8f5609a5c07068e3c2016d830e3b4 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 1 Aug 2026 10:41:44 -0700 Subject: [PATCH 036/259] gitattributes: document how external diff drivers relate to diff features The "Defining an external diff driver" section explains how to configure diff..command but not how the driver relates to the rest of Git's diff machinery. In particular, the command only replaces the textual patch: word diff, function context, color, and the like cannot apply to its output, while the summary formats, blame, and git log -L do not run it at all and keep using the builtin diff. Spell this out so the scope of an external diff driver is clear. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- Documentation/gitattributes.adoc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Documentation/gitattributes.adoc b/Documentation/gitattributes.adoc index bd76167a45eb71..da773e29247c46 100644 --- a/Documentation/gitattributes.adoc +++ b/Documentation/gitattributes.adoc @@ -784,6 +784,17 @@ with the above configuration, i.e. `j-c-diff`, with 7 parameters, just like `GIT_EXTERNAL_DIFF` program is called. See linkgit:git[1] for details. +An external diff driver replaces the patch Git would otherwise +produce for the path: Git runs the command and shows its output in +place of its own. Output features that post-process Git's diff do +not apply to the driver's output; word diff, function context (`-W`), +`--color-moved`, and coloring all act on Git's builtin diff, not the +driver's output. +The driver is consulted only when Git generates a textual patch. The +summary formats (`--stat`, `--numstat`, `--shortstat`, and +`--dirstat`), `git blame`, and `git log -L` do not run it and +continue to use Git's builtin diff. + If the program is able to ignore certain changes (similar to `git diff --ignore-space-change`), then also set the option `trustExitCode` to true. It is then expected to return exit code 1 if From b18db2edc5983a5ab492a5b035fa70dec5b9ff11 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 1 Aug 2026 10:41:45 -0700 Subject: [PATCH 037/259] diff: introduce a hunk provider interface To learn which line ranges changed between two blobs, every consumer in the diff machinery loads both blobs and runs xdiff. There is no other way to supply that answer, even when it is known elsewhere: a cache may hold the ranges from the last time the pair was diffed, and a format-aware process may have its own idea of which lines changed. Either could answer from the blob object ids alone, but the loading and computing are hard-wired into each consumer, so such an answer has no place to enter. Introduce the hunk provider interface, diff-provider.h, between asking the question and computing the answer. A provider answers a request made of the pair's identity, its blob object ids and the parameters that determine the diff. A provider is either authoritative, so its answer may deliberately differ from the builtin diff, or not, so its answer must reproduce the builtin result exactly. Every answer served from identity passes diff_provider_check_hunk() before a consumer sees it: coordinates fit int32, hunks are ordered and non-overlapping, and the unchanged runs between them match on both sides. A failing answer is discarded and the pair falls through as unanswered. Providers are repository-lifecycle objects. Each repository owns a chain of them, built on first consultation and released from repo_clear(), so a submodule gets its own providers and no provider state outlives the repository it serves. The chain has a fixed composition, and each provider gates itself per request, passing when it does not apply. Chain order is the authority: the first answer wins. A provider may instead refuse a pair whose request is shaped by parameters its recording key cannot express. After a refusal, no later provider answers the pair from identity, and the consumer must not record what it computes for it. The last provider is the builtin computation, the only one that computes rather than answering from identity, so a walk given a fill callback always ends in an answer, refusal or not. The walk in diff-provider.c maps a provider's four dispositions (answer, pass, fail, refuse) onto the consumer-facing outcomes, and checks with BUG() that only the computing provider fails and that it passes on a walk with no fill callback. The implementor contract, the provider struct, its dispositions, and the shared check, lives in diff-provider-internal.h, as refs/refs-internal.h is to refs.h; consumers see only diff-provider.h. The consumer surface is two types. struct diff_provider_request names what is diffed and under which parameters; each later commit that consults on more state adds the field it keys on (the object ids and diff options, then the path). enum diff_provider_outcome flattens two dependent axes into four points: the response state (answered, unanswered, failed) and, only when unanswered, whether the caller may record what it computes. The record rule rides in the outcome, not a separate flag, so -Wswitch forces every consumer to place the no-record arm. A provider added later maps onto these values inside the walk, so consumer code is written once. diff_provider_emit_hunks() is the consumer entry: the caller states the request, a hunk callback, and a content-loading callback that reaches the terminal provider only when the ranges are computed. Blame's pass_blame_to_parent() is the first consumer, since it knows both blob ids before reading either blob; its loads move into the fill callback. With only the terminal provider registered, every request still computes, so behavior is unchanged. (Blame's -C/-M split detection diffs partial buffers with no blob identity and stays on xdi_diff().) Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- Makefile | 1 + blame.c | 36 +++++++-- diff-provider-internal.h | 122 +++++++++++++++++++++++++++++++ diff-provider.c | 154 +++++++++++++++++++++++++++++++++++++++ diff-provider.h | 130 +++++++++++++++++++++++++++++++++ meson.build | 1 + repository.c | 3 + repository.h | 8 ++ 8 files changed, 447 insertions(+), 8 deletions(-) create mode 100644 diff-provider-internal.h create mode 100644 diff-provider.c create mode 100644 diff-provider.h diff --git a/Makefile b/Makefile index 98e995e4be0967..50c96807d67e55 100644 --- a/Makefile +++ b/Makefile @@ -1151,6 +1151,7 @@ LIB_OBJS += diff-delta.o LIB_OBJS += diff-merges.o LIB_OBJS += diff-lib.o LIB_OBJS += diff-no-index.o +LIB_OBJS += diff-provider.o LIB_OBJS += diff.o LIB_OBJS += diffcore-break.o LIB_OBJS += diffcore-delta.o diff --git a/blame.c b/blame.c index 126e2324162353..c3ef9c17f758c7 100644 --- a/blame.c +++ b/blame.c @@ -23,6 +23,7 @@ #include "commit-slab.h" #include "bloom.h" #include "commit-graph.h" +#include "diff-provider.h" define_commit_slab(blame_suspects, struct blame_origin *); static struct blame_suspects blame_suspects; @@ -1936,6 +1937,28 @@ static int blame_chunk_cb(long start_a, long count_a, return 0; } +struct blame_diff_fill_data { + struct blame_scoreboard *sb; + struct blame_origin *parent, *target; + int ignore_diffs; +}; + +/* + * Content load for diff_provider_emit_hunks(): runs when the diff is + * computed. + */ +static int blame_diff_fill(void *data, mmfile_t *old_file, mmfile_t *new_file) +{ + struct blame_diff_fill_data *f = data; + + fill_origin_blob(&f->sb->revs->diffopt, f->parent, old_file, + &f->sb->num_read_blob, f->ignore_diffs); + fill_origin_blob(&f->sb->revs->diffopt, f->target, new_file, + &f->sb->num_read_blob, f->ignore_diffs); + f->sb->num_get_patch++; + return 0; +} + /* * We are looking at the origin 'target' and aiming to pass blame * for the lines it is suspected to its parent. Run diff to find @@ -1945,9 +1968,11 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, struct blame_origin *target, struct blame_origin *parent, int ignore_diffs) { - mmfile_t file_p, file_o; struct blame_chunk_cb_data d; struct blame_entry *newdest = NULL; + struct blame_diff_fill_data fill_data = { sb, parent, target, ignore_diffs }; + xpparam_t xpp = { .flags = sb->xdl_opts }; + struct diff_provider_request req = { .repo = sb->repo, .xpp = &xpp }; if (!target->suspects) return; /* nothing remains for this target */ @@ -1958,13 +1983,8 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, d.ignore_diffs = ignore_diffs; d.dstq = &newdest; d.srcq = &target->suspects; - fill_origin_blob(&sb->revs->diffopt, parent, &file_p, - &sb->num_read_blob, ignore_diffs); - fill_origin_blob(&sb->revs->diffopt, target, &file_o, - &sb->num_read_blob, ignore_diffs); - sb->num_get_patch++; - - if (diff_hunks(&file_p, &file_o, blame_chunk_cb, &d, sb->xdl_opts)) + if (diff_provider_emit_hunks(&req, blame_diff_fill, &fill_data, + blame_chunk_cb, &d) == DIFF_PROVIDER_ERROR) die("unable to generate diff (%s -> %s)", oid_to_hex(&parent->commit->object.oid), oid_to_hex(&target->commit->object.oid)); diff --git a/diff-provider-internal.h b/diff-provider-internal.h new file mode 100644 index 00000000000000..8dd8e4b0dc84ec --- /dev/null +++ b/diff-provider-internal.h @@ -0,0 +1,122 @@ +#ifndef DIFF_PROVIDER_INTERNAL_H +#define DIFF_PROVIDER_INTERNAL_H + +#include "diff-provider.h" + +/* + * The implementor-facing half of the hunk provider interface: the + * provider chain a repository owns, and the rules a provider applies + * to its own answer before any consumer sees it. Provider + * implementations include this header; consumers of the interface + * use only diff-provider.h. + */ + +/* + * A provider's verdict on one request. Only the chain walk + * (diff-provider.c) sees these; it maps the dispositions of a whole + * walk onto the public outcome set. + */ +enum diff_provider_disposition { + /* + * The provider failed to produce the answer it owns. Only + * the computing provider returns this: its compute leg is + * the one part of a consultation that can fail, and the walk + * ends with the public error outcome. + */ + DIFF_PROVIDER_DISP_ERROR = -1, + + /* + * Answered: every hunk of the pair has been emitted through + * the consumer's callback. + */ + DIFF_PROVIDER_DISP_ANSWERED = 0, + + /* Not this provider's request: the walk consults the next one. */ + DIFF_PROVIDER_DISP_PASS, + + /* + * The pair must not be answered from identity nor recorded: + * the request is shaped by parameters the provider's + * recording key cannot express, so a recorded answer would + * not match this request, and this request's result must not + * be recorded under that key. The walk goes on, but consults + * only the computing provider, and its fall-through outcome + * tells the consumer not to record. + */ + DIFF_PROVIDER_DISP_STOP_NO_RECORD, +}; + +/* + * One provider in a repository's chain (repository.h). The chain is + * assembled in diff-provider.c with a fixed composition; whether a + * provider applies to a request is decided by nobody but the + * provider, whose consult gates itself and passes. Chain position + * carries the authority resolution: an earlier provider's answer or + * refusal outranks every provider after it. + */ +struct diff_provider { + /* + * Consult this provider for one request. fill is NULL on a + * consult-only walk; only the computing provider reads it, + * and it must pass when fill is NULL. + */ + enum diff_provider_disposition + (*consult)(struct diff_provider *provider, + const struct diff_provider_request *req, + diff_provider_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data); + + /* + * Tear down the provider's state, or NULL when it owns none. + * Runs when the owning repository is cleared; the chain frees + * the provider itself afterwards. + */ + void (*release)(struct diff_provider *provider); + + void *state; + + /* + * Set on the provider that loads content and computes rather + * than answering from the request's identity. It alone is + * still consulted after a stop-no-record: an identity answer + * may no longer be served, but the computation must still + * run. + */ + unsigned computes:1; + + struct diff_provider *next; +}; + +/* + * Incremental well-formedness check for a provider-supplied hunk + * sequence, shared by every provider. Each coordinate, and each + * hunk's end (its start plus count), must fit int32 (a consumer may + * truncate to int, and a provider may serialize as such); hunks must + * be in order and must not overlap; and the unchanged run between + * hunks must be the same length on both sides, or a consumer that + * walks the two files in lockstep desynchronizes. Every rule + * constrains differences between coordinates, so the check applies + * to 0-based and 1-based sequences alike. + * + * Feed the hunks in order to a zero-initialized struct; the first + * nonzero return names the violated rule, and the whole sequence must + * then be discarded unemitted. + */ +struct diff_provider_hunks_check { + int64_t prev_old_end, prev_new_end; +}; + +enum diff_provider_hunks_error { + DIFF_PROVIDER_HUNKS_OK = 0, + DIFF_PROVIDER_HUNKS_RANGE, /* negative or beyond int32 */ + DIFF_PROVIDER_HUNKS_OVERLAP, /* out of order or overlapping */ + DIFF_PROVIDER_HUNKS_MISALIGNED, /* unchanged runs differ in length */ +}; + +enum diff_provider_hunks_error +diff_provider_check_hunk(struct diff_provider_hunks_check *c, + long old_start, long old_count, + long new_start, long new_count); + +#endif /* DIFF_PROVIDER_INTERNAL_H */ diff --git a/diff-provider.c b/diff-provider.c new file mode 100644 index 00000000000000..b69854fb63b042 --- /dev/null +++ b/diff-provider.c @@ -0,0 +1,154 @@ +#include "git-compat-util.h" +#include "diff-provider-internal.h" +#include "repository.h" + +/* + * The terminal provider: the builtin computation. A request that + * carries a fill callback is answered by loading the pair's content + * and running xdiff, so a walk that reaches it never falls through + * to the consumer. On a consult-only walk it passes, and the walk's + * fall-through outcome tells the consumer to compute. + */ +static enum diff_provider_disposition +builtin_consult(struct diff_provider *provider UNUSED, + const struct diff_provider_request *req, + diff_provider_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data) +{ + xdemitconf_t xecfg = { .hunk_func = hunk_cb }; + xdemitcb_t ecb = { .priv = cb_data }; + mmfile_t old_file, new_file; + + if (!fill) + return DIFF_PROVIDER_DISP_PASS; + if (fill(fill_data, &old_file, &new_file) < 0) + return DIFF_PROVIDER_DISP_ERROR; + if (xdi_diff(&old_file, &new_file, req->xpp, &xecfg, &ecb) < 0) + return DIFF_PROVIDER_DISP_ERROR; + return DIFF_PROVIDER_DISP_ANSWERED; +} + +static struct diff_provider *builtin_provider_new(void) +{ + struct diff_provider *p = xcalloc(1, sizeof(*p)); + + p->consult = builtin_consult; + p->computes = 1; + return p; +} + +/* + * The repository's chain, assembled on first walk. The composition + * is fixed; the builtin computation is the terminal provider, so the + * chain always ends in an implementor that can answer. Nothing is + * decided per repository here; each provider gates itself per + * request. + */ +static struct diff_provider *provider_chain(struct repository *r) +{ + struct diff_provider **tail = &r->diff_providers; + + if (*tail) + return *tail; + *tail = builtin_provider_new(); + return r->diff_providers; +} + +void diff_providers_clear(struct repository *r) +{ + struct diff_provider *p = r->diff_providers; + + while (p) { + struct diff_provider *next = p->next; + + if (p->release) + p->release(p); + free(p); + p = next; + } + r->diff_providers = NULL; +} + +/* + * The walk behind diff_provider_emit_hunks(): consult the chain in + * order and map its dispositions onto the outcome set. The first + * answer ends the walk. A stop-no-record disposition + * (diff-provider-internal.h) is a refusal, not a pass: the provider + * does not answer, but rules the pair out of identity service and + * out of recording, so from then on the walk consults only the + * computing provider, and a walk that ends unanswered carries the + * no-record verdict. With a fill callback the terminal provider + * computes instead of passing, so an emit walk returns only + * answered or error. + */ +static enum diff_provider_outcome +walk_providers(const struct diff_provider_request *req, + diff_provider_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data) +{ + struct diff_provider *p; + int no_record = 0; + + for (p = provider_chain(req->repo); p; p = p->next) { + enum diff_provider_disposition disp; + + if (no_record && !p->computes) + continue; + disp = p->consult(p, req, fill, fill_data, + hunk_cb, cb_data); + if (disp == DIFF_PROVIDER_DISP_ERROR && !p->computes) + BUG("only the computing provider may return the " + "error disposition"); + if (p->computes && !fill && disp != DIFF_PROVIDER_DISP_PASS) + BUG("the computing provider must pass on a " + "fill-less walk"); + switch (disp) { + case DIFF_PROVIDER_DISP_ANSWERED: + return DIFF_PROVIDER_ANSWERED; + case DIFF_PROVIDER_DISP_PASS: + continue; + case DIFF_PROVIDER_DISP_STOP_NO_RECORD: + no_record = 1; + continue; + case DIFF_PROVIDER_DISP_ERROR: + return DIFF_PROVIDER_ERROR; + } + } + return no_record ? DIFF_PROVIDER_UNANSWERED_NO_RECORD : + DIFF_PROVIDER_UNANSWERED; +} + +enum diff_provider_hunks_error +diff_provider_check_hunk(struct diff_provider_hunks_check *c, + long old_start, long old_count, + long new_start, long new_count) +{ + if (old_start < 0 || old_count < 0 || + new_start < 0 || new_count < 0 || + old_start > INT32_MAX || old_count > INT32_MAX || + new_start > INT32_MAX || new_count > INT32_MAX || + (int64_t)old_start + old_count > INT32_MAX || + (int64_t)new_start + new_count > INT32_MAX) + return DIFF_PROVIDER_HUNKS_RANGE; + if (old_start < c->prev_old_end || new_start < c->prev_new_end) + return DIFF_PROVIDER_HUNKS_OVERLAP; + if (old_start - c->prev_old_end != new_start - c->prev_new_end) + return DIFF_PROVIDER_HUNKS_MISALIGNED; + /* + * With each field bounded to int32 above, the int64 sums cannot + * overflow even where long is 32-bit, and the range rule has + * already capped them at INT32_MAX. + */ + c->prev_old_end = (int64_t)old_start + old_count; + c->prev_new_end = (int64_t)new_start + new_count; + return DIFF_PROVIDER_HUNKS_OK; +} + +enum diff_provider_outcome +diff_provider_emit_hunks(const struct diff_provider_request *req, + diff_provider_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data) +{ + return walk_providers(req, fill, fill_data, hunk_cb, cb_data); +} diff --git a/diff-provider.h b/diff-provider.h new file mode 100644 index 00000000000000..1a7e4e299c04c0 --- /dev/null +++ b/diff-provider.h @@ -0,0 +1,130 @@ +#ifndef DIFF_PROVIDER_H +#define DIFF_PROVIDER_H + +#include "xdiff-interface.h" + +/* + * The hunk provider interface sits between naming a pair of file + * versions to diff and computing their changed line ranges. + * Consumers that operate on hunk coordinates route their diff + * through here, so that a provider can answer for the pair before + * its content is loaded. + * + * A hunk provider answers a consumer's request from the pair's + * identity (its blob object ids) and the parameters that determine + * the diff; a request no provider answers falls through to the + * consumer's own computation. A provider is either authoritative for + * its requests, meaning its answer may deliberately differ from the + * builtin diff, or not, meaning its answer must reproduce the builtin + * result exactly. The interface resolves that authority through a + * provider chain owned by the repository, built on first consultation + * and released by repo_clear(): chain order is the resolution, and + * the builtin computation itself is the chain's terminal provider. A + * consumer never names a provider; it reads the outcome below. + * Every answer a provider serves from identity passes the shared + * coordinate check (diff-provider-internal.h) before any consumer + * sees it. + */ + +struct repository; + +/* + * The result of a consultation: two dependent axes flattened into + * their four valid points. The first axis is the state of the + * response: the pair was answered, no provider answered, or (from + * diff_provider_emit_hunks() alone) the attempt failed. The second + * axis exists only in the unanswered state: whether what the caller + * computes for this request may be recorded, the one rule the + * interface imposes on an otherwise free caller. The rule travels + * in the outcome because the knowledge is a provider's while the + * recording is the caller's, and it shares the enum with the state, + * rather than riding a separate flag, so that no meaningless + * combination is representable and -Wswitch forces every consumer + * that switches to place the no-record arm. + * + * These values describe consultations, not providers: the set does + * not grow when a provider is added; a new provider maps onto these + * values inside the interface, so consumer code is written once. + * Each entry point returns a subrange of the set (stated at its + * declaration); a switch over this enum should list every value and + * omit "default:" so -Wswitch keeps it exhaustive, and a caller for + * whom only one value is actionable may compare against that value + * alone. + */ +enum diff_provider_outcome { + /* + * Loading or diffing the pair failed. Returned only by + * diff_provider_emit_hunks(), whose compute leg is the only + * part of a consultation that can fail. + */ + DIFF_PROVIDER_ERROR = -1, + + /* + * The request is answered: every hunk of the pair has been + * emitted through the callback. An authoritative provider + * that finds the pair equivalent answers with no hunks at + * all, so a callback that never fired is an answer, not an + * accident. + */ + DIFF_PROVIDER_ANSWERED = 0, + + /* + * No provider answered. What happens next is the caller's + * business, typically computing the diff itself; a result it + * computes for this request may be recorded. + */ + DIFF_PROVIDER_UNANSWERED, + + /* + * No provider answered, and what the caller computes for + * this request must not be recorded: either an authoritative + * provider owns the pair and declined this request, or the + * request is shaped by parameters outside the recording key, + * the key a recorded result is later served by. + */ + DIFF_PROVIDER_UNANSWERED_NO_RECORD, +}; + +/* + * A consultation request. The interface consults providers from + * these fields alone; no content is loaded before an answer. + * + * repo owns the provider chain the request walks. xpp carries the + * parameters the diff runs with. Each provider gates itself on the + * fields that concern it. + */ +struct diff_provider_request { + struct repository *repo; + const xpparam_t *xpp; +}; + +/* + * Load the pair's content. Called at most once per request, only + * when the ranges are computed rather than provided. The buffers + * borrow storage owned by the callback's owner. + */ +typedef int (*diff_provider_fill_fn)(void *data, mmfile_t *old_file, + mmfile_t *new_file); + +/* + * Consult the providers and, when no identity answer serves the + * request, load the pair's content through fill and compute its + * exact changed ranges (context 0). Emits to hunk_cb either way and + * returns DIFF_PROVIDER_ANSWERED, or DIFF_PROVIDER_ERROR when fill + * or the diff fails. The unanswered outcomes are never returned: a + * pair no provider answers is computed here instead of in the caller. + */ +enum diff_provider_outcome +diff_provider_emit_hunks(const struct diff_provider_request *req, + diff_provider_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data); + +/* + * Release the repository's provider chain: stop any provider-owned + * processes and free the providers. Called by repo_clear(); the + * chain builds again on the next consultation. + */ +void diff_providers_clear(struct repository *r); + +#endif /* DIFF_PROVIDER_H */ diff --git a/meson.build b/meson.build index f7c40ea079b797..539a50f90e597b 100644 --- a/meson.build +++ b/meson.build @@ -356,6 +356,7 @@ libgit_sources = [ 'diff-merges.c', 'diff-lib.c', 'diff-no-index.c', + 'diff-provider.c', 'diff.c', 'diffcore-break.c', 'diffcore-delta.c', diff --git a/repository.c b/repository.c index 2ef0778846bcf1..c9418f0ae1f231 100644 --- a/repository.c +++ b/repository.c @@ -5,6 +5,7 @@ #include "odb.h" #include "odb/source.h" #include "config.h" +#include "diff-provider.h" #include "gettext.h" #include "object.h" #include "lockfile.h" @@ -383,6 +384,8 @@ void repo_clear(struct repository *repo) FREE_AND_NULL(repo->submodule_prefix); FREE_AND_NULL(repo->ref_storage_payload); + diff_providers_clear(repo); + odb_free(repo->objects); repo->objects = NULL; diff --git a/repository.h b/repository.h index b7673079119e6f..8b4747fb2c127b 100644 --- a/repository.h +++ b/repository.h @@ -7,6 +7,7 @@ #include "environment.h" struct config_set; +struct diff_provider; struct git_hash_algo; struct index_state; struct lock_file; @@ -161,6 +162,13 @@ struct repository { /* Repository's remotes and associated structures. */ struct remote_state *remote_state; + /* + * The repository's diff hunk provider chain, NULL until the + * first consultation builds it (diff-provider.c); repo_clear() + * releases it. + */ + struct diff_provider *diff_providers; + /* Repository's current hash algorithm, as serialized on disk. */ const struct git_hash_algo *hash_algo; From 42824db1138dbe0e64d5c49b52202f0cd30fa967 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 1 Aug 2026 10:41:46 -0700 Subject: [PATCH 038/259] diff-hunks: add the store format, library, and command Blame and "git log --stat" recover hunk coordinates by diffing blob pairs, and recompute them on every run. Add a cache of those coordinates at $GIT_DIR/objects/info/diff-hunks, beside the commit-graph, so a later run can look them up instead of decompressing the blobs and running xdiff again. The store is a single chunk-format file (see gitformat-chunk(5)): an 8-byte header, a DHIX index of fixed-size entries sorted by key, a DHDT segment of hunk records, and a trailing hash checksum. An entry is keyed by the two blob object ids and the xdl_opts the pair was diffed under, so a stored result is served only where that exact key recurs, independent of path. A zero-context diff trims unchanged lines from hunk edges and can pick a different but equally valid set of hunks than an untrimmed diff, so a recording caller stores a pair only when its trimmed and untrimmed diffs are identical; such an entry answers any consumer at any context, and the rare divergent pair is always computed. Identical hunk blocks are interned once and shared across keys. The library provides a reader (repo_diff_hunks_store and _replay, gated by core.diffHunks), loaded once and cached on the object database as the commit-graph is, and a writer that accumulates entries and flushes them in one atomic pass. An absent, corrupt, or disabled store reads as all misses. A record with no hunks is invalid too: replaying it would claim the pair equivalent, which the store never asserts, so it reads as a miss. Ordinary reads are diagnostic-free. Loading parses the chunk table through read_table_of_contents_quiet(), new in chunk-format, which prints nothing on a malformed table and takes the repository's hash algorithm rather than the_hash_algo, so the file is bounds-checked under the algorithm it is keyed by. The flush closes the repository's mmapped store and forgets that loading was attempted before committing the lockfile. A warming run that also reads may hold the file it is replacing mapped, and the rename must not land on a live mapping, which Windows refuses; a read after the flush then observes the committed file. commit-graph closes its graph before committing for the same reason. Writing is off by default, enabled per run by GIT_DIFF_HUNKS_WRITE or persistently by diffHunks.write, the environment winning. A writer seeds from the existing store, so a flush merges rather than replaces. The seed's checksum is verified first: a corrupt store is discarded, not rewritten with a fresh checksum verify could no longer catch. An entry that fails the shared diff_provider_check_hunk() or names no blob is dropped with a warning, since it would only ever read as a miss. A seed that discarded or dropped anything forces the flush even when the warming run computed nothing new. The writer fsyncs through a new diff-hunks core.fsync component. "git diff-hunks" inspects and manages the file: "verify" checks the checksum, chunk table, sort order, entry bounds, and every entry's hunk sequence against that shared check, so a store whose entries could only read as misses fails verify; "clear" removes the file. Later patches wire the readers and the writer into the diff and blame paths. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- .gitignore | 1 + Documentation/Makefile | 1 + Documentation/config.adoc | 2 + Documentation/config/core.adoc | 10 +- Documentation/config/diff-hunks.adoc | 8 + Documentation/git-diff-hunks.adoc | 146 ++++ Documentation/gitformat-diff-hunks.adoc | 129 ++++ Documentation/meson.build | 2 + Makefile | 2 + builtin.h | 1 + builtin/diff-hunks.c | 53 ++ chunk-format.c | 62 +- chunk-format.h | 14 + command-list.txt | 2 + diff-hunks.c | 916 ++++++++++++++++++++++++ diff-hunks.h | 117 +++ environment.c | 1 + git.c | 1 + meson.build | 2 + odb.c | 2 + odb.h | 4 + repo-settings.c | 1 + repo-settings.h | 1 + write-or-die.h | 7 +- 24 files changed, 1467 insertions(+), 18 deletions(-) create mode 100644 Documentation/config/diff-hunks.adoc create mode 100644 Documentation/git-diff-hunks.adoc create mode 100644 Documentation/gitformat-diff-hunks.adoc create mode 100644 builtin/diff-hunks.c create mode 100644 diff-hunks.c create mode 100644 diff-hunks.h diff --git a/.gitignore b/.gitignore index 4da58c6754899e..4173111c01b2b6 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ /git-diagnose /git-diff /git-diff-files +/git-diff-hunks /git-diff-index /git-diff-pairs /git-diff-tree diff --git a/Documentation/Makefile b/Documentation/Makefile index 2699f0b24af192..170fcee66e23d5 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -33,6 +33,7 @@ MAN5_TXT += gitattributes.adoc MAN5_TXT += gitformat-bundle.adoc MAN5_TXT += gitformat-chunk.adoc MAN5_TXT += gitformat-commit-graph.adoc +MAN5_TXT += gitformat-diff-hunks.adoc MAN5_TXT += gitformat-index.adoc MAN5_TXT += gitformat-loose.adoc MAN5_TXT += gitformat-pack.adoc diff --git a/Documentation/config.adoc b/Documentation/config.adoc index 1ef72de62f2ba6..8a172d52f37ec7 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -472,6 +472,8 @@ include::config/credential.adoc[] include::config/diff.adoc[] +include::config/diff-hunks.adoc[] + include::config/difftool.adoc[] include::config/extensions.adoc[] diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index a0ebf03e2eb050..9595619c610425 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -670,12 +670,13 @@ but risks losing recent work in the event of an unclean system shutdown. * `pack` hardens objects added to the repo in packfile form. * `pack-metadata` hardens packfile bitmaps and indexes. * `commit-graph` hardens the commit-graph file. +* `diff-hunks` hardens the diff-hunks store. * `index` hardens the index when it is modified. * `objects` is an aggregate option that is equivalent to `loose-object,pack`. * `reference` hardens references modified in the repo. * `derived-metadata` is an aggregate option that is equivalent to - `pack-metadata,commit-graph`. + `pack-metadata,commit-graph,diff-hunks`. * `committed` is an aggregate option that is currently equivalent to `objects`. This mode sacrifices some performance to ensure that work that is committed to the repository with `git commit` or similar commands @@ -750,6 +751,13 @@ core.commitGraph:: to parse the graph structure of commits. Defaults to true. See linkgit:git-commit-graph[1] for more information. +core.diffHunks:: + If true, then Git will consult the diff-hunks store (if it + exists) to skip recomputing diff hunk coordinates in commands + such as `git log --stat` and linkgit:git-blame[1]. This controls + only reading; writing the store is controlled by `diffHunks.write`. + See linkgit:git-diff-hunks[1] for more information. Defaults to true. + core.useReplaceRefs:: If set to `false`, behave as if the `--no-replace-objects` option was given on the command line. See linkgit:git[1] and diff --git a/Documentation/config/diff-hunks.adoc b/Documentation/config/diff-hunks.adoc new file mode 100644 index 00000000000000..ad76d1c6a9a2e4 --- /dev/null +++ b/Documentation/config/diff-hunks.adoc @@ -0,0 +1,8 @@ +diffHunks.write:: + If true, diff-producing commands (`git diff`, `git log`, + `git show`, or `git diff-tree` with a `--stat`, `--numstat`, or + `--shortstat` format) write the hunks + they compute to the diff-hunks store, filling it as a side effect. + The `GIT_DIFF_HUNKS_WRITE` environment variable overrides this for + a single invocation. Reading the store is controlled separately by + `core.diffHunks`. See linkgit:git-diff-hunks[1]. Defaults to false. diff --git a/Documentation/git-diff-hunks.adoc b/Documentation/git-diff-hunks.adoc new file mode 100644 index 00000000000000..25cab2ea7db8d8 --- /dev/null +++ b/Documentation/git-diff-hunks.adoc @@ -0,0 +1,146 @@ +git-diff-hunks(1) +================= + +NAME +---- +git-diff-hunks - Inspect and manage the diff-hunks store + +SYNOPSIS +-------- +[synopsis] +git diff-hunks verify +git diff-hunks clear + +DESCRIPTION +----------- + +The diff-hunks store is a cache of diff hunk coordinates, the line +ranges that changed between two blobs, so that commands +which need them, such as linkgit:git-blame[1] and `git log` and `git diff` +with the `--stat`, `--numstat`, and `--shortstat` formats, can skip +running the diff algorithm, and blame can skip loading the blob +content. (The summary formats still test each pair for binariness, +which can load the blobs.) + +The store is a single file, `$GIT_DIR/objects/info/diff-hunks`. Reading is +enabled by default; writing is off by default. A `git diff`, `git log`, +`git show`, or `git diff-tree` that produces one of the stat formats +fills the store as a side effect, but only when writing is enabled for +that run (see "WARMING THE STORE" below), so ordinary reads never +modify the repository. When the store does not have the pair, holds a +different object hash, the file is unreadable, or an object replacement +redirects one of the blobs, the consumer falls back to computing the +diff. A store only speeds up these commands; it never changes their +output. + +`git diff-hunks` itself only inspects and manages the file. See +linkgit:gitformat-diff-hunks[5] for the file format. + +WARMING THE STORE +----------------- + +The store is filled by running ordinary commands with writing enabled. +Turn writing on for a single invocation with the `GIT_DIFF_HUNKS_WRITE` +environment variable, or persistently with the `diffHunks.write` +configuration; the environment variable takes precedence. A repository +owner warms the store by running the diff-producing commands they care +about with writing on, for example: + + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null + +A `--stat` walk records one entry per blob pair; +linkgit:git-blame[1] replays the coordinates and the summary formats +sum the counts, so a single warming walk serves both. +A warming run seeds from the existing store and rewrites the file +with the newly computed pairs merged in, so a later run adds to what +earlier runs recorded rather than discarding it. + +A walk records only the pairs it diffs. `git log --all --stat` diffs +each commit against its first parent, so a blame that follows a +merge's second parent computes those pairs itself: blame coverage is +partial on history with merges. Warming with a walk that also diffs +the other parents, for example `git log --all -m --stat`, raises +blame coverage at the cost of a larger store and a longer warming +run. + +COMMANDS +-------- + +`verify`:: + Check the integrity of the store: the trailing hash checksum, the + chunk table of contents, the sort order of the index, and the + bounds of every entry. Exits with non-zero status if the store is + corrupt. An absent store is valid. + +`clear`:: + Remove the store file. + +CORRECTNESS +----------- + +A stored result is interchangeable with a freshly computed one because an +entry is keyed by the inputs that determine the diff: + +* the object IDs of the old and new blob, so a result is used only for + the exact contents it was computed from; and +* the diff algorithm and ignore flags (`xdl_opts`) the hunks were + computed under. A lookup whose `xdl_opts` differ from a stored entry + misses. This is why, for example, `blame -w` and + `--diff-algorithm=` (including a per-path + `diff..algorithm`) do not reuse entries recorded under the + default settings: they change `xdl_opts`. + +The context length is not part of the key because only trim-stable +pairs are recorded: pairs whose zero-context trimmed diff and untrimmed +diff are identical, so one entry answers blame (zero context) and the +summary formats (any context) alike. The rare pair where +the zero-context trimming optimization picks a different but +equally valid set of hunks is +never recorded and is always computed. + +Some options shape the hunks in ways the key does not express, so a +diff that uses them is excluded from the store in both directions: +break detection (`-B`), `--ignore-matching-lines` (`-I`), and +`--anchored`. `--ignore-blank-lines` is different: it is an ignore +flag and therefore part of the key, but the summary formats exclude +it anyway, because it coalesces hunks differently between the code +path that emits text and the one that replays coordinates, so a +served answer would not match a store-less run. +linkgit:git-blame[1] additionally does not +consult the store for reverse blame, ignored revisions, or paths with a +textconv driver. + +The store carries a trailing hash checksum, but readers do not +re-checksum it on every load. As with the commit-graph and +multi-pack-index, the writer fsyncs the file (honoring `core.fsync`) and +commits it atomically, so a committed store is intact; every offset and +count is still bounds-checked as it is read. The checksum is verified by +`git diff-hunks verify`, not on the read path, so structural corruption +that fails a bounds check is read as an absent entry, while a record +that stays within bounds but whose bytes were altered is served until +`verify` detects the mismatch. + +CONFIGURATION +------------- + +`core.diffHunks`:: + Whether commands read the store. Defaults to true. See + linkgit:git-config[1]. + +`diffHunks.write`:: + Whether diff-producing commands write to the store. Defaults to + false. The `GIT_DIFF_HUNKS_WRITE` environment variable overrides it + for a single invocation. See linkgit:git-config[1]. + +Writing the store honors the `core.fsync` configuration through the +`diff-hunks` component; see linkgit:git-config[1]. + +SEE ALSO +-------- +linkgit:git-blame[1], +linkgit:git-log[1], +linkgit:gitformat-diff-hunks[5] + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/gitformat-diff-hunks.adoc b/Documentation/gitformat-diff-hunks.adoc new file mode 100644 index 00000000000000..f75ab73dd94a75 --- /dev/null +++ b/Documentation/gitformat-diff-hunks.adoc @@ -0,0 +1,129 @@ +gitformat-diff-hunks(5) +======================= + +NAME +---- +gitformat-diff-hunks - Precomputed diff hunk store format + +SYNOPSIS +-------- +[verse] +$GIT_DIR/objects/info/diff-hunks + +DESCRIPTION +----------- + +The diff-hunks store memoizes diff hunk coordinates so that commands +that need them, such as `git log --stat` and linkgit:git-blame[1], can +skip running the diff algorithm (and, for blame, loading the blob +content; the summary formats still test each pair for binariness, +which can load the blobs). See +linkgit:git-diff-hunks[1] for how the store is filled and managed and the +configuration that controls it. + +The store is a single file, `$GIT_DIR/objects/info/diff-hunks`, written +in one pass and replaced atomically, so a reader sees either the old +file or the complete new one. + +Entries are keyed by the object IDs of the blob pair that was diffed +and by the diff algorithm and ignore flags (`xdl_opts`) the pair was +diffed under. A blob pair fully determines the diff input, so an entry +is valid regardless of which commits, branches, or index states the +pair was encountered in, and identical diffs performed in different +contexts share one entry. A reader whose `xdl_opts` differ from an +entry does not match it and falls back to computing the diff. + +FILE FORMAT +----------- + +All multi-byte integers are stored in network byte order. The file is an +8-byte header, the chunk table of contents and chunk data described in +linkgit:gitformat-chunk[5], and a trailing checksum. + +HEADER +~~~~~~ + +- 4-byte signature: `DHPF` (diff-hunks precomputed format) +- 1-byte version number: currently 1 +- 1-byte hash version: 1 for SHA-1, 2 for SHA-256. A store whose hash + function differs from the repository's is ignored. +- 1-byte number of chunks +- 1-byte reserved + +CHUNK LOOKUP +~~~~~~~~~~~~ + +A table of contents in the format of linkgit:gitformat-chunk[5], listing +the offset of each chunk. Both chunks below are required; a file missing +either is treated as corrupt. + +CHUNK DATA +~~~~~~~~~~ + +DHIX (index):: + A sorted sequence of fixed-size entries. Each entry is the old + blob object ID, the new blob object ID, a 4-byte `xdl_opts` + value, and a 4-byte offset into the DHDT chunk. Entries are + sorted by old object ID, then new object ID, then `xdl_opts`, + so lookups can use binary search on the full key. + +DHDT (hunk data):: + For each index entry, at its offset: a 4-byte hunk count followed + by that many 16-byte hunk records. A hunk record is four 4-byte + values: old start, old count, new start, new count. + Starts are 0-based line numbers in the old and new blob; counts + are numbers of lines. The hunk count is at least 1: a record with + no hunks would claim the blob pair equivalent, which the store + never records, so readers treat such a record as invalid. + Identical hunk blocks are stored once: + distinct index entries whose recorded hunks are byte-for-byte + equal point at the same offset. + +TRAILER +~~~~~~~ + +A checksum of all preceding bytes, computed with the repository hash +function. + +CORRECTNESS +----------- + +Serving hunks from a valid store produces the same output as recomputing +the diff. The diff of a blob pair is not unique: a zero context length +triggers xdiff's common-tail trimming, which can pick a different but +equally valid set of hunks than an untrimmed diff does. A pair is +therefore recorded only when its trimmed and untrimmed diffs are +identical, which is the common case. Such an entry answers any consumer +at any context: git-blame replays its coordinates directly (it diffs at +zero context), and diffstat sums its per-hunk line counts, which the +context length does not change. The rare pair whose two diffs differ is +never recorded, so every consumer computes it. + +A store that cannot be used is ignored, and the consumer falls back to +computing the diff. Every offset and count read from the file is +bounds-checked, so a store that is missing, truncated, of an unknown +version, or of a different object hash does not change the diff output +and does not produce a diagnostic; `git diff-hunks verify` is what +reports corruption. + +The store is not re-checksummed on the read path. The writer fsyncs the +file (honoring `core.fsync`) and commits it atomically, so a +committed store is intact, the same trust model the commit-graph and +multi-pack-index use. The trailing checksum is recomputed by +`git diff-hunks verify` to detect corruption. + +The checksum detects corruption but does not prove who wrote the file. A +reader trusts the coordinates in a store that passes its checks, so +anything able to write a checksum-valid file at the store path can +influence output, the same as it could by writing objects directly. + +LIMITATIONS +----------- + +- Hunk counts, offsets, and line coordinates are 32-bit, capping the + hunk data at 4 GiB and a single entry at roughly 268 million hunks. + A result whose coordinates cannot be represented is not recorded. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/meson.build b/Documentation/meson.build index f4854f802d455f..85f37da47e9cac 100644 --- a/Documentation/meson.build +++ b/Documentation/meson.build @@ -41,6 +41,7 @@ manpages = { 'git-describe.adoc' : 1, 'git-diagnose.adoc' : 1, 'git-diff-files.adoc' : 1, + 'git-diff-hunks.adoc' : 1, 'git-diff-index.adoc' : 1, 'git-diff-pairs.adoc' : 1, 'git-difftool.adoc' : 1, @@ -175,6 +176,7 @@ manpages = { 'gitformat-bundle.adoc' : 5, 'gitformat-chunk.adoc' : 5, 'gitformat-commit-graph.adoc' : 5, + 'gitformat-diff-hunks.adoc' : 5, 'gitformat-index.adoc' : 5, 'gitformat-loose.adoc' : 5, 'gitformat-pack.adoc' : 5, diff --git a/Makefile b/Makefile index 50c96807d67e55..11a06934b3d350 100644 --- a/Makefile +++ b/Makefile @@ -1159,6 +1159,7 @@ LIB_OBJS += diffcore-order.o LIB_OBJS += diffcore-pickaxe.o LIB_OBJS += diffcore-rename.o LIB_OBJS += diffcore-rotate.o +LIB_OBJS += diff-hunks.o LIB_OBJS += dir-iterator.o LIB_OBJS += dir.o LIB_OBJS += editor.o @@ -1421,6 +1422,7 @@ BUILTIN_OBJS += builtin/credential.o BUILTIN_OBJS += builtin/describe.o BUILTIN_OBJS += builtin/diagnose.o BUILTIN_OBJS += builtin/diff-files.o +BUILTIN_OBJS += builtin/diff-hunks.o BUILTIN_OBJS += builtin/diff-index.o BUILTIN_OBJS += builtin/diff-pairs.o BUILTIN_OBJS += builtin/diff-tree.o diff --git a/builtin.h b/builtin.h index 4e47a4ebd30ba3..7e64da9f433836 100644 --- a/builtin.h +++ b/builtin.h @@ -175,6 +175,7 @@ int cmd_credential_store(int argc, const char **argv, const char *prefix, struct int cmd_describe(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diagnose(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff_files(int argc, const char **argv, const char *prefix, struct repository *repo); +int cmd_diff_hunks(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff_index(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff_pairs(int argc, const char **argv, const char *prefix, struct repository *repo); diff --git a/builtin/diff-hunks.c b/builtin/diff-hunks.c new file mode 100644 index 00000000000000..3aea2dad56aa4d --- /dev/null +++ b/builtin/diff-hunks.c @@ -0,0 +1,53 @@ +#include "builtin.h" +#include "config.h" +#include "diff-hunks.h" +#include "gettext.h" +#include "parse-options.h" +#include "repository.h" + +static const char * const diff_hunks_usage[] = { + N_("git diff-hunks verify"), + N_("git diff-hunks clear"), + NULL +}; + +static int cmd_diff_hunks_verify(int argc, const char **argv, + const char *prefix UNUSED, + struct repository *r) +{ + struct option options[] = { OPT_END() }; + + argc = parse_options(argc, argv, NULL, options, diff_hunks_usage, 0); + if (argc) + usage_with_options(diff_hunks_usage, options); + return diff_hunks_verify(r) ? 1 : 0; +} + +static int cmd_diff_hunks_clear(int argc, const char **argv, + const char *prefix UNUSED, + struct repository *r) +{ + struct option options[] = { OPT_END() }; + + argc = parse_options(argc, argv, NULL, options, diff_hunks_usage, 0); + if (argc) + usage_with_options(diff_hunks_usage, options); + return diff_hunks_clear(r) ? 1 : 0; +} + +int cmd_diff_hunks(int argc, const char **argv, const char *prefix, + struct repository *repo) +{ + parse_opt_subcommand_fn *fn = NULL; + struct option options[] = { + OPT_SUBCOMMAND("verify", &fn, cmd_diff_hunks_verify), + OPT_SUBCOMMAND("clear", &fn, cmd_diff_hunks_clear), + OPT_END() + }; + + repo_config(repo, git_default_config, NULL); + + argc = parse_options(argc, argv, prefix, options, diff_hunks_usage, 0); + + return fn(argc, argv, prefix, repo); +} diff --git a/chunk-format.c b/chunk-format.c index 51b5a2c959a1cf..34ab2750f7f14a 100644 --- a/chunk-format.c +++ b/chunk-format.c @@ -101,12 +101,14 @@ int write_chunkfile(struct chunkfile *cf, void *data) return result; } -int read_table_of_contents(struct chunkfile *cf, - const unsigned char *mfile, - size_t mfile_size, - uint64_t toc_offset, - int toc_length, - unsigned expected_alignment) +static int read_table_of_contents_1(struct chunkfile *cf, + const unsigned char *mfile, + size_t mfile_size, + uint64_t toc_offset, + int toc_length, + unsigned expected_alignment, + const struct git_hash_algo *algo, + int quiet) { int i; uint32_t chunk_id; @@ -121,12 +123,14 @@ int read_table_of_contents(struct chunkfile *cf, chunk_offset = get_be64(table_of_contents + 4); if (!chunk_id) { - error(_("terminating chunk id appears earlier than expected")); + if (!quiet) + error(_("terminating chunk id appears earlier than expected")); return 1; } if (chunk_offset % expected_alignment != 0) { - error(_("chunk id %"PRIx32" not %d-byte aligned"), - chunk_id, expected_alignment); + if (!quiet) + error(_("chunk id %"PRIx32" not %d-byte aligned"), + chunk_id, expected_alignment); return 1; } @@ -134,16 +138,18 @@ int read_table_of_contents(struct chunkfile *cf, next_chunk_offset = get_be64(table_of_contents + 4); if (next_chunk_offset < chunk_offset || - next_chunk_offset > mfile_size - the_hash_algo->rawsz) { - error(_("improper chunk offset(s) %"PRIx64" and %"PRIx64""), - chunk_offset, next_chunk_offset); + next_chunk_offset > mfile_size - algo->rawsz) { + if (!quiet) + error(_("improper chunk offset(s) %"PRIx64" and %"PRIx64""), + chunk_offset, next_chunk_offset); return -1; } for (i = 0; i < cf->chunks_nr; i++) { if (cf->chunks[i].id == chunk_id) { - error(_("duplicate chunk ID %"PRIx32" found"), - chunk_id); + if (!quiet) + error(_("duplicate chunk ID %"PRIx32" found"), + chunk_id); return -1; } } @@ -156,13 +162,39 @@ int read_table_of_contents(struct chunkfile *cf, chunk_id = get_be32(table_of_contents); if (chunk_id) { - error(_("final chunk has non-zero id %"PRIx32""), chunk_id); + if (!quiet) + error(_("final chunk has non-zero id %"PRIx32""), chunk_id); return -1; } return 0; } +int read_table_of_contents(struct chunkfile *cf, + const unsigned char *mfile, + size_t mfile_size, + uint64_t toc_offset, + int toc_length, + unsigned expected_alignment) +{ + return read_table_of_contents_1(cf, mfile, mfile_size, toc_offset, + toc_length, expected_alignment, + the_hash_algo, 0); +} + +int read_table_of_contents_quiet(struct chunkfile *cf, + const unsigned char *mfile, + size_t mfile_size, + uint64_t toc_offset, + int toc_length, + unsigned expected_alignment, + const struct git_hash_algo *algo) +{ + return read_table_of_contents_1(cf, mfile, mfile_size, toc_offset, + toc_length, expected_alignment, + algo, 1); +} + struct pair_chunk_data { const unsigned char **p; size_t *size; diff --git a/chunk-format.h b/chunk-format.h index 212a0a6af18356..bc31302ed09f1b 100644 --- a/chunk-format.h +++ b/chunk-format.h @@ -39,6 +39,20 @@ int read_table_of_contents(struct chunkfile *cf, int toc_length, unsigned expected_alignment); +/* + * Like read_table_of_contents(), for a reader that treats a malformed + * table as an absent file rather than reporting it: nothing is printed + * on failure, and the trailing-checksum bound is computed with the + * given hash algorithm instead of the_hash_algo. + */ +int read_table_of_contents_quiet(struct chunkfile *cf, + const unsigned char *mfile, + size_t mfile_size, + uint64_t toc_offset, + int toc_length, + unsigned expected_alignment, + const struct git_hash_algo *algo); + #define CHUNK_NOT_FOUND (-2) /* diff --git a/command-list.txt b/command-list.txt index 21b802c42026b3..e7b241e6ad8498 100644 --- a/command-list.txt +++ b/command-list.txt @@ -95,6 +95,7 @@ git-describe mainporcelain git-diagnose ancillaryinterrogators git-diff mainporcelain info git-diff-files plumbinginterrogators +git-diff-hunks plumbingmanipulators git-diff-index plumbinginterrogators git-diff-pairs plumbinginterrogators git-diff-tree plumbinginterrogators @@ -223,6 +224,7 @@ gitfaq guide gitformat-bundle developerinterfaces gitformat-chunk developerinterfaces gitformat-commit-graph developerinterfaces +gitformat-diff-hunks developerinterfaces gitformat-index developerinterfaces gitformat-pack developerinterfaces gitformat-signature developerinterfaces diff --git a/diff-hunks.c b/diff-hunks.c new file mode 100644 index 00000000000000..eeaaa2466a1a0c --- /dev/null +++ b/diff-hunks.c @@ -0,0 +1,916 @@ +/* + * Precomputed diff hunks, keyed by diff input. + * + * A single store at .git/objects/info/diff-hunks maps an (old blob, + * new blob, xdl_opts) key to the hunk coordinates of diffing the pair. + * The key determines the diff result (only trim-stable pairs are + * recorded; see diff-hunks.h), so an entry is valid in any context it + * recurs in, independent of path. Reading is on by default + * (core.diffHunks); writing is off by default and enabled per run or + * by configuration (see diff_hunks_write_enabled), so an ordinary + * command populates the store only during a warming run the + * repository owner opts into. + * + * File layout: + * Header: "DHPF"(4) + version(1) + hash_version(1) + * + num_chunks(1) + reserved(1) + * Table of contents (chunk-format) + * DHIX chunk: sorted entries, each + * old_blob_oid, new_blob_oid, xdl_opts(4), hdat_offset(4) + * DHDT chunk: per entry, num_hunks(4) followed by that many 16-byte hunks + * Trailing hash checksum + */ +#include "git-compat-util.h" +#include "chunk-format.h" +#include "config.h" +#include "csum-file.h" +#include "diff-hunks.h" +#include "diff-provider-internal.h" +#include "gettext.h" +#include "hash.h" +#include "hashmap.h" +#include "lockfile.h" +#include "odb.h" +#include "path.h" +#include "repo-settings.h" +#include "repository.h" +#include "strbuf.h" +#include "wrapper.h" + +#define DIFF_HUNKS_SIGNATURE 0x44485046 /* "DHPF" */ +/* + * Bump when the on-disk format changes, or when xdiff's emitted hunk + * coordinates change for a fixed (blobs, xdl_opts) key: an old store + * would otherwise serve stale hunks and change command output. + */ +#define DIFF_HUNKS_VERSION 1 +#define DIFF_HUNKS_HEADER_SIZE 8 + +#define DIFF_HUNKS_CHUNKID_INDEX 0x44484958 /* "DHIX" */ +#define DIFF_HUNKS_CHUNKID_DATA 0x44484454 /* "DHDT" */ + +/* + * Each hunk is 16 bytes on disk: + * old_start(4) old_count(4) new_start(4) new_count(4) + */ +#define DIFF_HUNKS_HUNK_SIZE (4 * sizeof(uint32_t)) + +/* + * Result of a store lookup: num_hunks records encoded in the store's mmap, + * valid until the store is freed. Read them with nth_precomputed_hunk(). + */ +struct precomputed_entry { + uint32_t num_hunks; + const unsigned char *hunk_data; +}; + +/* Decode a single hunk from the raw on-disk format. */ +static inline void decode_precomputed_hunk(const unsigned char *data, + struct precomputed_hunk *h) +{ + h->old_start = get_be32(data); + h->old_count = get_be32(data + 4); + h->new_start = get_be32(data + 8); + h->new_count = get_be32(data + 12); +} + +/* Decode the nth hunk of a lookup result into *h. */ +static inline void nth_precomputed_hunk(const struct precomputed_entry *e, + uint32_t n, struct precomputed_hunk *h) +{ + decode_precomputed_hunk(e->hunk_data + (size_t)n * DIFF_HUNKS_HUNK_SIZE, h); +} + +/* Byte length of the (old_oid, new_oid, xdl_opts) lookup key. */ +static size_t store_index_key_size(const struct git_hash_algo *algo) +{ + return 2 * algo->rawsz + sizeof(uint32_t); +} + +/* Index entry: the lookup key followed by the 4-byte offset into DHDT. */ +static size_t store_index_entry_size(const struct git_hash_algo *algo) +{ + return store_index_key_size(algo) + sizeof(uint32_t); +} + +/* + * The smallest a valid store file can be: the header, a table of contents + * with one entry per chunk plus a terminating entry, and the trailing + * checksum. + */ +static size_t store_min_size(const struct git_hash_algo *algo, + uint8_t num_chunks) +{ + size_t toc_size = (num_chunks + 1) * CHUNK_TOC_ENTRY_SIZE; + + return DIFF_HUNKS_HEADER_SIZE + toc_size + algo->rawsz; +} + +/* + * Decode an index entry's key into pointers to the two oids and the + * xdl_opts value (on-disk: old_oid, new_oid, then xdl_opts as a + * big-endian uint32). + */ +static void decode_store_index_key(const unsigned char *entry, unsigned int rawsz, + const unsigned char **old_hash, + const unsigned char **new_hash, + uint32_t *xdl_opts) +{ + *old_hash = entry; + *new_hash = entry + rawsz; + *xdl_opts = get_be32(entry + 2 * rawsz); +} + +/* The DHDT offset stored in an index entry, in the field after its key. */ +static uint32_t index_entry_hdat_offset(const unsigned char *entry, size_t keysz) +{ + return get_be32(entry + keysz); +} + +static char *diff_hunks_store_path(struct repository *r) +{ + return xstrfmt("%s/info/diff-hunks", repo_get_object_directory(r)); +} + +struct diff_hunks_store { + const unsigned char *data; + size_t data_len; + const struct git_hash_algo *hash_algo; + const unsigned char *index; + uint32_t num_entries; + const unsigned char *hdat; + size_t hdat_size; +}; + +static void free_store(struct diff_hunks_store *s) +{ + if (!s) + return; + if (s->data) + munmap((void *)s->data, s->data_len); + free(s); +} + +/* + * Open, mmap, and parse the store at fname. Returns the parsed store + * or NULL on any error. The diff output is unaffected either way; + * corruption is reported by verify, not treated as fatal here. + */ +static struct diff_hunks_store *load_store_at( + const struct git_hash_algo *repo_algo, const char *fname) +{ + struct diff_hunks_store *s; + struct chunkfile *cf; + int fd; + struct stat st; + void *data; + const unsigned char *p; + uint8_t num_chunks; + size_t index_size, entry_size, data_len; + + fd = git_open(fname); + if (fd < 0) + return NULL; + if (fstat(fd, &st) || st.st_size < DIFF_HUNKS_HEADER_SIZE) { + close(fd); + return NULL; + } + data_len = xsize_t(st.st_size); + data = xmmap(NULL, data_len, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + p = data; + + num_chunks = p[6]; + + /* + * Reject a file that is not a readable store: wrong signature, + * version, or object hash, or too small to hold the table of + * contents that read_table_of_contents() walks (it dereferences + * each entry before range-checking its offset). + */ + if (get_be32(p) != DIFF_HUNKS_SIGNATURE || + p[4] != DIFF_HUNKS_VERSION || + p[5] != oid_version(repo_algo) || + data_len < store_min_size(repo_algo, num_chunks)) { + munmap(data, data_len); + return NULL; + } + + /* + * The trailing checksum is not verified here: the writer fsyncs + * and commits atomically, so a committed file is intact, and + * every record is bounds-checked at read (see precomputed_entry_at). + * The checksum is checked separately, by diff_hunks_verify(). + */ + + CALLOC_ARRAY(s, 1); + s->data = data; + s->data_len = data_len; + s->hash_algo = repo_algo; + + cf = init_chunkfile(NULL); + if (read_table_of_contents_quiet(cf, p, data_len, + DIFF_HUNKS_HEADER_SIZE, num_chunks, 1, + repo_algo) || + pair_chunk(cf, DIFF_HUNKS_CHUNKID_INDEX, &s->index, &index_size) || + pair_chunk(cf, DIFF_HUNKS_CHUNKID_DATA, &s->hdat, &s->hdat_size)) { + free_chunkfile(cf); + goto corrupt; + } + free_chunkfile(cf); + + entry_size = store_index_entry_size(s->hash_algo); + if (index_size % entry_size) + goto corrupt; + s->num_entries = index_size / entry_size; + return s; + +corrupt: + free_store(s); + return NULL; +} + +static struct diff_hunks_store *diff_hunks_store_load(struct repository *r) +{ + struct diff_hunks_store *s; + char *fname; + + prepare_repo_settings(r); + if (!r->settings.core_diff_hunks) + return NULL; + + fname = diff_hunks_store_path(r); + s = load_store_at(r->hash_algo, fname); + free(fname); + return s; +} + +struct diff_hunks_store *repo_diff_hunks_store(struct repository *r) +{ + if (!r->objects) + return NULL; + if (r->objects->diff_hunks_store_attempted) + return r->objects->diff_hunks_store; + r->objects->diff_hunks_store_attempted = 1; + r->objects->diff_hunks_store = diff_hunks_store_load(r); + return r->objects->diff_hunks_store; +} + +void close_diff_hunks_store(struct object_database *o) +{ + if (!o->diff_hunks_store) + return; + free_store(o->diff_hunks_store); + o->diff_hunks_store = NULL; +} + +/* + * Fill *out with the hunk record at offset in the data chunk, and return + * 1 if the record is in bounds, 0 otherwise. The read path does not + * re-verify the checksum, and a valid checksum would not bound the count + * anyway, so a read must call this and use *out only when it returns + * non-zero. + * + * A record is a be32 hunk count followed by that many DIFF_HUNKS_HUNK_SIZE + * hunks. "remaining" tracks the bytes from offset to the end of the data + * chunk: it must hold the count, and after the count is consumed it must + * hold every hunk. The bounds are written as subtraction and division + * (never addition or multiplication) so a crafted offset or count cannot + * overflow them. + */ +static int precomputed_entry_at(const struct diff_hunks_store *s, + uint32_t offset, struct precomputed_entry *out) +{ + size_t remaining; + uint32_t num_hunks; + + if (offset >= s->hdat_size) + return 0; + remaining = s->hdat_size - offset; + if (remaining < sizeof(uint32_t)) + return 0; + + num_hunks = get_be32(s->hdat + offset); + remaining -= sizeof(uint32_t); + if (num_hunks > remaining / DIFF_HUNKS_HUNK_SIZE) + return 0; + + out->num_hunks = num_hunks; + out->hunk_data = s->hdat + offset + sizeof(uint32_t); + return 1; +} + +struct lookup_key { + const struct object_id *old_oid; + const struct object_id *new_oid; + int xdl_opts; + unsigned int rawsz; +}; + +/* + * The store's total order over (old_oid, new_oid, xdl_opts), defined + * once so the write-side sort (writer_entry_cmp) and the read-side + * search (store_bsearch_cmp) order the keys identically. + */ +static int cmp_store_index_key(const unsigned char *old_a, const unsigned char *new_a, + uint32_t opts_a, + const unsigned char *old_b, const unsigned char *new_b, + uint32_t opts_b, unsigned int rawsz) +{ + int cmp = memcmp(old_a, old_b, rawsz); + if (!cmp) + cmp = memcmp(new_a, new_b, rawsz); + if (!cmp) + cmp = (opts_a > opts_b) - (opts_a < opts_b); + return cmp; +} + +static int store_bsearch_cmp(const void *key, const void *entry_ptr) +{ + const struct lookup_key *k = key; + const unsigned char *old_hash, *new_hash; + uint32_t xdl_opts; + + decode_store_index_key(entry_ptr, k->rawsz, &old_hash, &new_hash, + &xdl_opts); + return cmp_store_index_key(k->old_oid->hash, k->new_oid->hash, + (uint32_t)k->xdl_opts, + old_hash, new_hash, xdl_opts, k->rawsz); +} + +static int store_get_one(struct diff_hunks_store *s, const struct lookup_key *key, + struct precomputed_entry *out) +{ + size_t entry_size = store_index_entry_size(s->hash_algo); + const unsigned char *found; + + found = bsearch(key, s->index, s->num_entries, entry_size, + store_bsearch_cmp); + if (!found) + return 0; + return precomputed_entry_at(s, + index_entry_hdat_offset(found, store_index_key_size(s->hash_algo)), + out); +} + +static int diff_hunks_store_get(struct diff_hunks_store *s, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + struct precomputed_entry *out) +{ + struct lookup_key key; + + if (!s) + return 0; + /* The null OID names no blob and cannot key an entry. */ + if (is_null_oid(old_oid) || is_null_oid(new_oid)) + return 0; + + key.old_oid = old_oid; + key.new_oid = new_oid; + key.xdl_opts = xdl_opts; + key.rawsz = s->hash_algo->rawsz; + + return store_get_one(s, &key, out); +} + +/* + * A recorded hunk sequence must satisfy the provider interface's + * shared check (diff_provider_check_hunk()) before it may be replayed: + * coordinates decode from be32 into long, which is 32-bit on some + * platforms, so a crafted value can decode negative or out of order. + * An entry that fails reads as a miss, so the caller recomputes. + */ +static int replayable_hunks(const struct precomputed_entry *e) +{ + struct diff_provider_hunks_check c = { 0 }; + uint32_t i; + + /* + * Replaying a record with no hunks would assert the blob pair + * equivalent, a claim the store must never make (the writer + * refuses to record one), so such a record is invalid. + */ + if (!e->num_hunks) + return 0; + for (i = 0; i < e->num_hunks; i++) { + struct precomputed_hunk h; + nth_precomputed_hunk(e, i, &h); + if (diff_provider_check_hunk(&c, h.old_start, h.old_count, + h.new_start, h.new_count)) + return 0; + } + return 1; +} + +int diff_hunks_replay(struct diff_hunks_store *s, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + xdl_emit_hunk_consume_func_t hunk_func, void *cb_data) +{ + struct precomputed_entry e; + uint32_t i; + + if (!diff_hunks_store_get(s, old_oid, new_oid, xdl_opts, &e) || + !replayable_hunks(&e)) + return 0; + for (i = 0; i < e.num_hunks; i++) { + struct precomputed_hunk h; + nth_precomputed_hunk(&e, i, &h); + hunk_func(h.old_start, h.old_count, + h.new_start, h.new_count, cb_data); + } + return 1; +} + +/* Validate one store file. Returns 0 if valid or absent, -1 on any error. */ +static int verify_store_at(struct repository *r, const char *fname) +{ + struct diff_hunks_store *s; + size_t entry_size; + uint32_t i; + int fd; + int ret = 0; + + /* + * A file that cannot be opened is not evidence of corruption: + * report the open error, and reserve the corruption diagnostics + * below for a file that was read and failed to parse. + */ + fd = git_open(fname); + if (fd < 0) { + if (errno == ENOENT) + return 0; /* absent is valid */ + return error_errno(_("unable to open diff-hunks store %s"), + fname); + } + close(fd); + s = load_store_at(r->hash_algo, fname); + if (!s) + return error(_("diff-hunks store failed to load (corrupt " + "header or hash mismatch): %s"), fname); + if (!hashfile_checksum_valid(r->hash_algo, s->data, s->data_len)) { + error(_("diff-hunks store has incorrect checksum and is " + "likely corrupt: %s"), fname); + free_store(s); + return -1; + } + + entry_size = store_index_entry_size(s->hash_algo); + for (i = 0; i < s->num_entries; i++) { + const unsigned char *ep = s->index + st_mult(entry_size, i); + size_t keysz = store_index_key_size(s->hash_algo); + uint32_t offset = index_entry_hdat_offset(ep, keysz); + struct precomputed_entry pe; + + /* + * Keyed by (old_oid, new_oid, xdl_opts), increasing. memcmp + * matches cmp_store_index_key's integer comparison of + * xdl_opts because it is non-negative, so its big-endian + * bytes order the same as its value. + */ + if (i > 0 && memcmp(ep - entry_size, ep, keysz) >= 0) { + error(_("diff-hunks entry %u not in sorted order"), i); + ret = -1; + } + if (!precomputed_entry_at(s, offset, &pe)) { + error(_("diff-hunks entry %u has out-of-bounds hunk " + "data"), i); + ret = -1; + } else if (!replayable_hunks(&pe)) { + error(_("diff-hunks entry %u holds an invalid hunk " + "sequence"), i); + ret = -1; + } + } + + free_store(s); + return ret; +} + +int diff_hunks_verify(struct repository *r) +{ + char *fname = diff_hunks_store_path(r); + int ret = 0; + + if (verify_store_at(r, fname)) + ret = -1; + free(fname); + return ret; +} + +int diff_hunks_clear(struct repository *r) +{ + char *fname = diff_hunks_store_path(r); + int ret = 0; + + if (unlink(fname) && errno != ENOENT) + ret = error_errno(_("unable to remove %s"), fname); + free(fname); + return ret; +} + +struct writer_entry { + struct object_id old_oid; + struct object_id new_oid; + int xdl_opts; + uint32_t hdat_offset; +}; + +struct diff_hunks_writer { + struct repository *r; + struct writer_entry *entries; + size_t nr, alloc; + size_t seed_nr; /* nr after seeding; finish skips a no-op flush */ + unsigned force_flush : 1; /* seed pruned: rewrite even a no-op warm */ + struct strbuf hdat; + struct hashmap dedup; /* hunk block content -> offset in hdat */ +}; + +/* A record of one distinct hunk block already present in hdat. */ +struct dedup_entry { + struct hashmap_entry ent; + uint32_t offset; + uint32_t len; +}; + +static int dedup_cmp(const void *cmp_data, + const struct hashmap_entry *a, + const struct hashmap_entry *b, + const void *keydata UNUSED) +{ + const struct diff_hunks_writer *writer = cmp_data; + const struct dedup_entry *ea = container_of(a, const struct dedup_entry, ent); + const struct dedup_entry *eb = container_of(b, const struct dedup_entry, ent); + + if (ea->len != eb->len) + return 1; + return memcmp(writer->hdat.buf + ea->offset, + writer->hdat.buf + eb->offset, ea->len); +} + +static struct diff_hunks_writer *diff_hunks_writer_new(struct repository *r) +{ + struct diff_hunks_writer *w; + + CALLOC_ARRAY(w, 1); + w->r = r; + strbuf_init(&w->hdat, 0); + hashmap_init(&w->dedup, dedup_cmp, w, 0); + return w; +} + +static void strbuf_put_be32(struct strbuf *sb, uint32_t val) +{ + unsigned char buf[4]; + put_be32(buf, val); + strbuf_add(sb, buf, 4); +} + +/* + * The hunk block just appended at `start` is deduplicated: if an + * identical block is already in hdat, this copy is dropped and the + * earlier offset returned; otherwise it is kept and remembered. + * Distinct keys that diff to the same hunks then share one block. + */ +static uint32_t intern_block(struct diff_hunks_writer *w, size_t start) +{ + size_t len = w->hdat.len - start; + struct dedup_entry key, *found, *added; + + hashmap_entry_init(&key.ent, memhash(w->hdat.buf + start, len)); + key.offset = (uint32_t)start; + key.len = (uint32_t)len; + + found = hashmap_get_entry(&w->dedup, &key, ent, NULL); + if (found) { + strbuf_setlen(&w->hdat, start); + return found->offset; + } + + added = xmalloc(sizeof(*added)); + hashmap_entry_init(&added->ent, key.ent.hash); + added->offset = key.offset; + added->len = key.len; + hashmap_add(&w->dedup, &added->ent); + return key.offset; +} + +int diff_hunks_writer_add(struct diff_hunks_writer *w, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + const struct precomputed_hunk *hunks, + size_t nr_hunks) +{ + struct writer_entry *e; + size_t i, block_start; + + if (!w) + return 0; + /* + * The block appended for this entry is sizeof(uint32_t) + + * nr_hunks * DIFF_HUNKS_HUNK_SIZE bytes. Bound nr_hunks so that + * length fits the uint32_t the dedup index records (and so the + * count itself fits the uint32_t written to the store). + */ + if (!nr_hunks || + nr_hunks > (UINT32_MAX - sizeof(uint32_t)) / DIFF_HUNKS_HUNK_SIZE || + is_null_oid(old_oid) || is_null_oid(new_oid)) + return 0; + if (w->hdat.len > UINT32_MAX) + return 0; + /* + * Coordinates are stored as 32-bit values; a result that cannot + * round-trip is dropped rather than silently truncated. + */ + for (i = 0; i < nr_hunks; i++) + if ((uintmax_t)hunks[i].old_start > (uintmax_t)INT32_MAX || + (uintmax_t)hunks[i].old_count > (uintmax_t)INT32_MAX || + (uintmax_t)hunks[i].new_start > (uintmax_t)INT32_MAX || + (uintmax_t)hunks[i].new_count > (uintmax_t)INT32_MAX) + return 0; + + ALLOC_GROW(w->entries, w->nr + 1, w->alloc); + e = &w->entries[w->nr++]; + oidcpy(&e->old_oid, old_oid); + oidcpy(&e->new_oid, new_oid); + e->xdl_opts = xdl_opts; + + block_start = w->hdat.len; + strbuf_put_be32(&w->hdat, (uint32_t)nr_hunks); + for (i = 0; i < nr_hunks; i++) { + strbuf_put_be32(&w->hdat, hunks[i].old_start); + strbuf_put_be32(&w->hdat, hunks[i].old_count); + strbuf_put_be32(&w->hdat, hunks[i].new_start); + strbuf_put_be32(&w->hdat, hunks[i].new_count); + } + e->hdat_offset = intern_block(w, block_start); + return 1; +} + +/* + * Seed the writer with fname's entries so a rewrite preserves them, + * setting *pruned when the rewrite will not carry the whole file + * forward: the file failed its checksum and was discarded outright, or + * individual entries were dropped because they failed the replayable + * check or the writer refused them (a key naming no blob). A + * rewrite re-checksums, so corruption must not be carried forward: + * that would launder it into a checksum-valid file that verify can no + * longer catch. This path already reads the whole file, so verify the + * checksum here (the reader keeps trusting committed files, without + * re-checksumming); an invalid + * entry reads as a miss anyway, so dropping it heals the store rather + * than losing anything a reader could use. + */ +static void diff_hunks_writer_seed(struct diff_hunks_writer *w, + const char *fname, int *pruned) +{ + struct diff_hunks_store *s = load_store_at(w->r->hash_algo, fname); + unsigned int rawsz; + size_t entry_size, keysz; + struct precomputed_hunk *hunks = NULL; + size_t hunks_alloc = 0; + uint32_t i, dropped = 0; + + if (!s) + return; + if (!hashfile_checksum_valid(w->r->hash_algo, s->data, s->data_len)) { + warning(_("diff-hunks store %s failed its checksum; " + "discarding it"), fname); + free_store(s); + *pruned = 1; + return; + } + rawsz = s->hash_algo->rawsz; + entry_size = store_index_entry_size(s->hash_algo); + keysz = store_index_key_size(s->hash_algo); + + for (i = 0; i < s->num_entries; i++) { + const unsigned char *ep = s->index + st_mult(entry_size, i); + const unsigned char *old_hash, *new_hash; + struct object_id old_oid, new_oid; + uint32_t xdl_opts, j; + struct precomputed_entry pe; + + decode_store_index_key(ep, rawsz, &old_hash, &new_hash, + &xdl_opts); + oidread(&old_oid, old_hash, s->hash_algo); + oidread(&new_oid, new_hash, s->hash_algo); + if (!precomputed_entry_at(s, index_entry_hdat_offset(ep, keysz), &pe) || + !replayable_hunks(&pe)) { + dropped++; + continue; + } + ALLOC_GROW(hunks, pe.num_hunks, hunks_alloc); + for (j = 0; j < pe.num_hunks; j++) + nth_precomputed_hunk(&pe, j, &hunks[j]); + if (!diff_hunks_writer_add(w, &old_oid, &new_oid, + (int)xdl_opts, hunks, pe.num_hunks)) + dropped++; + } + if (dropped) { + warning(Q_("diff-hunks store %s: dropping %u invalid entry", + "diff-hunks store %s: dropping %u invalid entries", + dropped), fname, dropped); + *pruned = 1; + } + free(hunks); + free_store(s); +} + +/* + * Writing is off by default. It is enabled per invocation by the + * GIT_DIFF_HUNKS_WRITE environment variable, or persistently by the + * diffHunks.write config, with the environment variable winning when + * set. Only a warming run (a diff or log the repository owner chooses + * to run with writing on) enables it, so ordinary reads never mutate + * the store. + */ +static int diff_hunks_write_enabled(struct repository *r) +{ + const char *env = getenv("GIT_DIFF_HUNKS_WRITE"); + int val; + + if (env) { + /* + * This is a warming opt-in, so an unparseable value must not + * abort an ordinary read command: treat it as disabled. + */ + val = git_parse_maybe_bool(env); + return val < 0 ? 0 : val; + } + if (!repo_config_get_bool(r, "diffhunks.write", &val)) + return val; + return 0; +} + +struct diff_hunks_writer *diff_hunks_writer_maybe_new(struct repository *r) +{ + struct diff_hunks_writer *w; + char *fname; + int pruned; + + if (!diff_hunks_write_enabled(r)) + return NULL; + /* + * Seed from the existing store so a flush merges with it rather + * than replacing it: a later warm adds newly computed pairs + * without discarding what earlier warms recorded. + */ + w = diff_hunks_writer_new(r); + fname = diff_hunks_store_path(r); + pruned = 0; + diff_hunks_writer_seed(w, fname, &pruned); + free(fname); + w->seed_nr = w->nr; + /* + * A pruning seed means the file on disk holds material the + * rewrite must not preserve; flush even if this warm computes + * nothing new, so the store on disk is repaired rather than + * left serving what the seed refused. + */ + w->force_flush = !!pruned; + return w; +} + +static int writer_entry_cmp(const void *va, const void *vb, void *ctx) +{ + const struct writer_entry *a = va, *b = vb; + unsigned int rawsz = *(const unsigned int *)ctx; + return cmp_store_index_key(a->old_oid.hash, a->new_oid.hash, + (uint32_t)a->xdl_opts, + b->old_oid.hash, b->new_oid.hash, + (uint32_t)b->xdl_opts, + rawsz); +} + +struct write_ctx { + struct diff_hunks_writer *w; + unsigned int rawsz; +}; + +static int write_index_chunk(struct hashfile *f, void *data) +{ + struct write_ctx *ctx = data; + size_t i; + + for (i = 0; i < ctx->w->nr; i++) { + hashwrite(f, ctx->w->entries[i].old_oid.hash, ctx->rawsz); + hashwrite(f, ctx->w->entries[i].new_oid.hash, ctx->rawsz); + hashwrite_be32(f, ctx->w->entries[i].xdl_opts); + hashwrite_be32(f, ctx->w->entries[i].hdat_offset); + } + return 0; +} + +static int write_data_chunk(struct hashfile *f, void *data) +{ + struct write_ctx *ctx = data; + hashwrite(f, ctx->w->hdat.buf, ctx->w->hdat.len); + return 0; +} + +/* Sort, dedup, and write the accumulated entries to the file at fname. */ +static int diff_hunks_writer_flush(struct diff_hunks_writer *w, char *fname) +{ + struct lock_file lk = LOCK_INIT; + struct hashfile *f; + struct chunkfile *cf; + unsigned int rawsz = w->r->hash_algo->rawsz; + struct write_ctx ctx = { w, rawsz }; + size_t entry_size; + + QSORT_S(w->entries, w->nr, writer_entry_cmp, &rawsz); + + /* + * The same blob pair recurs across history (reverts, cherry- + * picks); identical keys carry identical hunks, so keep one of + * each. The index must stay duplicate-free for binary search. + */ + if (w->nr > 1) { + size_t kept = 1, i; + for (i = 1; i < w->nr; i++) + if (writer_entry_cmp(&w->entries[kept - 1], + &w->entries[i], &rawsz)) + w->entries[kept++] = w->entries[i]; + w->nr = kept; + } + + if (safe_create_leading_directories(w->r, fname)) { + error(_("unable to create directory for %s"), fname); + return -1; + } + if (hold_lock_file_for_update(&lk, fname, 0) < 0) { + error_errno(_("unable to lock %s"), fname); + return -1; + } + adjust_shared_perm(w->r, get_lock_file_path(&lk)); + f = hashfd(w->r->hash_algo, get_lock_file_fd(&lk), + get_lock_file_path(&lk)); + + entry_size = store_index_entry_size(w->r->hash_algo); + cf = init_chunkfile(f); + add_chunk(cf, DIFF_HUNKS_CHUNKID_INDEX, w->nr * entry_size, + write_index_chunk); + add_chunk(cf, DIFF_HUNKS_CHUNKID_DATA, w->hdat.len, write_data_chunk); + + hashwrite_be32(f, DIFF_HUNKS_SIGNATURE); + hashwrite_u8(f, DIFF_HUNKS_VERSION); + hashwrite_u8(f, oid_version(w->r->hash_algo)); + hashwrite_u8(f, get_num_chunks(cf)); + hashwrite_u8(f, 0); /* reserved */ + + write_chunkfile(cf, &ctx); + free_chunkfile(cf); + + /* + * fsync per the user's configuration (like commit-graph and the + * multi-pack-index), then commit atomically. Readers trust the + * committed file rather than re-checksumming it; diff_hunks_verify() + * checks the checksum separately. + */ + finalize_hashfile(f, NULL, FSYNC_COMPONENT_DIFF_HUNKS, + CSUM_HASH_IN_STREAM | CSUM_FSYNC); + /* + * This same process may hold the current store mmapped (a warm + * that also reads); the commit below renames over it, which must + * never land on a live mapping (Windows refuses it). Close the + * store and clear the load-attempted flag first, so the next + * read loads the committed file. + */ + if (w->r->objects) { + close_diff_hunks_store(w->r->objects); + w->r->objects->diff_hunks_store_attempted = 0; + } + if (commit_lock_file(&lk)) { + error_errno(_("unable to write %s"), fname); + return -1; + } + return 0; +} + +static void diff_hunks_writer_free(struct diff_hunks_writer *w) +{ + if (!w) + return; + hashmap_clear_and_free(&w->dedup, struct dedup_entry, ent); + free(w->entries); + strbuf_release(&w->hdat); + free(w); +} + +void diff_hunks_writer_finish(struct diff_hunks_writer *w) +{ + if (!w) + return; + /* Skip the flush when the warm recorded nothing beyond its seed. */ + if (w->nr != w->seed_nr || w->force_flush) { + char *fname = diff_hunks_store_path(w->r); + diff_hunks_writer_flush(w, fname); + free(fname); + } + diff_hunks_writer_free(w); +} diff --git a/diff-hunks.h b/diff-hunks.h new file mode 100644 index 00000000000000..ef9ee3f417bcae --- /dev/null +++ b/diff-hunks.h @@ -0,0 +1,117 @@ +#ifndef DIFF_HUNKS_H +#define DIFF_HUNKS_H + +#include "hash.h" +#include "xdiff-interface.h" /* xdl_emit_hunk_consume_func_t */ + +struct object_id; +struct repository; +struct object_database; + +/* + * A persistent store of precomputed diff hunk coordinates, at + * .git/objects/info/diff-hunks. Entries are keyed by the two blobs diffed + * and the xdl_opts they were diffed under, so a cached result is valid + * in any context that key recurs in, independent of path. The xdl_opts + * key component mirrors the (always non-negative) diff_options field it + * projects from, and is serialized and compared as a 4-byte big-endian + * integer. + * + * The hunks a pair produces are not unique. They vary with the xdiff + * algorithm and ignore flags (xdl_opts, part of the key), and with + * whether the diff was trimmed: a zero-context diff runs + * trim_common_tail, which can pick a different but equally valid set of + * hunks than an untrimmed diff. The store holds one entry per key, so a + * pair is recorded only when its trimmed and untrimmed diffs are + * identical (the recording caller checks); such an entry serves a + * consumer at any context. The rare pair where the two diffs differ is + * never recorded and is always computed. + * + * The store is a cache: ordinary commands read it and fall back to + * computing the diff when it is absent, stale, or corrupt. It is filled + * as a side effect of diff and log runs, but only when writing is + * enabled (such a write-enabled run is a warming run); writing is off + * by default, so an ordinary command reads the store without recording + * into it. + */ + +/* + * A hunk's coordinates. The type is long to match the xdiff emit + * callback; the values are a diff's line numbers and counts, always + * within the int32 range the on-disk format stores (see + * diff_hunks_writer_add()). + */ +struct precomputed_hunk { + long old_start; + long old_count; + long new_start; + long new_count; +}; + +/* + * The repository's store, loaded once on first use and cached on the + * object database. Returns NULL when reading is disabled + * (core.diffHunks=false), the store is absent, or it fails to parse + * (wrong signature, version, or object hash, or a corrupt structure). + * The lookup functions below accept a NULL store and treat it as + * empty (every lookup misses), so callers need not check for NULL. + * The object database owns the store; callers must not free it. + */ +struct diff_hunks_store *repo_diff_hunks_store(struct repository *r); + +/* Free the repository's cached store, at object-database teardown. */ +void close_diff_hunks_store(struct object_database *o); + +/* + * Replay the recorded hunks of an (old blob, new blob) pair diffed + * under xdl_opts through hunk_func. The sequence is validated before + * any callback runs: on a hit (return 1) every hunk is emitted, on a + * miss (return 0: absent pair, xdl_opts mismatch, or an entry that + * fails validation) nothing is emitted, so a caller may accumulate + * directly into its result. + */ +int diff_hunks_replay(struct diff_hunks_store *s, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + xdl_emit_hunk_consume_func_t hunk_func, void *cb_data); + +/* + * A warming run's writer: it accumulates the hunks it computes in memory + * and flushes them to the store in one pass at finish. + */ +struct diff_hunks_writer; + +/* + * Return a writer for a warming run, or NULL when writing is disabled + * (the default). diff_hunks_writer_add() tolerates a NULL writer, so a + * caller may attach the result unconditionally. Pair with + * diff_hunks_writer_finish(). + */ +struct diff_hunks_writer *diff_hunks_writer_maybe_new(struct repository *r); + +/* + * Record a blob pair's hunks as computed under xdl_opts; a later lookup + * with a matching key is served these hunks. The caller must have + * checked that the pair's trimmed and untrimmed diffs are identical + * (see the top of this file), so the entry answers at any context. + * NULL-safe. Returns 1 when the entry was recorded, 0 when the writer + * refused it (no hunks, a null object id, or values the on-disk + * 32-bit fields cannot hold). + */ +int diff_hunks_writer_add(struct diff_hunks_writer *w, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + const struct precomputed_hunk *hunks, + size_t nr_hunks); + +/* Flush the accumulated entries to the store and free the writer. NULL-safe. */ +void diff_hunks_writer_finish(struct diff_hunks_writer *w); + +/* Remove the store file. Returns 0 (incl. absent) or -1. */ +int diff_hunks_clear(struct repository *r); +/* Validate the store. Returns 0 if valid/absent, -1 if corrupt. */ +int diff_hunks_verify(struct repository *r); + +#endif /* DIFF_HUNKS_H */ diff --git a/environment.c b/environment.c index c663113e8a6dcc..a0e6d0b9b35ad9 100644 --- a/environment.c +++ b/environment.c @@ -239,6 +239,7 @@ static const struct fsync_component_name { { "pack", FSYNC_COMPONENT_PACK }, { "pack-metadata", FSYNC_COMPONENT_PACK_METADATA }, { "commit-graph", FSYNC_COMPONENT_COMMIT_GRAPH }, + { "diff-hunks", FSYNC_COMPONENT_DIFF_HUNKS }, { "index", FSYNC_COMPONENT_INDEX }, { "objects", FSYNC_COMPONENTS_OBJECTS }, { "reference", FSYNC_COMPONENT_REFERENCE }, diff --git a/git.c b/git.c index e5f1811b6bb762..cb149e4b4e96ec 100644 --- a/git.c +++ b/git.c @@ -566,6 +566,7 @@ static struct cmd_struct commands[] = { { "diagnose", cmd_diagnose, RUN_SETUP_GENTLY }, { "diff", cmd_diff, NO_PARSEOPT }, { "diff-files", cmd_diff_files, RUN_SETUP | NEED_WORK_TREE | NO_PARSEOPT }, + { "diff-hunks", cmd_diff_hunks, RUN_SETUP }, { "diff-index", cmd_diff_index, RUN_SETUP | NO_PARSEOPT }, { "diff-pairs", cmd_diff_pairs, RUN_SETUP | NO_PARSEOPT }, { "diff-tree", cmd_diff_tree, RUN_SETUP | NO_PARSEOPT }, diff --git a/meson.build b/meson.build index 539a50f90e597b..391d9da93c0cf8 100644 --- a/meson.build +++ b/meson.build @@ -364,6 +364,7 @@ libgit_sources = [ 'diffcore-pickaxe.c', 'diffcore-rename.c', 'diffcore-rotate.c', + 'diff-hunks.c', 'dir-iterator.c', 'dir.c', 'editor.c', @@ -633,6 +634,7 @@ builtin_sources = [ 'builtin/describe.c', 'builtin/diagnose.c', 'builtin/diff-files.c', + 'builtin/diff-hunks.c', 'builtin/diff-index.c', 'builtin/diff-pairs.c', 'builtin/diff-tree.c', diff --git a/odb.c b/odb.c index cf6e7938c01e56..b300cd522d4d68 100644 --- a/odb.c +++ b/odb.c @@ -2,6 +2,7 @@ #include "abspath.h" #include "commit-graph.h" #include "config.h" +#include "diff-hunks.h" #include "dir.h" #include "environment.h" #include "gettext.h" @@ -1033,6 +1034,7 @@ void odb_close(struct object_database *o) for (source = o->sources; source; source = source->next) odb_source_close(source); close_commit_graph(o); + close_diff_hunks_store(o); } static void odb_free_sources(struct object_database *o) diff --git a/odb.h b/odb.h index 7995bed97bc36e..949d55668f82e7 100644 --- a/odb.h +++ b/odb.h @@ -8,6 +8,7 @@ #include "thread-utils.h" struct cached_object_entry; +struct diff_hunks_store; struct list_objects_filter_options; struct odb_source_inmemory; struct packed_git; @@ -76,6 +77,9 @@ struct object_database { struct commit_graph *commit_graph; unsigned commit_graph_attempted : 1; /* if loading has been attempted */ + struct diff_hunks_store *diff_hunks_store; + unsigned diff_hunks_store_attempted : 1; /* if loading has been attempted */ + /* * This is meant to hold a *small* number of objects that you would * want odb_read_object() to be able to return, but yet you do not want diff --git a/repo-settings.c b/repo-settings.c index f3be3b8c5a3d09..c3015356ba81e6 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -77,6 +77,7 @@ void prepare_repo_settings(struct repository *r) repo_cfg_bool(r, "pack.usesparse", &r->settings.pack_use_sparse, 1); repo_cfg_bool(r, "pack.usepathwalk", &r->settings.pack_use_path_walk, 0); repo_cfg_bool(r, "core.multipackindex", &r->settings.core_multi_pack_index, 1); + repo_cfg_bool(r, "core.diffhunks", &r->settings.core_diff_hunks, 1); repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 0); repo_cfg_bool(r, "index.skiphash", &r->settings.index_skip_hash, r->settings.index_skip_hash); repo_cfg_bool(r, "pack.readreverseindex", &r->settings.pack_read_reverse_index, 1); diff --git a/repo-settings.h b/repo-settings.h index e5253ead025c83..615a55cac49b1c 100644 --- a/repo-settings.h +++ b/repo-settings.h @@ -22,6 +22,7 @@ struct repo_settings { int core_commit_graph; int commit_graph_generation_version; int commit_graph_changed_paths_version; + int core_diff_hunks; int gc_write_commit_graph; int fetch_write_commit_graph; int command_requires_full_index; diff --git a/write-or-die.h b/write-or-die.h index ff0408bd849fd8..35ed324307a21b 100644 --- a/write-or-die.h +++ b/write-or-die.h @@ -22,13 +22,15 @@ enum fsync_component { FSYNC_COMPONENT_INDEX = 1 << 4, FSYNC_COMPONENT_REFERENCE = 1 << 5, FSYNC_COMPONENT_OBJECT_MAP = 1 << 6, + FSYNC_COMPONENT_DIFF_HUNKS = 1 << 7, }; #define FSYNC_COMPONENTS_OBJECTS (FSYNC_COMPONENT_LOOSE_OBJECT | \ FSYNC_COMPONENT_PACK) #define FSYNC_COMPONENTS_DERIVED_METADATA (FSYNC_COMPONENT_PACK_METADATA | \ - FSYNC_COMPONENT_COMMIT_GRAPH) + FSYNC_COMPONENT_COMMIT_GRAPH | \ + FSYNC_COMPONENT_DIFF_HUNKS) #define FSYNC_COMPONENTS_DEFAULT ((FSYNC_COMPONENTS_OBJECTS | \ FSYNC_COMPONENTS_DERIVED_METADATA) & \ @@ -46,7 +48,8 @@ enum fsync_component { FSYNC_COMPONENT_COMMIT_GRAPH | \ FSYNC_COMPONENT_INDEX | \ FSYNC_COMPONENT_REFERENCE | \ - FSYNC_COMPONENT_OBJECT_MAP) + FSYNC_COMPONENT_OBJECT_MAP | \ + FSYNC_COMPONENT_DIFF_HUNKS) #ifndef FSYNC_COMPONENTS_PLATFORM_DEFAULT #define FSYNC_COMPONENTS_PLATFORM_DEFAULT FSYNC_COMPONENTS_DEFAULT From 55b052a198f43984e2913788f013f6b7b56c1a54 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 1 Aug 2026 10:41:47 -0700 Subject: [PATCH 039/259] diff: record precomputed hunks during stat output The diff-hunks store has a writer, but nothing fills it. Teach builtin_diffstat() to do so: on a warming run (a writer is attached), a modified pair's stat is produced by collecting the pair's hunk coordinates instead of emitting text, the counts are summed from those hunks, and the pair is recorded. A run without a writer is unchanged, and nothing reads the store yet; the read side arrives next. The store records one context-free entry per pair, and only for a trim-stable pair: one whose zero-context trimmed diff (what blame will read) and untrimmed diff (whose counts a nonzero-context stat matches) are identical. The warming path computes both and hands them to diff_hunks_writer_record_stable(), new here, which records only when they agree; a divergent pair is never recorded and every consumer computes it. The warming run displays the counts it shows a store-less run: the trimmed ones, since xdi_diff trims at zero context, while the untrimmed counts serve only the stability comparison. Not everything the stat path computes may be recorded. --ignore-blank-lines is part of the key, but it coalesces hunks differently between the text-emitting and coordinate-callback paths, so a recorded entry would not match a store-less run's --stat. -I patterns, --anchored, and break detection (-B) shape the diff outside the key entirely; the guard for those three sits in this consumer for now and moves into the store's own provider when it registers, next. A "log -L" range-scoped stat is not the whole-pair diff the key describes, so it does not record. Recording also requires both sides to be valid regular files whose blobs the key can name: a working-tree side, textconv output, or a gitlink has no usable id. "git diff", "git log", "git show", and "git diff-tree" with the --stat, --numstat, and --shortstat formats attach a writer when writing is enabled and flush it when the traversal finishes, so a warming run such as GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null fills the cache as a side effect of the diff work the command already does. Writing is controlled by diffHunks.write and GIT_DIFF_HUNKS_WRITE. Add the write half of t4220: - ordinary commands never create the store, and creation is gated off by default, the environment overriding the config; - a warming run builds a store that verifies, and a second refreshes it in place; - a warming run displays parity at zero context on a trim-divergent pair, committed as a fixture (small synthetic pairs cannot diverge: minimal diffs add and delete equal counts, and trimming preserves that); - binary and mode-only pairs do not break the writer; - a corrupt store is discarded at seed; - verify and clear run against the files a warming run builds. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- builtin/diff-tree.c | 3 + builtin/diff.c | 10 ++ builtin/log.c | 7 + diff-hunks.c | 32 ++++ diff-hunks.h | 18 ++- diff.c | 218 +++++++++++++++++++++---- diff.h | 17 ++ t/meson.build | 1 + t/t4220-diff-hunks.sh | 184 +++++++++++++++++++++ t/t4220/README | 55 +++++++ t/t4220/trim-divergent-new | 319 +++++++++++++++++++++++++++++++++++++ t/t4220/trim-divergent-old | 316 ++++++++++++++++++++++++++++++++++++ 12 files changed, 1150 insertions(+), 30 deletions(-) create mode 100755 t/t4220-diff-hunks.sh create mode 100644 t/t4220/README create mode 100644 t/t4220/trim-divergent-new create mode 100644 t/t4220/trim-divergent-old diff --git a/builtin/diff-tree.c b/builtin/diff-tree.c index 8b8f8b54e40664..296c6a137e23c6 100644 --- a/builtin/diff-tree.c +++ b/builtin/diff-tree.c @@ -170,6 +170,8 @@ int cmd_diff_tree(int argc, opt->diffopt.rotate_to_strict = 1; + diff_hunks_attach(&opt->diffopt); + /* * NOTE! We expect "a..b" to expand to "^a b" but it is * perfectly valid for revision range parser to yield "b ^a", @@ -234,5 +236,6 @@ int cmd_diff_tree(int argc, diff_free(&opt->diffopt); } + diff_hunks_detach(&opt->diffopt); return diff_result_code(opt); } diff --git a/builtin/diff.c b/builtin/diff.c index 18b1083e984a35..a2ad63ac8c4b3e 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -568,6 +568,15 @@ int cmd_diff(int argc, } } + /* + * The hunk store is keyed by blob pair, so any diff whose + * file pairs carry known blob object IDs (tree-to-tree, + * index-to-tree) can consult the same entries that + * "git log --stat" and "git blame" use; pairs without known + * blobs bypass it at lookup time. + */ + diff_hunks_attach(&rev.diffopt); + symdiff_prepare(&rev, &sdiff); for (i = 0; i < rev.pending.nr; i++) { struct object_array_entry *entry = &rev.pending.objects[i]; @@ -648,6 +657,7 @@ int cmd_diff(int argc, result = diff_result_code(&rev); if (1 < rev.diffopt.skip_stat_unmatch) refresh_index_quietly(); + diff_hunks_detach(&rev.diffopt); release_revisions(&rev); object_array_clear(&ent); symdiff_release(&sdiff); diff --git a/builtin/log.c b/builtin/log.c index 350b35c556362d..6903bdd5fff055 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -693,8 +693,11 @@ int cmd_show(int argc, opt.tweak = show_setup_revisions_tweak; cmd_log_init(argc, argv, prefix, &rev, &opt, &cfg); + diff_hunks_attach(&rev.diffopt); + if (!rev.no_walk) { ret = cmd_log_walk(&rev); + diff_hunks_detach(&rev.diffopt); release_revisions(&rev); log_config_release(&cfg); return ret; @@ -767,6 +770,7 @@ int cmd_show(int argc, } rev.diffopt.no_free = 0; + diff_hunks_detach(&rev.diffopt); diff_free(&rev.diffopt); release_revisions(&rev); log_config_release(&cfg); @@ -846,8 +850,11 @@ int cmd_log(int argc, opt.tweak = log_setup_revisions_tweak; cmd_log_init(argc, argv, prefix, &rev, &opt, &cfg); + diff_hunks_attach(&rev.diffopt); + ret = cmd_log_walk(&rev); + diff_hunks_detach(&rev.diffopt); release_revisions(&rev); log_config_release(&cfg); return ret; diff --git a/diff-hunks.c b/diff-hunks.c index eeaaa2466a1a0c..23cefe055b2071 100644 --- a/diff-hunks.c +++ b/diff-hunks.c @@ -651,6 +651,38 @@ int diff_hunks_writer_add(struct diff_hunks_writer *w, return 1; } +void diff_hunks_writer_record_stable(struct diff_hunks_writer *w, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + const struct precomputed_hunk *trimmed, + size_t nr_trimmed, + const struct precomputed_hunk *full, + size_t nr_full) +{ + size_t i; + + if (!w) + return; + /* + * Record only a trim-stable pair, one whose trimmed and + * untrimmed diffs are identical, so the single entry answers + * any consumer at any context (see the top of this file). A + * pair where the two diffs differ is never recorded and every + * consumer computes it. + */ + if (nr_trimmed != nr_full) + return; + for (i = 0; i < nr_trimmed; i++) + if (trimmed[i].old_start != full[i].old_start || + trimmed[i].old_count != full[i].old_count || + trimmed[i].new_start != full[i].new_start || + trimmed[i].new_count != full[i].new_count) + return; + diff_hunks_writer_add(w, old_oid, new_oid, xdl_opts, + trimmed, nr_trimmed); +} + /* * Seed the writer with fname's entries so a rewrite preserves them, * setting *pruned when the rewrite will not carry the whole file diff --git a/diff-hunks.h b/diff-hunks.h index ef9ee3f417bcae..89e56f4b1bcd32 100644 --- a/diff-hunks.h +++ b/diff-hunks.h @@ -94,7 +94,8 @@ struct diff_hunks_writer *diff_hunks_writer_maybe_new(struct repository *r); * Record a blob pair's hunks as computed under xdl_opts; a later lookup * with a matching key is served these hunks. The caller must have * checked that the pair's trimmed and untrimmed diffs are identical - * (see the top of this file), so the entry answers at any context. + * (see the top of this file), so the entry answers at any context; + * diff_hunks_writer_record_stable() below performs that check. * NULL-safe. Returns 1 when the entry was recorded, 0 when the writer * refused it (no hunks, a null object id, or values the on-disk * 32-bit fields cannot hold). @@ -106,6 +107,21 @@ int diff_hunks_writer_add(struct diff_hunks_writer *w, const struct precomputed_hunk *hunks, size_t nr_hunks); +/* + * Record the pair only if it is trim-stable: the recording caller + * hands over both the trimmed (xdi_diff) and untrimmed (xdl_diff) + * zero-context hunk sequences it computed, and the entry is added + * only when the two are identical. NULL-safe. + */ +void diff_hunks_writer_record_stable(struct diff_hunks_writer *w, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + const struct precomputed_hunk *trimmed, + size_t nr_trimmed, + const struct precomputed_hunk *full, + size_t nr_full); + /* Flush the accumulated entries to the store and free the writer. NULL-safe. */ void diff_hunks_writer_finish(struct diff_hunks_writer *w); diff --git a/diff.c b/diff.c index 9ef4328afad541..11ec88dc8cdeb7 100644 --- a/diff.c +++ b/diff.c @@ -16,6 +16,7 @@ #include "revision.h" #include "quote.h" #include "diff.h" +#include "diff-hunks.h" #include "diffcore.h" #include "delta.h" #include "hex.h" @@ -2929,6 +2930,72 @@ static struct diffstat_file *diffstat_add(struct diffstat_t *diffstat, return x; } +struct diffstat_hunk_cb_data { + struct precomputed_hunk **h; + size_t *nr, *alloc; +}; + +/* + * Hunk callback that appends each hunk's coordinates to a growable + * array, so one xdiff pass can both sum a diffstat and record hunks for + * the store. + */ +static int diffstat_hunk_cb(long start_a, long count_a, + long start_b, long count_b, + void *cb_data) +{ + struct diffstat_hunk_cb_data *d = cb_data; + + ALLOC_GROW(*d->h, *d->nr + 1, *d->alloc); + (*d->h)[*d->nr].old_start = start_a; + (*d->h)[*d->nr].old_count = count_a; + (*d->h)[*d->nr].new_start = start_b; + (*d->h)[*d->nr].new_count = count_b; + (*d->nr)++; + return 0; +} + +/* + * Collect the hunks of the two files at zero context. diff_fn chooses + * whether trimming runs: xdi_diff applies trim_common_tail, yielding the + * zero-context hunks blame reads; xdl_diff does not, yielding the + * untrimmed hunks. Both run at zero context, so the untrimmed hunks are + * not grouped the way a nonzero context would group them; diffstat only + * sums their counts, which grouping does not change. Sets *ph (caller + * frees) and *ph_nr. + */ +typedef int (*xdiff_fn)(mmfile_t *, mmfile_t *, xpparam_t const *, + xdemitconf_t const *, xdemitcb_t *); +static int collect_hunks(xdiff_fn diff_fn, mmfile_t *mf1, mmfile_t *mf2, + xpparam_t *xpp, struct precomputed_hunk **ph, + size_t *ph_nr) +{ + size_t ph_alloc = 0; + xdemitcb_t ecb = { 0 }; + xdemitconf_t xecfg = { 0 }; + struct diffstat_hunk_cb_data cd = { ph, ph_nr, &ph_alloc }; + + *ph = NULL; + *ph_nr = 0; + xecfg.hunk_func = diffstat_hunk_cb; + ecb.priv = &cd; + return diff_fn(mf1, mf2, xpp, &xecfg, &ecb); +} + +void diff_hunks_attach(struct diff_options *o) +{ + if (!(o->output_format & + (DIFF_FORMAT_DIFFSTAT | DIFF_FORMAT_SHORTSTAT | DIFF_FORMAT_NUMSTAT))) + return; + o->hunks_writer = diff_hunks_writer_maybe_new(o->repo); +} + +void diff_hunks_detach(struct diff_options *o) +{ + diff_hunks_writer_finish(o->hunks_writer); + o->hunks_writer = NULL; +} + static int diffstat_consume(void *priv, char *line, unsigned long len) { struct diffstat_t *diffstat = priv; @@ -4253,6 +4320,87 @@ static const char *get_compact_summary(const struct diff_filepair *p, int is_ren return NULL; } +/* + * Fill data->added/deleted for a modified pair by collecting its hunk + * coordinates, and record them into the store. Runs only on a warming + * run; returns 1 when it produced the counts, 0 when the caller must + * compute the diffstat itself. + * + * --ignore-blank-lines is excluded: that flag is part of the store + * key, but it coalesces hunks differently between the emit and + * hunk-callback paths, so a recorded entry would not match a + * store-less run's --stat output. (--inter-hunk-context is not + * excluded: it only groups hunks, and diffstat sums their counts, + * which grouping does not change.) Recording requires both sides to + * be valid regular files whose blobs the key can name. + */ +static int diffstat_from_hunks(struct diff_options *o, + struct diff_filespec *one, + struct diff_filespec *two, + struct diffstat_file *data) +{ + struct precomputed_hunk *ph_trim, *ph_full, *counts; + size_t n_trim, n_full, n_counts, k; + mmfile_t mf1, mf2; + xpparam_t xpp = { .flags = o->xdl_opts, + .ignore_regex = o->ignore_regex, + .ignore_regex_nr = o->ignore_regex_nr, + .anchors = o->anchors, + .anchors_nr = o->anchors_nr }; + + if (o->xdl_opts & XDF_IGNORE_BLANK_LINES) + return 0; + + /* Not a warming run: the caller computes the diffstat. */ + if (!o->hunks_writer) + return 0; + /* + * -I patterns, --anchored anchors, and break detection (-B) + * shape the diff outside the store key, so what they compute + * must not be recorded under it. + */ + if (o->ignore_regex_nr || o->anchors_nr || o->break_opt != -1) + return 0; + /* Recording needs blobs the key can name, on both sides. */ + if (!one->oid_valid || !two->oid_valid || + S_ISGITLINK(one->mode) || S_ISGITLINK(two->mode) || + !DIFF_FILE_VALID(one) || !DIFF_FILE_VALID(two) || + !S_ISREG(one->mode) || !S_ISREG(two->mode)) + return 0; + + if (fill_mmfile(o->repo, &mf1, one) < 0 || + fill_mmfile(o->repo, &mf2, two) < 0) + die("unable to read files to diff"); + + /* + * Compute the zero-context trimmed diff (what blame reads) and the + * untrimmed diff (whose counts a nonzero-context stat matches). + * xdi_diff runs first: it enforces the size limit, so the xdl_diff + * call is already bounded. + */ + if (collect_hunks(xdi_diff, &mf1, &mf2, &xpp, &ph_trim, &n_trim) || + collect_hunks(xdl_diff, &mf1, &mf2, &xpp, &ph_full, &n_full)) + die("unable to generate diffstat for %s", one->path); + + /* + * Match a store-less run: at zero context xdi_diff trims, so sum the + * trimmed diff; otherwise sum the untrimmed one. + */ + counts = o->context ? ph_full : ph_trim; + n_counts = o->context ? n_full : n_trim; + for (k = 0; k < n_counts; k++) { + data->added += counts[k].new_count; + data->deleted += counts[k].old_count; + } + + diff_hunks_writer_record_stable(o->hunks_writer, &one->oid, &two->oid, + o->xdl_opts, ph_trim, n_trim, + ph_full, n_full); + free(ph_trim); + free(ph_full); + return 1; +} + static void builtin_diffstat(const char *name_a, const char *name_b, struct diff_filespec *one, struct diff_filespec *two, @@ -4304,38 +4452,50 @@ static void builtin_diffstat(const char *name_a, const char *name_b, } else if (may_differ) { - /* Crazy xdl interfaces.. */ - xpparam_t xpp; - xdemitconf_t xecfg; - - if (fill_mmfile(o->repo, &mf1, one) < 0 || - fill_mmfile(o->repo, &mf2, two) < 0) - die("unable to read files to diff"); - - memset(&xpp, 0, sizeof(xpp)); - memset(&xecfg, 0, sizeof(xecfg)); - xpp.flags = o->xdl_opts; - xpp.ignore_regex = o->ignore_regex; - xpp.ignore_regex_nr = o->ignore_regex_nr; - xpp.anchors = o->anchors; - xpp.anchors_nr = o->anchors_nr; - xecfg.ctxlen = o->context; - xecfg.interhunkctxlen = o->interhunkcontext; - xecfg.flags = XDL_EMIT_NO_HUNK_HDR; - - if (p->line_ranges) { - struct line_range_filter lr_filter; - - line_range_filter_init(&lr_filter, p->line_ranges, - diffstat_consume, diffstat); + /* + * Record into the diff-hunks store on a warming run. A + * "log -L" range-scoped stat is not the whole-pair diff + * the store keys, so it does not record. Otherwise diff + * normally. + */ + if (p->line_ranges || !diffstat_from_hunks(o, one, two, data)) { + /* Crazy xdl interfaces.. */ + xpparam_t xpp; + xdemitconf_t xecfg; + + if (fill_mmfile(o->repo, &mf1, one) < 0 || + fill_mmfile(o->repo, &mf2, two) < 0) + die("unable to read files to diff"); + + memset(&xpp, 0, sizeof(xpp)); + memset(&xecfg, 0, sizeof(xecfg)); + xpp.flags = o->xdl_opts; + xpp.ignore_regex = o->ignore_regex; + xpp.ignore_regex_nr = o->ignore_regex_nr; + xpp.anchors = o->anchors; + xpp.anchors_nr = o->anchors_nr; + xecfg.ctxlen = o->context; + xecfg.interhunkctxlen = o->interhunkcontext; + xecfg.flags = XDL_EMIT_NO_HUNK_HDR; - if (line_range_filter_diff(&lr_filter, &mf1, &mf2, - &xpp, &xecfg)) + if (p->line_ranges) { + struct line_range_filter lr_filter; + + line_range_filter_init(&lr_filter, + p->line_ranges, + diffstat_consume, + diffstat); + + if (line_range_filter_diff(&lr_filter, &mf1, + &mf2, &xpp, &xecfg)) + die("unable to generate diffstat for %s", + one->path); + } else if (xdi_diff_outf(&mf1, &mf2, NULL, + diffstat_consume, diffstat, + &xpp, &xecfg)) die("unable to generate diffstat for %s", one->path); - } else if (xdi_diff_outf(&mf1, &mf2, NULL, - diffstat_consume, diffstat, &xpp, &xecfg)) - die("unable to generate diffstat for %s", one->path); + } if (DIFF_FILE_VALID(one) && DIFF_FILE_VALID(two)) { struct diffstat_file *file = diff --git a/diff.h b/diff.h index bb5cddaf3499e9..3d44de39ff9056 100644 --- a/diff.h +++ b/diff.h @@ -420,6 +420,13 @@ struct diff_options { */ int max_depth; int max_depth_valid; + + /* + * Precomputed diff hunks (see diff-hunks.h). When hunks_writer is + * set (a warming run), diffstat records the hunks it computes; + * the writer is attached only for the stat output formats. + */ + struct diff_hunks_writer *hunks_writer; }; unsigned diff_filter_bit(char status); @@ -668,6 +675,16 @@ void diffcore_fix_diff_index(void); int diff_queue_is_empty(struct diff_options *o); void diff_flush(struct diff_options*); void diff_free(struct diff_options*); + +/* + * Attach a diff-hunks writer to a diff producing a stat format, so a + * warming run records the hunks it computes; a no-op when writing is off + * or for other formats. Pair with diff_hunks_detach() once the diff is + * done. + */ +void diff_hunks_attach(struct diff_options *o); +void diff_hunks_detach(struct diff_options *o); + void diff_warn_rename_limit(const char *varname, int needed, int degraded_cc); /* diff-raw status letters */ diff --git a/t/meson.build b/t/meson.build index 8ae6ab6c5fe1e2..c63c30ba631d03 100644 --- a/t/meson.build +++ b/t/meson.build @@ -583,6 +583,7 @@ integration_tests = [ 't4216-log-bloom.sh', 't4217-log-limit.sh', 't4219-log-follow-merge.sh', + 't4220-diff-hunks.sh', 't4252-am-options.sh', 't4253-am-keep-cr-dos.sh', 't4254-am-corrupt.sh', diff --git a/t/t4220-diff-hunks.sh b/t/t4220-diff-hunks.sh new file mode 100755 index 00000000000000..c6778319460354 --- /dev/null +++ b/t/t4220-diff-hunks.sh @@ -0,0 +1,184 @@ +#!/bin/sh + +test_description='precomputed diff hunks store (git diff-hunks) + +The store maps an (old blob, new blob, diff settings) key to the hunks of +diffing the pair. It is a cache: reading is on by default +(core.diffHunks), while writing is +off by default and enabled per run by GIT_DIFF_HUNKS_WRITE (or the +diffHunks.write config), so a diff or log warms the store only when the +owner opts in. These tests check that a warmed store never changes +output, that lookups honor the diff settings, and that a corrupt store is +read as absent while verify reports the corruption.' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +STORE=.git/objects/info/diff-hunks + +# Warm the store the way a repository owner would: a stat walk with +# writing enabled. A --stat walk records one entry per trim-stable blob +# pair, serving blame and the summary formats alike. Extra arguments +# (e.g. -c options) are passed to git before "log". +warm () { + GIT_DIFF_HUNKS_WRITE=1 git "$@" log --all --stat >/dev/null +} + +# Run a command with the store disabled, for ground truth. +no_store () { + git -c core.diffhunks=false "$@" +} + +test_expect_success 'setup' ' + test_commit initial file.txt "line 1" && + test_commit second file.txt "line 1 +line 2" && + test_commit third file.txt "line 1 +line 2 +line 3" && + test_commit fourth file.txt "changed line 1 +line 2 +line 3 +line 4" +' + +test_expect_success 'ordinary commands do not create the store' ' + git log --stat >/dev/null && + git blame file.txt >/dev/null && + git diff --stat second third >/dev/null && + test_path_is_missing $STORE +' + +test_expect_success 'writing is gated by env and config, env wins' ' + test_when_finished "git diff-hunks clear" && + # The diffHunks.write config enables writing. + git -c diffHunks.write=true log --all --stat >/dev/null && + test_path_is_file $STORE && + git diff-hunks clear && + # GIT_DIFF_HUNKS_WRITE overrides the config: 0 disables it. + GIT_DIFF_HUNKS_WRITE=0 git -c diffHunks.write=true log --all --stat >/dev/null && + test_path_is_missing $STORE && + # and enables it without any config. + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null && + test_path_is_file $STORE +' + +test_expect_success 'a warm builds a store that verifies' ' + warm && + test_path_is_file $STORE && + git diff-hunks verify +' + +test_expect_success 'a second warming run refreshes the store in place' ' + warm && + test_commit fifth file.txt "brand new line" && + warm && + git diff-hunks verify && + no_store log --stat >expect && + git log --stat >actual && + test_cmp expect actual +' + +# A warming run displays the diffstat it computes. At zero context xdi_diff +# trims, so the displayed counts must be the trimmed ones (what a store-less +# run shows), not the untrimmed ones the writer compares against when it +# decides whether the pair is stable enough to record. +test_expect_success 'warming --stat at zero context matches a store-less run' ' + git init -q warm-u0 && + ( + cd warm-u0 && + cp "$TEST_DIRECTORY/t4220/trim-divergent-old" div.sh && + git add div.sh && git commit -q -m old && + cp "$TEST_DIRECTORY/t4220/trim-divergent-new" div.sh && + git add div.sh && git commit -q -m new && + git -c core.diffhunks=false log -1 --format= -U0 --stat -- div.sh >expect && + GIT_DIFF_HUNKS_WRITE=1 git log -1 --format= -U0 --stat -- div.sh >got && + test_cmp expect got + ) +' + +test_expect_success 'show and diff-tree --stat use the store' ' + test_when_finished "git diff-hunks clear" && + # diff_hunks_attach() runs for show and diff-tree: a write-enabled + # --stat records into the store (without the attach there is no + # writer, so nothing is written). + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git show --stat fourth >/dev/null && + test_path_is_file "$STORE" && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git diff-tree --stat fourth >/dev/null && + test_path_is_file "$STORE" && + # Reading never changes their output. + git diff-hunks clear && + no_store show --stat fourth >expect_show && + no_store diff-tree --stat fourth >expect_dt && + warm && + git show --stat fourth >got_show && + git diff-tree --stat fourth >got_dt && + test_cmp expect_show got_show && + test_cmp expect_dt got_dt +' + +# Cover the pair shapes an object walk encounters: binary and +# mode-only changes produce no text hunks to record. +test_expect_success 'binary and mode-only changes do not break the writer' ' + printf "\\000\\001\\002" >bin.dat && + git add bin.dat && + git commit -m binary-1 && + printf "\\000\\001\\003\\004" >bin.dat && + git add bin.dat && + git commit -m binary-2 && + echo "mode content" >mode.txt && + git add mode.txt && + git commit -m mode-1 && + test_chmod +x mode.txt && + git commit -m mode-2 && + no_store log --stat >expect && + warm && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'verify succeeds on a valid store and on an absent one' ' + warm && + git diff-hunks verify && + git diff-hunks clear && + test_path_is_missing $STORE && + git diff-hunks verify +' + +test_expect_success 'verify detects a checksum mismatch' ' + test_when_finished "git diff-hunks clear" && + warm && + fsize=$(test_file_size $STORE) && + mid=$((fsize / 2)) && + printf "\\377" | dd of=$STORE bs=1 seek=$mid count=1 conv=notrunc 2>/dev/null && + test_must_fail git diff-hunks verify +' + +test_expect_success 'a warm discards a corrupt store rather than seeding from it' ' + test_when_finished "git diff-hunks clear" && + warm && + # Corrupt the checksum: the next warm must not carry the corrupt + # entries forward into a fresh checksum-valid file; it discards + # them (with a warning) and rewrites a store that verifies. + fsize=$(test_file_size $STORE) && + printf "\\377" | dd of=$STORE bs=1 seek=$((fsize / 2)) count=1 conv=notrunc 2>/dev/null && + warm 2>err && + test_grep "failed its checksum" err && + git diff-hunks verify && + no_store log --stat >expect && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'diff-hunks clear removes the store file' ' + warm && + test_path_is_file $STORE && + git diff-hunks clear && + test_path_is_missing $STORE +' + +test_done diff --git a/t/t4220/README b/t/t4220/README new file mode 100644 index 00000000000000..b850fe7c61bea0 --- /dev/null +++ b/t/t4220/README @@ -0,0 +1,55 @@ +t4220 diff-hunks test fixtures +============================== + +trim-divergent-old, trim-divergent-new +-------------------------------------- + +Two revisions of a single real file, used by t4220-diff-hunks.sh to +exercise a "trim-divergent" blob pair: one whose diff hunk counts change +with the amount of context, so the trimmed and untrimmed results +disagree. + +They are two versions of git's own t/t6002-rev-list-bisect.sh, taken from +git.git history around: + + 090af9957c ("t6002: fix use of `expr` with `set -e`", + Patrick Steinhardt, 2026-04-21) + +which rewrites `$(expr ...)` arithmetic as `$((...))` and reformats a few +test_expect_success blocks. + + trim-divergent-old = 090af9957c^:t/t6002-rev-list-bisect.sh (blob daa009c9a1) + trim-divergent-new = 090af9957c :t/t6002-rev-list-bisect.sh (blob f2de40b5ed) + +To regenerate them from any git.git checkout: + + git show 090af9957c^:t/t6002-rev-list-bisect.sh >trim-divergent-old + git show 090af9957c:t/t6002-rev-list-bisect.sh >trim-divergent-new + +Why this pair +------------- + +The diff-hunks store records only "trim-stable" pairs: those whose hunks +are identical whether or not xdiff trims the common head and tail (which +it does at zero context, in trim_common_tail). This pair is deliberately +NOT trim-stable: + + diff -U0 reports 9 added / 6 deleted + diff -U3 reports 10 added / 7 deleted + +Because the counts diverge with context, the writer must refuse to record +this pair and every command must recompute it from the blobs. t4220 uses +it to prove that the displayed counts stay correct at each context, and +that a divergent pair is never served from the store. See +t4220-diff-hunks.sh ("a trim-divergent file is correct at each context" +and the store-poison test). + +Why not a synthesized fixture +----------------------------- + +The divergence needs real content that makes xdiff's common-tail trimming +shift a hunk boundary while the added/deleted balance stays equal. A +minimal hand-written file that reliably triggers the -U0 vs -U3 count +disagreement has not been found yet; until one is, this real pair is kept +verbatim. If you synthesize a smaller equivalent, replace these two files +and delete this note. diff --git a/t/t4220/trim-divergent-new b/t/t4220/trim-divergent-new new file mode 100644 index 00000000000000..f2de40b5ed8f14 --- /dev/null +++ b/t/t4220/trim-divergent-new @@ -0,0 +1,319 @@ +#!/bin/sh +# +# Copyright (c) 2005 Jon Seymour +# +test_description='Tests git rev-list --bisect functionality' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-t6000.sh # t6xxx specific functions + +# usage: test_bisection max-diff bisect-option head ^prune... +# +# e.g. test_bisection 1 --bisect l1 ^l0 +# +test_bisection_diff() +{ + _max_diff=$1 + _bisect_option=$2 + shift 2 + _bisection=$(git rev-list $_bisect_option "$@") + _list_size=$(git rev-list "$@" | wc -l) + _head=$1 + shift 1 + _bisection_size=$(git rev-list $_bisection "$@" | wc -l) + [ -n "$_list_size" -a -n "$_bisection_size" ] || + error "test_bisection_diff failed" + + # Test if bisection size is close to half of list size within + # tolerance. + # + _bisect_err=$(($_list_size - $_bisection_size * 2)) + if test "$_bisect_err" -lt 0 + then + _bisect_err=$((0 - $_bisect_err)) + fi + _bisect_err=$(($_bisect_err / 2)) ; # floor + + test_expect_success "bisection diff $_bisect_option $_head $* <= $_max_diff" ' + test $_bisect_err -le $_max_diff + ' +} + +date >path0 +git update-index --add path0 +save_tag tree git write-tree +on_committer_date "00:00" hide_error save_tag root unique_commit root tree +on_committer_date "00:01" save_tag l0 unique_commit l0 tree -p root +on_committer_date "00:02" save_tag l1 unique_commit l1 tree -p l0 +on_committer_date "00:03" save_tag l2 unique_commit l2 tree -p l1 +on_committer_date "00:04" save_tag a0 unique_commit a0 tree -p l2 +on_committer_date "00:05" save_tag a1 unique_commit a1 tree -p a0 +on_committer_date "00:06" save_tag b1 unique_commit b1 tree -p a0 +on_committer_date "00:07" save_tag c1 unique_commit c1 tree -p b1 +on_committer_date "00:08" save_tag b2 unique_commit b2 tree -p b1 +on_committer_date "00:09" save_tag b3 unique_commit b2 tree -p b2 +on_committer_date "00:10" save_tag c2 unique_commit c2 tree -p c1 -p b2 +on_committer_date "00:11" save_tag c3 unique_commit c3 tree -p c2 +on_committer_date "00:12" save_tag a2 unique_commit a2 tree -p a1 +on_committer_date "00:13" save_tag a3 unique_commit a3 tree -p a2 +on_committer_date "00:14" save_tag b4 unique_commit b4 tree -p b3 -p a3 +on_committer_date "00:15" save_tag a4 unique_commit a4 tree -p a3 -p b4 -p c3 +on_committer_date "00:16" save_tag l3 unique_commit l3 tree -p a4 +on_committer_date "00:17" save_tag l4 unique_commit l4 tree -p l3 +on_committer_date "00:18" save_tag l5 unique_commit l5 tree -p l4 +git update-ref HEAD $(tag l5) + + +# E +# / \ +# e1 | +# | | +# e2 | +# | | +# e3 | +# | | +# e4 | +# | | +# | f1 +# | | +# | f2 +# | | +# | f3 +# | | +# | f4 +# | | +# e5 | +# | | +# e6 | +# | | +# e7 | +# | | +# e8 | +# \ / +# F + + +on_committer_date "00:00" hide_error save_tag F unique_commit F tree +on_committer_date "00:01" save_tag e8 unique_commit e8 tree -p F +on_committer_date "00:02" save_tag e7 unique_commit e7 tree -p e8 +on_committer_date "00:03" save_tag e6 unique_commit e6 tree -p e7 +on_committer_date "00:04" save_tag e5 unique_commit e5 tree -p e6 +on_committer_date "00:05" save_tag f4 unique_commit f4 tree -p F +on_committer_date "00:06" save_tag f3 unique_commit f3 tree -p f4 +on_committer_date "00:07" save_tag f2 unique_commit f2 tree -p f3 +on_committer_date "00:08" save_tag f1 unique_commit f1 tree -p f2 +on_committer_date "00:09" save_tag e4 unique_commit e4 tree -p e5 +on_committer_date "00:10" save_tag e3 unique_commit e3 tree -p e4 +on_committer_date "00:11" save_tag e2 unique_commit e2 tree -p e3 +on_committer_date "00:12" save_tag e1 unique_commit e1 tree -p e2 +on_committer_date "00:13" save_tag E unique_commit E tree -p e1 -p f1 + +on_committer_date "00:00" hide_error save_tag U unique_commit U tree +on_committer_date "00:01" save_tag u0 unique_commit u0 tree -p U +on_committer_date "00:01" save_tag u1 unique_commit u1 tree -p u0 +on_committer_date "00:02" save_tag u2 unique_commit u2 tree -p u0 +on_committer_date "00:03" save_tag u3 unique_commit u3 tree -p u0 +on_committer_date "00:04" save_tag u4 unique_commit u4 tree -p u0 +on_committer_date "00:05" save_tag u5 unique_commit u5 tree -p u0 +on_committer_date "00:06" save_tag V unique_commit V tree -p u1 -p u2 -p u3 -p u4 -p u5 + +test_sequence() +{ + _bisect_option=$1 + + test_bisection_diff 0 $_bisect_option l0 ^root + test_bisection_diff 0 $_bisect_option l1 ^root + test_bisection_diff 0 $_bisect_option l2 ^root + test_bisection_diff 0 $_bisect_option a0 ^root + test_bisection_diff 0 $_bisect_option a1 ^root + test_bisection_diff 0 $_bisect_option a2 ^root + test_bisection_diff 0 $_bisect_option a3 ^root + test_bisection_diff 0 $_bisect_option b1 ^root + test_bisection_diff 0 $_bisect_option b2 ^root + test_bisection_diff 0 $_bisect_option b3 ^root + test_bisection_diff 0 $_bisect_option c1 ^root + test_bisection_diff 0 $_bisect_option c2 ^root + test_bisection_diff 0 $_bisect_option c3 ^root + test_bisection_diff 0 $_bisect_option E ^F + test_bisection_diff 0 $_bisect_option e1 ^F + test_bisection_diff 0 $_bisect_option e2 ^F + test_bisection_diff 0 $_bisect_option e3 ^F + test_bisection_diff 0 $_bisect_option e4 ^F + test_bisection_diff 0 $_bisect_option e5 ^F + test_bisection_diff 0 $_bisect_option e6 ^F + test_bisection_diff 0 $_bisect_option e7 ^F + test_bisection_diff 0 $_bisect_option f1 ^F + test_bisection_diff 0 $_bisect_option f2 ^F + test_bisection_diff 0 $_bisect_option f3 ^F + test_bisection_diff 0 $_bisect_option f4 ^F + test_bisection_diff 0 $_bisect_option E ^F + + test_bisection_diff 1 $_bisect_option V ^U + test_bisection_diff 0 $_bisect_option V ^U ^u1 ^u2 ^u3 + test_bisection_diff 0 $_bisect_option u1 ^U + test_bisection_diff 0 $_bisect_option u2 ^U + test_bisection_diff 0 $_bisect_option u3 ^U + test_bisection_diff 0 $_bisect_option u4 ^U + test_bisection_diff 0 $_bisect_option u5 ^U + +# +# the following illustrates Linus' binary bug blatt idea. +# +# assume the bug is actually at l3, but you don't know that - all you know is that l3 is broken +# and it wasn't broken before +# +# keep bisecting the list, advancing the "bad" head and accumulating "good" heads until +# the bisection point is the head - this is the bad point. +# + +test_output_expect_success "$_bisect_option l5 ^root" 'git rev-list $_bisect_option l5 ^root' <expect && + git rev-list --bisect >actual && + test_cmp expect actual +' + +test_expect_success 'rev-parse --bisect can default to good/bad refs' ' + git rev-parse c3 ^b1 ^c1 >expect && + git rev-parse --bisect >actual && + + # output order depends on the refnames, which in turn depends on + # the exact sha1s. We just want to make sure we have the same set + # of lines in any order. + sort expect.sorted && + sort actual.sorted && + test_cmp expect.sorted actual.sorted +' + +test_output_expect_success '--bisect --first-parent' 'git rev-list --bisect --first-parent E ^F' <expect.unsorted <<-EOF && + $(git rev-parse E) (tag: E, dist=0) + $(git rev-parse e1) (tag: e1, dist=1) + $(git rev-parse e2) (tag: e2, dist=2) + $(git rev-parse e3) (tag: e3, dist=3) + $(git rev-parse e4) (tag: e4, dist=4) + $(git rev-parse e5) (tag: e5, dist=4) + $(git rev-parse e6) (tag: e6, dist=3) + $(git rev-parse e7) (tag: e7, dist=2) + $(git rev-parse e8) (tag: e8, dist=1) + EOF + + # expect results to be ordered by distance (descending), + # commit hash (ascending) + sort -k4,4r -k1,1 expect.unsorted >expect && + git rev-list --bisect-all --first-parent E ^F >actual && + test_cmp expect actual +' + +test_expect_success '--bisect without any revisions' ' + git rev-list --bisect HEAD..HEAD >out && + test_must_be_empty out +' + +test_done diff --git a/t/t4220/trim-divergent-old b/t/t4220/trim-divergent-old new file mode 100644 index 00000000000000..daa009c9a1b4b6 --- /dev/null +++ b/t/t4220/trim-divergent-old @@ -0,0 +1,316 @@ +#!/bin/sh +# +# Copyright (c) 2005 Jon Seymour +# +test_description='Tests git rev-list --bisect functionality' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-t6000.sh # t6xxx specific functions + +# usage: test_bisection max-diff bisect-option head ^prune... +# +# e.g. test_bisection 1 --bisect l1 ^l0 +# +test_bisection_diff() +{ + _max_diff=$1 + _bisect_option=$2 + shift 2 + _bisection=$(git rev-list $_bisect_option "$@") + _list_size=$(git rev-list "$@" | wc -l) + _head=$1 + shift 1 + _bisection_size=$(git rev-list $_bisection "$@" | wc -l) + [ -n "$_list_size" -a -n "$_bisection_size" ] || + error "test_bisection_diff failed" + + # Test if bisection size is close to half of list size within + # tolerance. + # + _bisect_err=$(expr $_list_size - $_bisection_size \* 2) + test "$_bisect_err" -lt 0 && _bisect_err=$(expr 0 - $_bisect_err) + _bisect_err=$(expr $_bisect_err / 2) ; # floor + + test_expect_success \ + "bisection diff $_bisect_option $_head $* <= $_max_diff" \ + 'test $_bisect_err -le $_max_diff' +} + +date >path0 +git update-index --add path0 +save_tag tree git write-tree +on_committer_date "00:00" hide_error save_tag root unique_commit root tree +on_committer_date "00:01" save_tag l0 unique_commit l0 tree -p root +on_committer_date "00:02" save_tag l1 unique_commit l1 tree -p l0 +on_committer_date "00:03" save_tag l2 unique_commit l2 tree -p l1 +on_committer_date "00:04" save_tag a0 unique_commit a0 tree -p l2 +on_committer_date "00:05" save_tag a1 unique_commit a1 tree -p a0 +on_committer_date "00:06" save_tag b1 unique_commit b1 tree -p a0 +on_committer_date "00:07" save_tag c1 unique_commit c1 tree -p b1 +on_committer_date "00:08" save_tag b2 unique_commit b2 tree -p b1 +on_committer_date "00:09" save_tag b3 unique_commit b2 tree -p b2 +on_committer_date "00:10" save_tag c2 unique_commit c2 tree -p c1 -p b2 +on_committer_date "00:11" save_tag c3 unique_commit c3 tree -p c2 +on_committer_date "00:12" save_tag a2 unique_commit a2 tree -p a1 +on_committer_date "00:13" save_tag a3 unique_commit a3 tree -p a2 +on_committer_date "00:14" save_tag b4 unique_commit b4 tree -p b3 -p a3 +on_committer_date "00:15" save_tag a4 unique_commit a4 tree -p a3 -p b4 -p c3 +on_committer_date "00:16" save_tag l3 unique_commit l3 tree -p a4 +on_committer_date "00:17" save_tag l4 unique_commit l4 tree -p l3 +on_committer_date "00:18" save_tag l5 unique_commit l5 tree -p l4 +git update-ref HEAD $(tag l5) + + +# E +# / \ +# e1 | +# | | +# e2 | +# | | +# e3 | +# | | +# e4 | +# | | +# | f1 +# | | +# | f2 +# | | +# | f3 +# | | +# | f4 +# | | +# e5 | +# | | +# e6 | +# | | +# e7 | +# | | +# e8 | +# \ / +# F + + +on_committer_date "00:00" hide_error save_tag F unique_commit F tree +on_committer_date "00:01" save_tag e8 unique_commit e8 tree -p F +on_committer_date "00:02" save_tag e7 unique_commit e7 tree -p e8 +on_committer_date "00:03" save_tag e6 unique_commit e6 tree -p e7 +on_committer_date "00:04" save_tag e5 unique_commit e5 tree -p e6 +on_committer_date "00:05" save_tag f4 unique_commit f4 tree -p F +on_committer_date "00:06" save_tag f3 unique_commit f3 tree -p f4 +on_committer_date "00:07" save_tag f2 unique_commit f2 tree -p f3 +on_committer_date "00:08" save_tag f1 unique_commit f1 tree -p f2 +on_committer_date "00:09" save_tag e4 unique_commit e4 tree -p e5 +on_committer_date "00:10" save_tag e3 unique_commit e3 tree -p e4 +on_committer_date "00:11" save_tag e2 unique_commit e2 tree -p e3 +on_committer_date "00:12" save_tag e1 unique_commit e1 tree -p e2 +on_committer_date "00:13" save_tag E unique_commit E tree -p e1 -p f1 + +on_committer_date "00:00" hide_error save_tag U unique_commit U tree +on_committer_date "00:01" save_tag u0 unique_commit u0 tree -p U +on_committer_date "00:01" save_tag u1 unique_commit u1 tree -p u0 +on_committer_date "00:02" save_tag u2 unique_commit u2 tree -p u0 +on_committer_date "00:03" save_tag u3 unique_commit u3 tree -p u0 +on_committer_date "00:04" save_tag u4 unique_commit u4 tree -p u0 +on_committer_date "00:05" save_tag u5 unique_commit u5 tree -p u0 +on_committer_date "00:06" save_tag V unique_commit V tree -p u1 -p u2 -p u3 -p u4 -p u5 + +test_sequence() +{ + _bisect_option=$1 + + test_bisection_diff 0 $_bisect_option l0 ^root + test_bisection_diff 0 $_bisect_option l1 ^root + test_bisection_diff 0 $_bisect_option l2 ^root + test_bisection_diff 0 $_bisect_option a0 ^root + test_bisection_diff 0 $_bisect_option a1 ^root + test_bisection_diff 0 $_bisect_option a2 ^root + test_bisection_diff 0 $_bisect_option a3 ^root + test_bisection_diff 0 $_bisect_option b1 ^root + test_bisection_diff 0 $_bisect_option b2 ^root + test_bisection_diff 0 $_bisect_option b3 ^root + test_bisection_diff 0 $_bisect_option c1 ^root + test_bisection_diff 0 $_bisect_option c2 ^root + test_bisection_diff 0 $_bisect_option c3 ^root + test_bisection_diff 0 $_bisect_option E ^F + test_bisection_diff 0 $_bisect_option e1 ^F + test_bisection_diff 0 $_bisect_option e2 ^F + test_bisection_diff 0 $_bisect_option e3 ^F + test_bisection_diff 0 $_bisect_option e4 ^F + test_bisection_diff 0 $_bisect_option e5 ^F + test_bisection_diff 0 $_bisect_option e6 ^F + test_bisection_diff 0 $_bisect_option e7 ^F + test_bisection_diff 0 $_bisect_option f1 ^F + test_bisection_diff 0 $_bisect_option f2 ^F + test_bisection_diff 0 $_bisect_option f3 ^F + test_bisection_diff 0 $_bisect_option f4 ^F + test_bisection_diff 0 $_bisect_option E ^F + + test_bisection_diff 1 $_bisect_option V ^U + test_bisection_diff 0 $_bisect_option V ^U ^u1 ^u2 ^u3 + test_bisection_diff 0 $_bisect_option u1 ^U + test_bisection_diff 0 $_bisect_option u2 ^U + test_bisection_diff 0 $_bisect_option u3 ^U + test_bisection_diff 0 $_bisect_option u4 ^U + test_bisection_diff 0 $_bisect_option u5 ^U + +# +# the following illustrates Linus' binary bug blatt idea. +# +# assume the bug is actually at l3, but you don't know that - all you know is that l3 is broken +# and it wasn't broken before +# +# keep bisecting the list, advancing the "bad" head and accumulating "good" heads until +# the bisection point is the head - this is the bad point. +# + +test_output_expect_success "$_bisect_option l5 ^root" 'git rev-list $_bisect_option l5 ^root' <expect && + git rev-list --bisect >actual && + test_cmp expect actual +' + +test_expect_success 'rev-parse --bisect can default to good/bad refs' ' + git rev-parse c3 ^b1 ^c1 >expect && + git rev-parse --bisect >actual && + + # output order depends on the refnames, which in turn depends on + # the exact sha1s. We just want to make sure we have the same set + # of lines in any order. + sort expect.sorted && + sort actual.sorted && + test_cmp expect.sorted actual.sorted +' + +test_output_expect_success '--bisect --first-parent' 'git rev-list --bisect --first-parent E ^F' <expect.unsorted <<-EOF && + $(git rev-parse E) (tag: E, dist=0) + $(git rev-parse e1) (tag: e1, dist=1) + $(git rev-parse e2) (tag: e2, dist=2) + $(git rev-parse e3) (tag: e3, dist=3) + $(git rev-parse e4) (tag: e4, dist=4) + $(git rev-parse e5) (tag: e5, dist=4) + $(git rev-parse e6) (tag: e6, dist=3) + $(git rev-parse e7) (tag: e7, dist=2) + $(git rev-parse e8) (tag: e8, dist=1) + EOF + + # expect results to be ordered by distance (descending), + # commit hash (ascending) + sort -k4,4r -k1,1 expect.unsorted >expect && + git rev-list --bisect-all --first-parent E ^F >actual && + test_cmp expect actual +' + +test_expect_success '--bisect without any revisions' ' + git rev-list --bisect HEAD..HEAD >out && + test_must_be_empty out +' + +test_done From 02c68ee9cbdab50362281c72ba11be36385b8f93 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 1 Aug 2026 10:41:48 -0700 Subject: [PATCH 040/259] diff: read precomputed hunks for stat output Teach builtin_diffstat() to consult the hunk provider interface through diff_provider_consult(), new here: the consult-only entry that answers without loading content or computing, so it never returns DIFF_PROVIDER_ERROR. On an answer, the summing callback accumulates the provided counts directly into the diffstat entry; the blobs were already loaded for the binary check, so an answer saves the diff run, not the content load (blame, taught next, skips its loads too). On an unanswered outcome it computes as before and, with a writer attached, records what it computed; on unanswered-no-record it computes without recording. The provider behind the consult is the diff-hunks store, registered in front of the terminal builtin computation. Its consult serves a recorded pair through diff_hunks_replay(), which validates the sequence before any hunk reaches the callback, so direct accumulation is safe. The request gains the pair's object ids and the diff options read by the exclusions below. A side whose bytes are not a stored blob, such as a working-tree file or a gitlink, has a NULL id; the store passes it by and the terminal provider computes it. diff_provider_emit_hunks() walks the same chain, so blame's requests follow these rules the moment blame supplies identity. The walk also insists, as a BUG check, that a request's diff options belong to the repository whose chain it walks. Each exclusion lives with the provider whose key cannot express it. -I patterns and --anchored shape the diff outside the store key, and break detection (-B) rescores the pair outside it; the store's consult maps all three to stop-no-record, so such a request is neither served nor recorded for any consumer. The consumer-side guard the recording commit carried for those three comes out here. The compile-time assert on xpparam_t's layout sits next to that decision, forcing an explicit keying decision whenever a diff parameter is added. The stat consumer keeps only the exclusion that is not about the key: --ignore-blank-lines is part of the key but coalesces hunks differently between the text-emitting and coordinate-callback paths, so the consumer returns before consulting. A "log -L" range-scoped stat neither reads nor records; the line-range filter computes it as before. "git diff", "git log", "git show", and "git diff-tree" with the --stat, --numstat, and --shortstat formats consult the interface. Reading is controlled by core.diffHunks. An answer is invisible in the output, so the store counts the pairs it serves and the consultations it cannot, and diff_hunks_read_stats() reports both; the stat path emits the hits as a trace2 "read-hits" datum for tests and tuning. The counters live on the store because only the store knows whether a consultation reached it, and none of its exclusion legs reaches the replay, so none counts as a miss. Extend t4220 with the read half: - output parity with and without the store, at several context lengths and both directions, and reversed pairs keying apart; - the consultation made visible through the read-hits datum, and the trim-divergent pair correct at every context; - the settings that must bypass the store doing so in both directions (-I, -B, --anchored, --ignore-blank-lines), asserted through the trace rather than output parity alone, which a coincidentally equal count could satisfy; - a driver-forced algorithm keying apart rather than bypassing: it is part of the key, so a read under it misses the default entries and a warm records under its own. A "log -L" range-scoped stat neither reads nor records. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- builtin/log.c | 7 + diff-hunks.c | 88 +++++++++++- diff-hunks.h | 8 ++ diff-provider-internal.h | 7 + diff-provider.c | 62 +++++++-- diff-provider.h | 28 +++- diff.c | 86 ++++++++---- diff.h | 20 ++- t/t4220-diff-hunks.sh | 280 +++++++++++++++++++++++++++++++++++++++ 9 files changed, 539 insertions(+), 47 deletions(-) diff --git a/builtin/log.c b/builtin/log.c index 6903bdd5fff055..0b72477f56ee2a 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -2225,6 +2225,13 @@ int cmd_format_patch(int argc, if (argc > 1) die(_("unrecognized argument: %s"), argv[1]); + /* + * A patch generated by format-patch carries the builtin diffstat, + * not one served from a local store, so its counts do not depend + * on whether the sender warmed the store. + */ + rev.diffopt.flags.no_precomputed_hunks = 1; + if (rev.diffopt.output_format & DIFF_FORMAT_NAME) die(_("--name-only does not make sense")); if (rev.diffopt.output_format & DIFF_FORMAT_NAME_STATUS) diff --git a/diff-hunks.c b/diff-hunks.c index 23cefe055b2071..9830df07fa1aa5 100644 --- a/diff-hunks.c +++ b/diff-hunks.c @@ -26,6 +26,7 @@ #include "csum-file.h" #include "diff-hunks.h" #include "diff-provider-internal.h" +#include "diff.h" #include "gettext.h" #include "hash.h" #include "hashmap.h" @@ -140,6 +141,10 @@ struct diff_hunks_store { uint32_t num_entries; const unsigned char *hdat; size_t hdat_size; + + /* Consultation counters; see diff_hunks_read_stats(). */ + unsigned long read_hits; + unsigned long read_misses; }; static void free_store(struct diff_hunks_store *s) @@ -256,6 +261,15 @@ struct diff_hunks_store *repo_diff_hunks_store(struct repository *r) return r->objects->diff_hunks_store; } +void diff_hunks_read_stats(struct repository *r, + unsigned long *hits, unsigned long *misses) +{ + struct diff_hunks_store *s = repo_diff_hunks_store(r); + + *hits = s ? s->read_hits : 0; + *misses = s ? s->read_misses : 0; +} + void close_diff_hunks_store(struct object_database *o) { if (!o->diff_hunks_store) @@ -413,18 +427,90 @@ int diff_hunks_replay(struct diff_hunks_store *s, struct precomputed_entry e; uint32_t i; + if (!s) + return 0; if (!diff_hunks_store_get(s, old_oid, new_oid, xdl_opts, &e) || - !replayable_hunks(&e)) + !replayable_hunks(&e)) { + s->read_misses++; return 0; + } for (i = 0; i < e.num_hunks; i++) { struct precomputed_hunk h; nth_precomputed_hunk(&e, i, &h); hunk_func(h.old_start, h.old_count, h.new_start, h.new_count, cb_data); } + s->read_hits++; return 1; } +/* + * The store's consult implementation. The store is not + * authoritative, so it serves a recorded pair or passes; what the + * recording key cannot express, it excludes here with the + * stop-no-record disposition. None of those legs reaches + * diff_hunks_replay(), so none of them counts as a miss. + */ +static enum diff_provider_disposition +diff_hunks_store_consult(struct diff_provider *provider UNUSED, + const struct diff_provider_request *req, + diff_provider_fill_fn fill UNUSED, + void *fill_data UNUSED, + xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data) +{ + /* + * xpparam_t is the consult's parameter input. Its flags are + * the store key's xdl_opts; ignore_regex (-I) and anchors + * (--anchored) shape the diff outside the key, so such a + * request is neither served nor recorded. + * + * Adding an xpparam_t field fires this assert (its size no + * longer matches the reference struct). To clear it: (1) add + * the field to the reference struct below; then (2) decide how + * it affects the key: make it part of the key, or exclude + * diffs that use it here with the disposition below. The + * assert only tracks size: a same-size reorder or a changed + * field meaning slips past, so re-read the fields when it + * fires. + */ + (void)BUILD_ASSERT_OR_ZERO(sizeof(xpparam_t) == sizeof(struct { + unsigned long flags; + regex_t **ignore_regex; + size_t ignore_regex_nr; + char **anchors; + size_t anchors_nr; + })); + if (req->xpp->ignore_regex_nr || req->xpp->anchors_nr) + return DIFF_PROVIDER_DISP_STOP_NO_RECORD; + /* + * Break detection (-B) rescores the pair outside xpparam_t, so + * it is outside the key for the same reason. + */ + if (req->diffopt && req->diffopt->break_opt != -1) + return DIFF_PROVIDER_DISP_STOP_NO_RECORD; + + if (!req->old_oid || !req->new_oid) + return DIFF_PROVIDER_DISP_PASS; + if (diff_hunks_replay(repo_diff_hunks_store(req->repo), + req->old_oid, req->new_oid, + req->xpp->flags, hunk_cb, cb_data)) + return DIFF_PROVIDER_DISP_ANSWERED; + return DIFF_PROVIDER_DISP_PASS; +} + +/* + * The provider borrows the repository's store through + * repo_diff_hunks_store() per request; the object database owns the + * file and tears it down, so there is nothing to release here. + */ +struct diff_provider *diff_hunks_store_provider_new(void) +{ + struct diff_provider *p = xcalloc(1, sizeof(*p)); + + p->consult = diff_hunks_store_consult; + return p; +} + /* Validate one store file. Returns 0 if valid or absent, -1 on any error. */ static int verify_store_at(struct repository *r, const char *fname) { diff --git a/diff-hunks.h b/diff-hunks.h index 89e56f4b1bcd32..c58500d05c0de1 100644 --- a/diff-hunks.h +++ b/diff-hunks.h @@ -62,6 +62,14 @@ struct diff_hunks_store *repo_diff_hunks_store(struct repository *r); /* Free the repository's cached store, at object-database teardown. */ void close_diff_hunks_store(struct object_database *o); +/* + * Consultation counters for the repository's store: pairs the store + * served (hits) and pairs it was consulted for but could not serve + * (misses). Both zero when reading is disabled or no store exists. + */ +void diff_hunks_read_stats(struct repository *r, + unsigned long *hits, unsigned long *misses); + /* * Replay the recorded hunks of an (old blob, new blob) pair diffed * under xdl_opts through hunk_func. The sequence is validated before diff --git a/diff-provider-internal.h b/diff-provider-internal.h index 8dd8e4b0dc84ec..8ad3e481e7bd35 100644 --- a/diff-provider-internal.h +++ b/diff-provider-internal.h @@ -88,6 +88,13 @@ struct diff_provider { struct diff_provider *next; }; +/* + * The providers Git ships, besides the builtin computation that + * diff-provider.c holds itself. Each call returns a fresh provider + * for one repository's chain. + */ +struct diff_provider *diff_hunks_store_provider_new(void); + /* * Incremental well-formedness check for a provider-supplied hunk * sequence, shared by every provider. Each coordinate, and each diff --git a/diff-provider.c b/diff-provider.c index b69854fb63b042..66a9909eaa6943 100644 --- a/diff-provider.c +++ b/diff-provider.c @@ -1,5 +1,7 @@ #include "git-compat-util.h" +#include "diff.h" #include "diff-provider-internal.h" +#include "replace-object.h" #include "repository.h" /* @@ -39,10 +41,11 @@ static struct diff_provider *builtin_provider_new(void) /* * The repository's chain, assembled on first walk. The composition - * is fixed; the builtin computation is the terminal provider, so the - * chain always ends in an implementor that can answer. Nothing is - * decided per repository here; each provider gates itself per - * request. + * is fixed, and the order is the authority resolution: the store is + * consulted before the builtin computation, the terminal provider, + * so the chain always ends in an implementor that can answer. + * Nothing is decided per repository here; each provider gates itself + * per request. */ static struct diff_provider *provider_chain(struct repository *r) { @@ -50,6 +53,8 @@ static struct diff_provider *provider_chain(struct repository *r) if (*tail) return *tail; + *tail = diff_hunks_store_provider_new(); + tail = &(*tail)->next; *tail = builtin_provider_new(); return r->diff_providers; } @@ -70,16 +75,16 @@ void diff_providers_clear(struct repository *r) } /* - * The walk behind diff_provider_emit_hunks(): consult the chain in - * order and map its dispositions onto the outcome set. The first - * answer ends the walk. A stop-no-record disposition - * (diff-provider-internal.h) is a refusal, not a pass: the provider - * does not answer, but rules the pair out of identity service and - * out of recording, so from then on the walk consults only the - * computing provider, and a walk that ends unanswered carries the - * no-record verdict. With a fill callback the terminal provider - * computes instead of passing, so an emit walk returns only - * answered or error. + * The walk shared by diff_provider_consult() and + * diff_provider_emit_hunks(): consult the chain in order and map its + * dispositions onto the outcome set. The first answer ends the + * walk. A stop-no-record disposition (diff-provider-internal.h) + * is a refusal, not a pass: the provider does not answer, but rules + * the pair out of identity service and out of recording, so from + * then on the walk consults only the computing provider, and a walk + * that ends unanswered carries the no-record verdict. With a fill + * callback the terminal provider computes instead of passing, so an + * emit walk returns only answered or error. */ static enum diff_provider_outcome walk_providers(const struct diff_provider_request *req, @@ -89,6 +94,28 @@ walk_providers(const struct diff_provider_request *req, struct diff_provider *p; int no_record = 0; + if (req->diffopt && req->diffopt->repo != req->repo) + BUG("diff provider request walks one repository's chain " + "with another repository's diff options"); + + /* + * An object replacement redirects a blob's content + * (OBJECT_INFO_LOOKUP_REPLACE) while leaving the id that names it + * unchanged, so an answer keyed on the raw id would be the + * pre-replacement diff. A replacement is therefore a parameter + * outside the recording key: no provider may serve a replaced pair + * from its identity, and a result computed for it must not be + * recorded under the raw id. Mark the walk no-record so the + * identity providers step aside and the builtin computes from the + * replaced content. The check is a no-op when the repository has + * no replace refs. + */ + if ((req->old_oid && + lookup_replace_object(req->repo, req->old_oid) != req->old_oid) || + (req->new_oid && + lookup_replace_object(req->repo, req->new_oid) != req->new_oid)) + no_record = 1; + for (p = provider_chain(req->repo); p; p = p->next) { enum diff_provider_disposition disp; @@ -118,6 +145,13 @@ walk_providers(const struct diff_provider_request *req, DIFF_PROVIDER_UNANSWERED; } +enum diff_provider_outcome +diff_provider_consult(const struct diff_provider_request *req, + xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data) +{ + return walk_providers(req, NULL, NULL, hunk_cb, cb_data); +} + enum diff_provider_hunks_error diff_provider_check_hunk(struct diff_provider_hunks_check *c, long old_start, long old_count, diff --git a/diff-provider.h b/diff-provider.h index 1a7e4e299c04c0..c5478b570025a2 100644 --- a/diff-provider.h +++ b/diff-provider.h @@ -26,6 +26,8 @@ * sees it. */ +struct diff_options; +struct object_id; struct repository; /* @@ -89,15 +91,35 @@ enum diff_provider_outcome { * A consultation request. The interface consults providers from * these fields alone; no content is loaded before an answer. * - * repo owns the provider chain the request walks. xpp carries the - * parameters the diff runs with. Each provider gates itself on the - * fields that concern it. + * repo owns the provider chain the request walks. old_oid/new_oid + * name the blobs whose bytes are diffed; pass NULL for a side whose + * bytes are not a stored blob (a working-tree file, textconv output, + * a gitlink), so no provider answers from an id it cannot look up. + * diffopt carries the diff settings that live outside xpp; xpp + * carries the parameters the diff runs with. Each provider gates + * itself on the fields that concern it. */ struct diff_provider_request { struct repository *repo; + const struct object_id *old_oid; + const struct object_id *new_oid; + struct diff_options *diffopt; const xpparam_t *xpp; }; +/* + * Consult the providers for the request's pair without computing. + * On DIFF_PROVIDER_ANSWERED the hunks were emitted through hunk_cb + * (0-based emission coordinates, context 0) and were validated + * before the first callback ran, so a consumer may accumulate + * directly into its result. Never returns DIFF_PROVIDER_ERROR. + * The callback's return value is not consulted: emission of a + * validated answer has no error leg, so the callback must return 0. + */ +enum diff_provider_outcome +diff_provider_consult(const struct diff_provider_request *req, + xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data); + /* * Load the pair's content. Called at most once per request, only * when the ranges are computed rather than provided. The buffers diff --git a/diff.c b/diff.c index 11ec88dc8cdeb7..0d4cb3fcfd742a 100644 --- a/diff.c +++ b/diff.c @@ -17,6 +17,7 @@ #include "quote.h" #include "diff.h" #include "diff-hunks.h" +#include "diff-provider.h" #include "diffcore.h" #include "delta.h" #include "hex.h" @@ -35,6 +36,7 @@ #include "tmp-objdir.h" #include "graph.h" #include "oid-array.h" +#include "trace2.h" #include "packfile.h" #include "pager.h" #include "parse-options.h" @@ -2992,6 +2994,11 @@ void diff_hunks_attach(struct diff_options *o) void diff_hunks_detach(struct diff_options *o) { + unsigned long hits, misses; + + diff_hunks_read_stats(o->repo, &hits, &misses); + if (hits) + trace2_data_intmax("diff-hunks", o->repo, "read-hits", hits); diff_hunks_writer_finish(o->hunks_writer); o->hunks_writer = NULL; } @@ -4321,18 +4328,35 @@ static const char *get_compact_summary(const struct diff_filepair *p, int is_ren } /* - * Fill data->added/deleted for a modified pair by collecting its hunk - * coordinates, and record them into the store. Runs only on a warming - * run; returns 1 when it produced the counts, 0 when the caller must - * compute the diffstat itself. + * Hunk callback for the provider interface: sum counts into a + * diffstat entry. + */ +static int diffstat_sum_hunk_cb(long start_a UNUSED, long count_a, + long start_b UNUSED, long count_b, + void *cb_data) +{ + struct diffstat_file *data = cb_data; + + data->added += count_b; + data->deleted += count_a; + return 0; +} + +/* + * Fill data->added/deleted for a modified pair through the hunk provider + * interface: on an answer, sum the provided counts; on a warming run, + * compute and record them. Returns 1 when it produced the counts, 0 when + * the caller must compute the diffstat itself. * - * --ignore-blank-lines is excluded: that flag is part of the store - * key, but it coalesces hunks differently between the emit and - * hunk-callback paths, so a recorded entry would not match a - * store-less run's --stat output. (--inter-hunk-context is not - * excluded: it only groups hunks, and diffstat sums their counts, - * which grouping does not change.) Recording requires both sides to - * be valid regular files whose blobs the key can name. + * The providers own the exclusions the request can express (-B, -I, + * and --anchored are outside the store key). This consumer additionally + * excludes --ignore-blank-lines before consulting: that flag is part of + * the key, but it coalesces hunks differently between the emit and + * hunk-callback paths, so a served answer would not match a store-less + * run's --stat output. (--inter-hunk-context is not excluded: it only + * groups hunks, and diffstat sums their counts, which grouping does not + * change.) Recording requires both sides to be valid regular files whose + * blobs the key can name. */ static int diffstat_from_hunks(struct diff_options *o, struct diff_filespec *one, @@ -4347,23 +4371,37 @@ static int diffstat_from_hunks(struct diff_options *o, .ignore_regex_nr = o->ignore_regex_nr, .anchors = o->anchors, .anchors_nr = o->anchors_nr }; + struct diff_provider_request req = { + .repo = o->repo, + .old_oid = (one->oid_valid && !S_ISGITLINK(one->mode)) ? + &one->oid : NULL, + .new_oid = (two->oid_valid && !S_ISGITLINK(two->mode)) ? + &two->oid : NULL, + .diffopt = o, + .xpp = &xpp, + }; if (o->xdl_opts & XDF_IGNORE_BLANK_LINES) return 0; + /* format-patch keeps its diffstat off the store (see the flag). */ + if (o->flags.no_precomputed_hunks) + return 0; - /* Not a warming run: the caller computes the diffstat. */ - if (!o->hunks_writer) + switch (diff_provider_consult(&req, diffstat_sum_hunk_cb, data)) { + case DIFF_PROVIDER_ANSWERED: + return 1; + case DIFF_PROVIDER_UNANSWERED: + break; + case DIFF_PROVIDER_ERROR: /* not returned by a consult */ + case DIFF_PROVIDER_UNANSWERED_NO_RECORD: return 0; - /* - * -I patterns, --anchored anchors, and break detection (-B) - * shape the diff outside the store key, so what they compute - * must not be recorded under it. - */ - if (o->ignore_regex_nr || o->anchors_nr || o->break_opt != -1) + } + + /* A miss on a read-only run: let the caller compute the diffstat. */ + if (!o->hunks_writer) return 0; /* Recording needs blobs the key can name, on both sides. */ - if (!one->oid_valid || !two->oid_valid || - S_ISGITLINK(one->mode) || S_ISGITLINK(two->mode) || + if (!req.old_oid || !req.new_oid || !DIFF_FILE_VALID(one) || !DIFF_FILE_VALID(two) || !S_ISREG(one->mode) || !S_ISREG(two->mode)) return 0; @@ -4453,9 +4491,9 @@ static void builtin_diffstat(const char *name_a, const char *name_b, else if (may_differ) { /* - * Record into the diff-hunks store on a warming run. A - * "log -L" range-scoped stat is not the whole-pair diff - * the store keys, so it does not record. Otherwise diff + * Serve or record via the diff-hunks store. A "log -L" + * range-scoped stat is not the whole-pair diff the store + * keys, so it neither reads nor records. Otherwise diff * normally. */ if (p->line_ranges || !diffstat_from_hunks(o, one, two, data)) { diff --git a/diff.h b/diff.h index 3d44de39ff9056..6166598eeae67d 100644 --- a/diff.h +++ b/diff.h @@ -206,6 +206,13 @@ struct diff_flags { unsigned suppress_diff_headers; unsigned dual_color_diffed_diffs; unsigned suppress_hunk_header_line_count; + + /* + * Do not serve the diffstat from the precomputed-hunks store. + * Set by format-patch so a generated patch carries the builtin + * counts and does not depend on the sender's local store state. + */ + unsigned no_precomputed_hunks; }; static inline void diff_flags_or(struct diff_flags *a, @@ -422,9 +429,11 @@ struct diff_options { int max_depth_valid; /* - * Precomputed diff hunks (see diff-hunks.h). When hunks_writer is - * set (a warming run), diffstat records the hunks it computes; - * the writer is attached only for the stat output formats. + * Precomputed diff hunks (see diff-hunks.h). diffstat consults the + * hunk provider interface before running xdiff, keyed by each file + * pair's blob object IDs. When hunks_writer is set (a warming run), + * diffstat also records the hunks it computes; the writer is + * attached only for the stat output formats. */ struct diff_hunks_writer *hunks_writer; }; @@ -679,8 +688,9 @@ void diff_free(struct diff_options*); /* * Attach a diff-hunks writer to a diff producing a stat format, so a * warming run records the hunks it computes; a no-op when writing is off - * or for other formats. Pair with diff_hunks_detach() once the diff is - * done. + * or for other formats. (Reading is separate: consumers consult the + * providers through diff_provider_consult(); see diff-provider.h.) Pair + * with diff_hunks_detach() once the diff is done. */ void diff_hunks_attach(struct diff_options *o); void diff_hunks_detach(struct diff_options *o); diff --git a/t/t4220-diff-hunks.sh b/t/t4220-diff-hunks.sh index c6778319460354..62329e1070e0aa 100755 --- a/t/t4220-diff-hunks.sh +++ b/t/t4220-diff-hunks.sh @@ -81,6 +81,79 @@ test_expect_success 'a second warming run refreshes the store in place' ' test_cmp expect actual ' +test_expect_success 'log --stat matches with and without the store' ' + no_store log --stat >expect && + warm && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'log --numstat and --shortstat match' ' + no_store log --numstat >expect_num && + no_store log --shortstat >expect_short && + warm && + git log --numstat >actual_num && + git log --shortstat >actual_short && + test_cmp expect_num actual_num && + test_cmp expect_short actual_short +' + +# A built store must reproduce diffstat output at every context +# length. Only trim-stable pairs are recorded, so one entry serves +# every context; a trim-divergent pair is never recorded and always +# computed. Zero context is where trim_common_tail runs, which is +# what makes the two diffs differ. +test_expect_success 'diffstat matches at several context lengths' ' + no_store log --stat >expect_def && + no_store log -U0 --stat >expect_u0 && + no_store log -U7 --stat >expect_u7 && + warm && + git log --stat >got_def && + git log -U0 --stat >got_u0 && + git log -U7 --stat >got_u7 && + test_cmp expect_def got_def && + test_cmp expect_u0 got_u0 && + test_cmp expect_u7 got_u7 +' + +test_expect_success 'store built at a nonzero context stays correct at that context' ' + no_store -c diff.context=5 log --stat >expect && + warm -c diff.context=5 && + git -c diff.context=5 log --stat >actual && + test_cmp expect actual +' + +# This blob pair (a real git test file being modernized) has different +# valid diffs at different contexts: at zero context, where +# trim_common_tail runs, "diff -U0" reports 9/6, while "diff -U3" +# reports 10/7. Such a trim-divergent pair is exactly what the writer +# must never record, since no single entry could serve both readers. +# A compact synthetic pair cannot show this count split: on small +# inputs xdiff produces minimal diffs, minimal diffs of one pair all +# add and delete the same number of lines, and trimming the common +# tail preserves minimality, so the counts agree by construction (a +# search over thousands of synthetic pairs up to 8 lines found no +# split). The split needs the cost-capping heuristics that only larger +# inputs trigger, so the pair is shipped as a fixture under t4220/. +test_expect_success 'a trim-divergent file is correct at each context' ' + cp "$TEST_DIRECTORY/t4220/trim-divergent-old" div.sh && + git add div.sh && + git commit -m divergent-old && + cp "$TEST_DIRECTORY/t4220/trim-divergent-new" div.sh && + git add div.sh && + git commit -m divergent-new && + no_store log -1 --format= --stat -- div.sh >expect_def && + no_store log -1 --format= -U0 --stat -- div.sh >expect_u0 && + warm && + git log -1 --format= --stat -- div.sh >got_def && + git log -1 --format= -U0 --stat -- div.sh >got_u0 && + test_cmp expect_def got_def && + test_cmp expect_u0 got_u0 && + # The fixture must actually diverge, or the test would pass without + # exercising the split; fail loudly if a diff change ever levels it. + ! test_cmp expect_def expect_u0 +' + # A warming run displays the diffstat it computes. At zero context xdi_diff # trims, so the displayed counts must be the trimmed ones (what a store-less # run shows), not the untrimmed ones the writer compares against when it @@ -99,6 +172,16 @@ test_expect_success 'warming --stat at zero context matches a store-less run' ' ) ' +test_expect_success 'diff --stat matches with and without the store, both directions' ' + no_store diff --stat second fourth >expect_fwd && + no_store diff --stat fourth second >expect_rev && + warm && + git diff --stat second fourth >got_fwd && + git diff --stat fourth second >got_rev && + test_cmp expect_fwd got_fwd && + test_cmp expect_rev got_rev +' + test_expect_success 'show and diff-tree --stat use the store' ' test_when_finished "git diff-hunks clear" && # diff_hunks_attach() runs for show and diff-tree: a write-enabled @@ -121,6 +204,193 @@ test_expect_success 'show and diff-tree --stat use the store' ' test_cmp expect_dt got_dt ' +test_expect_success 'log -R --stat matches (reversed pairs keyed apart)' ' + no_store log -R --stat >expect && + warm && + git log -R --stat >actual && + test_cmp expect actual +' + +# The diffstat read path produces identical output on a hit or a miss, so +# it emits a trace2 "read-hits" count to prove it consulted the store. +test_expect_success 'diffstat consults the store (trace shows read hits)' ' + warm && + GIT_TRACE2_EVENT="$PWD/trace_on.json" git log --stat >/dev/null && + test_grep read-hits trace_on.json && + test_env GIT_TRACE2_EVENT="$PWD/trace_off.json" no_store log --stat >/dev/null && + test_grep ! read-hits trace_off.json +' + +# Diff settings that change hunks but are not part of the store key must +# bypass it in both directions, so output stays byte-identical to a +# store-less run. +test_expect_success 'setup ignore fixture' ' + git init ignore-repo && + ( + cd ignore-repo && + test_write_lines code keep "# c" >f && + git add f && + git commit -m c1 && + test_write_lines codeCH keep "# cX" >f && + git add f && + git commit -m c2 && + warm + ) +' + +# Output parity alone cannot prove the guard: served counts can +# coincide with computed ones, so each bypass below also asserts the +# consultation itself (no read hit with the option, a hit without it) +# and that a warming run under the option records nothing. +test_expect_success '-I bypasses the store in both directions' ' + ( + cd ignore-repo && + no_store diff -I"^#" --numstat HEAD~ HEAD >expect && + git diff -I"^#" --numstat HEAD~ HEAD >actual && + test_cmp expect actual && + # -I does not change the key, so only the ignore_regex + # guard keeps the warmed entry from serving here. + GIT_TRACE2_EVENT="$PWD/trace_i.json" \ + git diff -I"^#" --numstat HEAD~ HEAD >/dev/null && + test_grep ! read-hits trace_i.json && + GIT_TRACE2_EVENT="$PWD/trace_i_ctl.json" \ + git diff --numstat HEAD~ HEAD >/dev/null && + test_grep read-hits trace_i_ctl.json && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git diff -I"^#" --numstat HEAD~ HEAD >/dev/null && + test_path_is_missing .git/objects/info/diff-hunks && + # Restore the warmed fixture for the tests below. + warm + ) +' + +test_expect_success '-B bypasses the store in both directions' ' + git init break-repo && + ( + cd break-repo && + test_write_lines a b c d e f g h >f && + git add f && + git commit -m orig && + test_write_lines 1 2 3 4 5 6 7 8 >f && + git add f && + git commit -m rewrite && + warm && + no_store diff -B --stat HEAD~ HEAD >expect && + git diff -B --stat HEAD~ HEAD >actual && + test_cmp expect actual && + GIT_TRACE2_EVENT="$PWD/trace_b.json" \ + git diff -B --stat HEAD~ HEAD >/dev/null && + test_grep ! read-hits trace_b.json && + GIT_TRACE2_EVENT="$PWD/trace_ctl.json" \ + git diff --stat HEAD~ HEAD >/dev/null && + test_grep read-hits trace_ctl.json && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git diff -B --stat HEAD~ HEAD >/dev/null && + test_path_is_missing .git/objects/info/diff-hunks + ) +' + +test_expect_success '--anchored bypasses the store in both directions' ' + ( + cd ignore-repo && + no_store diff --stat --anchored=keep HEAD~ HEAD >expect && + git diff --stat --anchored=keep HEAD~ HEAD >actual && + test_cmp expect actual && + # Anchors do not change the key, so only the anchors guard + # keeps the warmed entry from serving here. + GIT_TRACE2_EVENT="$PWD/trace_anchor.json" \ + git diff --stat --anchored=keep HEAD~ HEAD >/dev/null && + test_grep ! read-hits trace_anchor.json && + GIT_TRACE2_EVENT="$PWD/trace_plain.json" \ + git diff --stat HEAD~ HEAD >/dev/null && + test_grep read-hits trace_plain.json && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 \ + git diff --stat --anchored=keep HEAD~ HEAD >/dev/null && + test_path_is_missing .git/objects/info/diff-hunks + ) +' + +test_expect_success '--ignore-blank-lines bypasses the store in both directions' ' + git init ibl-repo && + ( + cd ibl-repo && + printf "a\n\nx\ny\nb\n" >f && + git add f && + git commit -m v1 && + printf "a\nx\ny\nB\n" >f && + git add f && + git commit -m v2 && + warm && + no_store diff --stat --ignore-blank-lines HEAD~ HEAD >expect && + git diff --stat --ignore-blank-lines HEAD~ HEAD >actual && + test_cmp expect actual && + # The flag is an xdl_opts bit and thus part of the key; the + # stat consumer excludes it before consulting at all. + GIT_TRACE2_EVENT="$PWD/trace_ibl.json" \ + git diff --stat --ignore-blank-lines HEAD~ HEAD >/dev/null && + test_grep ! read-hits trace_ibl.json && + GIT_TRACE2_EVENT="$PWD/trace_plain.json" \ + git diff --stat HEAD~ HEAD >/dev/null && + test_grep read-hits trace_plain.json && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 \ + git diff --stat --ignore-blank-lines HEAD~ HEAD >/dev/null && + test_path_is_missing .git/objects/info/diff-hunks + ) +' + +test_expect_success 'a whitespace-ignoring diff is not served default entries' ' + git init ws-repo && + ( + cd ws-repo && + test_write_lines alpha beta gamma >f && + git add f && + git commit -m c1 && + test_write_lines " alpha" beta gamma delta >f && + git add f && + git commit -m c2 && + warm && + no_store diff -w --numstat HEAD~ HEAD >expect && + git diff -w --numstat HEAD~ HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success 'a driver algorithm override keeps output correct and keys apart' ' + git init driver-algo && + ( + cd driver-algo && + echo "file.foo diff=foo" >.gitattributes && + git add .gitattributes && + git commit -m attributes && + test_write_lines 1 2 3 4 5 >file.foo && + git add file.foo && + git commit -m one && + test_write_lines 1 2 X 4 5 6 >file.foo && + git add file.foo && + git commit -m two && + warm -c diff.foo.algorithm=histogram && + no_store -c diff.foo.algorithm=histogram log --stat >expect && + git -c diff.foo.algorithm=histogram log --stat >actual && + test_cmp expect actual && + # The driver algorithm is an xdl_opts key bit: entries + # warmed at the default settings must not serve a + # driver-forced histogram read, and output stays correct. + git diff-hunks clear && + warm && + no_store -c diff.foo.algorithm=histogram log --stat >expect2 && + git -c diff.foo.algorithm=histogram log --stat >actual2 && + test_cmp expect2 actual2 && + GIT_TRACE2_EVENT="$PWD/trace_algo.json" \ + git -c diff.foo.algorithm=histogram log --stat >/dev/null && + test_grep ! read-hits trace_algo.json && + GIT_TRACE2_EVENT="$PWD/trace_algo_ctl.json" \ + git log --stat >/dev/null && + test_grep read-hits trace_algo_ctl.json + ) +' + # Cover the pair shapes an object walk encounters: binary and # mode-only changes produce no text hunks to record. test_expect_success 'binary and mode-only changes do not break the writer' ' @@ -141,6 +411,16 @@ test_expect_success 'binary and mode-only changes do not break the writer' ' test_cmp expect actual ' +test_expect_success 'log -L --stat neither reads nor records' ' + warm && + GIT_TRACE2_EVENT="$PWD/trace_linelog.json" \ + git log -L1,1:file.txt --stat >/dev/null && + test_grep ! read-hits trace_linelog.json && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git log -L1,1:file.txt --stat >/dev/null && + test_path_is_missing $STORE +' + test_expect_success 'verify succeeds on a valid store and on an absent one' ' warm && git diff-hunks verify && From 4e6973492ff71a4ba6178527649b01e2fa8ba633 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 1 Aug 2026 10:41:49 -0700 Subject: [PATCH 041/259] blame: read precomputed hunks Before diffing a target blob against a parent, offer the pair's identity to the hunk provider interface. Blame's requests have gone through diff_provider_emit_hunks() since the interface arrived, but carried no identity, so nothing could answer them. Now blame fills in the pair's blob object ids and its diff options, and the chain serves the pair from the store, keyed by the ids and the request's xdiff flags, before the terminal provider falls back to fill-and-compute. Blame diffs at zero context, which is not part of the key. An answer replays the recorded hunks through blame_chunk_cb without loading either blob; a request carrying -I patterns or anchors is outside the key and always computes. Blame withholds the identity where its diff is not the plain blob-pair diff the key describes: reverse blame, ignored revisions, textconv paths, and the working-tree or --contents pseudo-commit, whose blob is not a stored object. Those requests always compute. Whitespace and algorithm options such as -w instead change blame's xdl_opts, so the consult keys a different entry and misses a store warmed without them. Blame's default xdl_opts now come from DIFF_HUNKS_DEFAULT_XDL_OPTS, new here, which records the key-relevant defaults a diff_options-based consumer already carries (today the indent heuristic), so a default blame run and a default "log --stat" warming run share keys by construction. "--show-stats" reports how many pairs the store served and how many consultations it could not, read from diff_hunks_read_stats(); the store counts its own consultations, so blame keeps no tally. Extend t4220 with the blame side: - parity for plain, --porcelain, and --incremental output, and hit and miss accounting across warming runs; - the blame inputs that must bypass or miss the store: -w, indent heuristics, --reverse, textconv, -M/-C, and the --ignore-rev pass; - rename and merge handling, and --contents; - reading a truncated or corrupt store as absent, and a crafted zero-hunk record as a miss that verify flags. Add p4218, measuring the cost of a warming run and the read speedups. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- blame.c | 39 ++++ builtin/blame.c | 8 +- diff.h | 11 ++ t/meson.build | 1 + t/perf/p4218-diff-hunks.sh | 48 +++++ t/t4220-diff-hunks.sh | 355 +++++++++++++++++++++++++++++++++++++ 6 files changed, 461 insertions(+), 1 deletion(-) create mode 100755 t/perf/p4218-diff-hunks.sh diff --git a/blame.c b/blame.c index c3ef9c17f758c7..7d7671ef5dd128 100644 --- a/blame.c +++ b/blame.c @@ -24,6 +24,7 @@ #include "bloom.h" #include "commit-graph.h" #include "diff-provider.h" +#include "userdiff.h" define_commit_slab(blame_suspects, struct blame_origin *); static struct blame_suspects blame_suspects; @@ -1937,6 +1938,24 @@ static int blame_chunk_cb(long start_a, long count_a, return 0; } +/* + * A hunk provider's key names the (old blob, new blob) pair and may only + * serve a diff whose result is determined by that pair and the xdiff + * settings. Textconv rewrites the buffers being diffed away from the + * blob contents the key names, so any origin whose path has a textconv + * driver must withhold the pair's identity. + */ +static int blame_textconv_active(struct blame_scoreboard *sb, + const char *path) +{ + struct userdiff_driver *drv; + + if (!sb->revs->diffopt.flags.allow_textconv) + return 0; + drv = userdiff_find_by_path(sb->repo->index, path); + return drv && drv->textconv; +} + struct blame_diff_fill_data { struct blame_scoreboard *sb; struct blame_origin *parent, *target; @@ -1973,6 +1992,7 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, struct blame_diff_fill_data fill_data = { sb, parent, target, ignore_diffs }; xpparam_t xpp = { .flags = sb->xdl_opts }; struct diff_provider_request req = { .repo = sb->repo, .xpp = &xpp }; + int provider_usable; if (!target->suspects) return; /* nothing remains for this target */ @@ -1983,6 +2003,25 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, d.ignore_diffs = ignore_diffs; d.dstq = &newdest; d.srcq = &target->suspects; + /* + * Offer the pair's identity only where blame's diff is the plain + * blob-pair diff the recording key describes; reverse blame, + * ignored revisions, and textconv paths withhold it and always + * compute. The working-tree/--contents pseudo-commit (marked by + * its null commit id) holds a blob that is not a stored object, + * so its pairs withhold identity too: no id may be sent that + * names bytes a provider cannot look up. + */ + provider_usable = !sb->reverse && !ignore_diffs && + !is_null_oid(&target->commit->object.oid) && + !blame_textconv_active(sb, target->path) && + !blame_textconv_active(sb, parent->path); + + if (provider_usable) { + req.old_oid = &parent->blob_oid; + req.new_oid = &target->blob_oid; + } + req.diffopt = &sb->revs->diffopt; if (diff_provider_emit_hunks(&req, blame_diff_fill, &fill_data, blame_chunk_cb, &d) == DIFF_PROVIDER_ERROR) die("unable to generate diff (%s -> %s)", diff --git a/builtin/blame.c b/builtin/blame.c index 48d5251c6df700..7891d82ae6b093 100644 --- a/builtin/blame.c +++ b/builtin/blame.c @@ -15,6 +15,7 @@ #include "hex.h" #include "commit.h" #include "diff.h" +#include "diff-hunks.h" #include "revision.h" #include "quote.h" #include "string-list.h" @@ -1060,7 +1061,7 @@ int cmd_blame(int argc, parse_done: revision_opts_finish(&revs); no_whole_file_rename = !revs.diffopt.flags.follow_renames; - xdl_opts |= revs.diffopt.xdl_opts & XDF_INDENT_HEURISTIC; + xdl_opts |= revs.diffopt.xdl_opts & DIFF_HUNKS_DEFAULT_XDL_OPTS; revs.diffopt.flags.follow_renames = 0; argc = parse_options_end(&ctx); @@ -1315,9 +1316,14 @@ int cmd_blame(int argc, output(&sb, output_option); if (show_stats) { + unsigned long hunk_hits, hunk_misses; + + diff_hunks_read_stats(sb.repo, &hunk_hits, &hunk_misses); printf("num read blob: %d\n", sb.num_read_blob); printf("num get patch: %d\n", sb.num_get_patch); printf("num commits: %d\n", sb.num_commits); + printf("num precomputed hits: %lu\n", hunk_hits); + printf("num precomputed misses: %lu\n", hunk_misses); } cleanup: diff --git a/diff.h b/diff.h index 6166598eeae67d..380a25887839a2 100644 --- a/diff.h +++ b/diff.h @@ -231,6 +231,17 @@ static inline void diff_flags_or(struct diff_flags *a, #define DIFF_WITH_ALG(opts, flag) (((opts)->xdl_opts & ~XDF_DIFF_ALGORITHM_MASK) | XDF_##flag) +/* + * The xdl_opts bits git turns on by default that a from-scratch xdl_opts + * (git blame's own option parsing) does not set, and so must OR in to match + * a store warmed at the default diff settings; a diff_options-based consumer + * (diffstat) already has them in o->xdl_opts. Today this is only the indent + * heuristic. It does NOT cover a non-default diff.algorithm: a repo that + * configures one records under that algorithm, and a consumer keying without + * it misses (a lost hit, not wrong output). + */ +#define DIFF_HUNKS_DEFAULT_XDL_OPTS XDF_INDENT_HEURISTIC + enum diff_words_type { DIFF_WORDS_NONE = 0, DIFF_WORDS_PORCELAIN, diff --git a/t/meson.build b/t/meson.build index c63c30ba631d03..3f45b09dd668bd 100644 --- a/t/meson.build +++ b/t/meson.build @@ -1158,6 +1158,7 @@ benchmarks = [ 'perf/p4205-log-pretty-formats.sh', 'perf/p4209-pickaxe.sh', 'perf/p4211-line-log.sh', + 'perf/p4218-diff-hunks.sh', 'perf/p4220-log-grep-engines.sh', 'perf/p4221-log-grep-engines-fixed.sh', 'perf/p5302-pack-index.sh', diff --git a/t/perf/p4218-diff-hunks.sh b/t/perf/p4218-diff-hunks.sh new file mode 100755 index 00000000000000..f849e97832eeda --- /dev/null +++ b/t/perf/p4218-diff-hunks.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +test_description='diff-hunks store performance' +. ./perf-lib.sh + +test_perf_default_repo + +# Pick a file to blame pseudo-randomly. The sort key is the blob +# hash, so it is stable. +test_expect_success 'select a file' ' + git ls-tree -r HEAD | grep ^100644 | + sort -k 3 | head -n 1 | cut -f 2 >filelist +' + +file=$(cat filelist) +export file + +# Warm the store the way an owner would: a stat walk with writing on. +test_perf 'warm the store' ' + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null +' + +test_expect_success 'ensure the store is warm for the timed reads' ' + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null +' + +test_perf 'log --stat -1000 (store)' ' + git log --stat -1000 >/dev/null +' + +test_perf 'log --stat -1000 (no store)' ' + git -c core.diffhunks=false log --stat -1000 >/dev/null +' + +test_perf 'blame $file (store)' ' + git blame "$file" >/dev/null +' + +test_perf 'blame $file (no store)' ' + git -c core.diffhunks=false blame "$file" >/dev/null +' + +test_expect_success 'clean up store' ' + git diff-hunks clear +' + +test_done diff --git a/t/t4220-diff-hunks.sh b/t/t4220-diff-hunks.sh index 62329e1070e0aa..086c53f651438f 100755 --- a/t/t4220-diff-hunks.sh +++ b/t/t4220-diff-hunks.sh @@ -81,6 +81,49 @@ test_expect_success 'a second warming run refreshes the store in place' ' test_cmp expect actual ' +test_expect_success 'core.diffhunks=false disables lookups' ' + warm && + git -c core.diffhunks=false blame --show-stats file.txt >out 2>&1 && + test_grep "num precomputed hits: 0" out +' + +# Writing seeds from the current store and merges into it, so a later +# warming run keeps the entries an earlier one recorded rather than +# rebuilding. Warm one pair, then a different pair, and confirm the first +# is still served. +test_expect_success 'a later warming run preserves earlier entries' ' + git init incr && + ( + cd incr && + test_commit a1 f.txt "1" && + test_commit a2 f.txt "1 +2" && + test_commit a3 f.txt "1 +2 +3" && + GIT_DIFF_HUNKS_WRITE=1 git diff --stat a1 a2 >/dev/null && + git diff-hunks verify && + GIT_DIFF_HUNKS_WRITE=1 git diff --stat a2 a3 >/dev/null && + git diff-hunks verify && + + # Blaming as of a2 diffs the a1..a2 pair. If seeding had + # dropped it when the a2..a3 pair was warmed, this would + # report zero precomputed hits. + git blame --show-stats a2 -- f.txt >out 2>&1 && + test_grep "num precomputed hits: [1-9]" out && + + # The second warm ADDED the a2..a3 pair; blaming a3 diffs + # both a2..a3 and a1..a2, so a hit on each shows the store + # gained the new pair while keeping the earlier one. + git blame --show-stats a3 -- f.txt >out3 2>&1 && + test_grep "num precomputed hits: 2" out3 && + + no_store log --stat >expect && + git log --stat >actual && + test_cmp expect actual + ) +' + test_expect_success 'log --stat matches with and without the store' ' no_store log --stat >expect && warm && @@ -211,6 +254,14 @@ test_expect_success 'log -R --stat matches (reversed pairs keyed apart)' ' test_cmp expect actual ' +# One warm serves both diffstat and blame: the blob pairs a blame +# walks are the same parent-child pairs the diffstat warm recorded. +test_expect_success 'a single warming run serves both blame and diffstat' ' + warm && + git blame --show-stats file.txt >out 2>&1 && + test_grep "num precomputed hits: [1-9][0-9]*" out +' + # The diffstat read path produces identical output on a hit or a miss, so # it emits a trace2 "read-hits" count to prove it consulted the store. test_expect_success 'diffstat consults the store (trace shows read hits)' ' @@ -221,6 +272,23 @@ test_expect_success 'diffstat consults the store (trace shows read hits)' ' test_grep ! read-hits trace_off.json ' +test_expect_success 'blame matches with and without the store' ' + no_store blame file.txt >expect && + warm && + git blame file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'blame --porcelain and --incremental match' ' + no_store blame --porcelain file.txt >expect_p && + no_store blame --incremental file.txt >expect_i && + warm && + git blame --porcelain file.txt >got_p && + git blame --incremental file.txt >got_i && + test_cmp expect_p got_p && + test_cmp expect_i got_i +' + # Diff settings that change hunks but are not part of the store key must # bypass it in both directions, so output stays byte-identical to a # store-less run. @@ -357,6 +425,26 @@ test_expect_success 'a whitespace-ignoring diff is not served default entries' ' ) ' +test_expect_success 'blame -w stays correct and does not hit default entries' ' + ( + cd ws-repo && + no_store blame -w f >expect && + git blame -w --show-stats f >out 2>&1 && + test_grep "num precomputed hits: 0" out && + git blame -w f >actual && + test_cmp expect actual + ) +' + +test_expect_success 'blame with indentHeuristic off stays correct and misses' ' + warm && + git -c diff.indentHeuristic=false blame --show-stats file.txt >out 2>&1 && + test_grep "num precomputed hits: 0" out && + no_store -c diff.indentHeuristic=false blame file.txt >expect && + git -c diff.indentHeuristic=false blame file.txt >actual && + test_cmp expect actual +' + test_expect_success 'a driver algorithm override keeps output correct and keys apart' ' git init driver-algo && ( @@ -391,6 +479,94 @@ test_expect_success 'a driver algorithm override keeps output correct and keys a ) ' +test_expect_success 'blame --reverse never consults the store' ' + warm && + git blame --reverse HEAD~3..HEAD file.txt >actual 2>/dev/null && + no_store blame --reverse HEAD~3..HEAD file.txt >expect 2>/dev/null && + test_cmp expect actual && + # Reverse blame withholds the pair identity. Zero hits alone + # cannot prove that: reverse pairs are never warmed, so a + # consulted pair would miss, not hit. Zero misses is what shows + # the store was never consulted. + git blame --reverse --show-stats HEAD~3..HEAD file.txt \ + >stats 2>/dev/null && + test_grep "num precomputed hits: 0" stats && + test_grep "num precomputed misses: 0" stats +' + +test_expect_success 'blame with a textconv driver bypasses the store' ' + echo "tc.txt diff=tc" >>.gitattributes && + git add .gitattributes && + git commit -m tc-attr && + git config diff.tc.textconv "sed -e s/1/one/" && + test_commit tc1 tc.txt "line 1" && + test_commit tc2 tc.txt "line 1 +line 2" && + warm && + git blame --show-stats tc.txt >out 2>&1 && + test_grep "num precomputed hits: 0" out && + no_store blame tc.txt >expect && + git blame tc.txt >actual && + test_cmp expect actual +' + +test_expect_success 'a replaced blob makes the store step aside' ' + git init replace-repo && + ( + cd replace-repo && + test_commit r1 f.txt "a" && + test_commit r2 f.txt "a +b" && + warm && + # Control: without a replacement the pair is served. + GIT_TRACE2_EVENT="$PWD/trace_ctl.json" \ + git log -1 --format= --numstat -- f.txt >/dev/null && + test_grep read-hits trace_ctl.json && + # Replace r2 blob: the diff now reads different content + # (through OBJECT_INFO_LOOKUP_REPLACE) under the id the store + # keyed, so a served answer would be the pre-replacement diff. + # Identity is withheld, the store steps aside, and the builtin + # computes from the replaced content. + new_blob=$(git rev-parse HEAD:f.txt) && + repl=$(printf "a\nB\nC\nD\n" | git hash-object -w --stdin) && + git replace "$new_blob" "$repl" && + no_store log -1 --format= --numstat -- f.txt >expect && + git log -1 --format= --numstat -- f.txt >actual && + test_cmp expect actual && + GIT_TRACE2_EVENT="$PWD/trace_repl.json" \ + git log -1 --format= --numstat -- f.txt >/dev/null && + test_grep ! read-hits trace_repl.json + ) +' + +test_expect_success 'blame -M and -C stay correct with the store' ' + warm && + no_store blame -M file.txt >expect_m && + no_store blame -C file.txt >expect_c && + git blame -M file.txt >got_m && + git blame -C file.txt >got_c && + test_cmp expect_m got_m && + test_cmp expect_c got_c +' + +# Copy-detecting (and reverse) blame still diff blob pairs through +# pass_blame_to_parent, so they must use the real blame xdl_opts. A +# whitespace-only change is invisible under -w; if -w were dropped on +# these paths the -w and non-w results would coincide. +test_expect_success 'blame -C honors -w' ' + git init -q blame-cw && + ( + cd blame-cw && + printf "one\ntwo\nthree\n" >f && + git add f && git commit -q -m base && + printf "one\n two \nthree\n" >f && + git add f && git commit -q -m reindent && + git blame -C -w f >with_w && + git blame -C f >without_w && + ! test_cmp with_w without_w + ) +' + # Cover the pair shapes an object walk encounters: binary and # mode-only changes produce no text hunks to record. test_expect_success 'binary and mode-only changes do not break the writer' ' @@ -411,6 +587,92 @@ test_expect_success 'binary and mode-only changes do not break the writer' ' test_cmp expect actual ' +test_expect_success 'blame across a rename matches' ' + echo "original content" >rename-src.txt && + git add rename-src.txt && + git commit -m "add rename-src" && + echo "more" >>rename-src.txt && + git add rename-src.txt && + git commit -m "modify rename-src" && + git mv rename-src.txt rename-dst.txt && + git commit -m "rename" && + echo "post" >>rename-dst.txt && + git add rename-dst.txt && + git commit -m "modify after rename" && + no_store blame rename-dst.txt >expect && + warm && + git blame rename-dst.txt >actual && + test_cmp expect actual +' + +test_expect_success 'blame handles merge commits' ' + git checkout -b merge-side main~2 && + test_commit merge-change merge-file.txt "side content" && + git checkout main && + git merge --no-edit merge-side && + no_store blame merge-file.txt >expect && + warm && + git blame merge-file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'distinct --contents against one revision do not collide' ' + warm && + test_write_lines "line 1" "appended line" >c1 && + test_write_lines "rewritten line" >c2 && + # Ground truth without the store. + no_store blame -s --contents=c2 file.txt initial >expect && + # With the store, an intervening c1 run must not poison the c2 lookup. + git blame -s --contents=c1 file.txt initial >/dev/null && + git blame -s --contents=c2 file.txt initial >actual && + test_cmp expect actual && + # The --contents side is a working-tree pseudo-commit (a null commit + # id), so its pairs withhold identity and never consult the store. + # Output parity alone cannot show that: a consulted unwarmed pair + # would miss, not hit, so zero misses is what proves the pair was + # never looked up. + git blame -s --show-stats --contents=c2 file.txt initial >stats 2>&1 && + test_grep "num precomputed hits: 0" stats && + test_grep "num precomputed misses: 0" stats +' + +test_expect_success 'blame --ignore-rev bypasses the store for ignored pairs' ' + git init ignore-rev-repo && + ( + cd ignore-rev-repo && + test_commit ir1 f.txt "base" && + test_commit ir2 f.txt "base +more" && + warm && + # Control: the ordinary pass is served, nothing is computed. + git blame --show-stats f.txt >ctl 2>&1 && + test_grep "num precomputed hits: 1" ctl && + test_grep "num get patch: 0" ctl && + no_store blame --ignore-rev ir2 f.txt >expect && + git blame --ignore-rev ir2 f.txt >actual && + test_cmp expect actual && + # The ignored revision adds a pass that withholds identity: + # it computes its diff (get patch rises) instead of being + # served or even counted as a store consultation. + git blame --ignore-rev ir2 --show-stats f.txt >stats 2>&1 && + test_grep "num precomputed hits: 1" stats && + test_grep "num precomputed misses: 0" stats && + test_grep "num get patch: 1" stats + ) +' + +test_expect_success 'blame counts misses for pairs the store does not hold' ' + ( + cd ignore-rev-repo && + test_commit ir3 f.txt "base +more +third" && + git blame --show-stats f.txt >stats 2>&1 && + test_grep "num precomputed hits: 1" stats && + test_grep "num precomputed misses: 1" stats + ) +' + test_expect_success 'log -L --stat neither reads nor records' ' warm && GIT_TRACE2_EVENT="$PWD/trace_linelog.json" \ @@ -421,6 +683,67 @@ test_expect_success 'log -L --stat neither reads nor records' ' test_path_is_missing $STORE ' +# Integrity: a structurally broken header is read as absent (the reader +# falls back to xdiff and stays correct); a checksum mismatch is caught +# by verify, which is when integrity is checked. +test_expect_success 'a truncated store is read as absent' ' + warm && + test_copy_bytes 20 <$STORE >truncated && + mv truncated $STORE && + no_store blame file.txt >expect && + git blame file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'a corrupt signature is read as absent' ' + warm && + printf "XXXX" >corrupt && + tail -c +5 <$STORE >>corrupt && + mv corrupt $STORE && + no_store blame file.txt >expect && + git blame file.txt >actual && + test_cmp expect actual +' + +# Byte 6 of the header is the chunk count; a value larger than the file +# can hold must be rejected before the chunk table is walked. +test_expect_success 'an over-claimed chunk count is read as absent' ' + warm && + printf "\377" | dd of=$STORE bs=1 seek=6 count=1 conv=notrunc 2>/dev/null && + no_store blame file.txt >expect && + git blame file.txt >actual && + test_cmp expect actual +' + +# A record with no hunks would replay as an equivalence claim, which +# the writer never records; the reader must treat such a record as a +# miss and recompute, and verify must flag it. +test_expect_success 'a zero-hunk record is read as a miss and fails verify' ' + git init zero-hunk && + ( + cd zero-hunk && + test_commit z1 f.txt "base" && + test_commit z2 f.txt "base +more" && + warm && + # The store holds one entry of one hunk: a 4-byte count and + # one 16-byte hunk record, just before the trailing + # checksum. Zero the count to craft the record the writer + # refuses to produce. + rawsz=$(test_oid rawsz) && + fsize=$(test_file_size $STORE) && + printf "\\0\\0\\0\\0" | dd of=$STORE bs=1 \ + seek=$((fsize - rawsz - 20)) count=4 conv=notrunc \ + 2>/dev/null && + no_store blame f.txt >expect && + git blame --show-stats f.txt >stats 2>&1 && + test_grep "num precomputed hits: 0" stats && + git blame f.txt >actual && + test_cmp expect actual && + test_must_fail git diff-hunks verify + ) +' + test_expect_success 'verify succeeds on a valid store and on an absent one' ' warm && git diff-hunks verify && @@ -454,6 +777,38 @@ test_expect_success 'a warm discards a corrupt store rather than seeding from it test_cmp expect actual ' +# A generated patch must carry the builtin diffstat, not one served from +# the sender's local store, so its counts do not depend on whether the +# sender warmed the store. Poison the store so a served answer diverges +# from the builtin, then confirm format-patch shows the builtin counts. +test_expect_success 'format-patch keeps its diffstat off the store' ' + git init fp-repo && + ( + cd fp-repo && + test_commit p1 f.txt "a" && + test_commit p2 f.txt "a +b" && + warm && + # Bump the new-side count of the single recorded hunk. The + # record stays structurally valid, and a read skips the + # trailing checksum, so the store serves this poisoned count. + rawsz=$(test_oid rawsz) && + fsize=$(test_file_size .git/objects/info/diff-hunks) && + printf "\\0\\0\\0\\7" | dd of=.git/objects/info/diff-hunks bs=1 \ + seek=$((fsize - rawsz - 4)) count=4 conv=notrunc 2>/dev/null && + # The store now serves a divergent count, proving the poison + # is live and observable through a store consumer. + printf "7\t0\tf.txt\n" >poisoned && + git log -1 --format= --numstat -- f.txt >served && + test_cmp poisoned served && + # format-patch does not consult the store, so its output is + # identical with the store poisoned and with it disabled. + no_store format-patch -1 --stdout --stat -- f.txt >expect && + git format-patch -1 --stdout --stat -- f.txt >actual && + test_cmp expect actual + ) +' + test_expect_success 'diff-hunks clear removes the store file' ' warm && test_path_is_file $STORE && From 03e089b091988c91efa918b6b60235386216ed5d Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 1 Aug 2026 10:41:50 -0700 Subject: [PATCH 042/259] sub-process: separate process lifecycle from hashmap management subprocess_start() and subprocess_stop() couple two concerns: managing a child process (setup, handshake, teardown) and managing a hashmap that indexes running processes by command string. The hashmap suits callers like convert.c where many files may share one filter process looked up by name, but callers that manage process membership under their own rules do not need the coupled operations. Extract subprocess_start_command() and subprocess_stop_command() so callers can reuse the child process setup and handshake machinery without the map operations. subprocess_start() and subprocess_stop() become thin wrappers that add hashmap operations on top. The diff process support added later in this series keeps its processes in a pool owned by a per-repository provider object, and an entry for a failed command must stay behind there so the command is not retried. That membership follows rules subprocess_start() and subprocess_stop() do not know. The pool therefore uses the _command variants for process lifecycle and manages its own map. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- sub-process.c | 28 +++++++++++++++++++++++----- sub-process.h | 9 ++++++++- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/sub-process.c b/sub-process.c index 2d5c965169727b..3cef42b0880e3f 100644 --- a/sub-process.c +++ b/sub-process.c @@ -49,7 +49,7 @@ int subprocess_read_status(int fd, struct strbuf *status) return (len < 0) ? len : 0; } -void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) +void subprocess_stop_command(struct subprocess_entry *entry) { if (!entry) return; @@ -57,7 +57,14 @@ void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) entry->process.clean_on_exit = 0; kill(entry->process.pid, SIGTERM); finish_command(&entry->process); +} +void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) +{ + if (!entry) + return; + + subprocess_stop_command(entry); hashmap_remove(hashmap, &entry->ent, NULL); } @@ -72,7 +79,7 @@ static void subprocess_exit_handler(struct child_process *process) finish_command(process); } -int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, +int subprocess_start_command(struct subprocess_entry *entry, const char *cmd, subprocess_start_fn startfn) { int err; @@ -96,15 +103,26 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co return err; } - hashmap_entry_init(&entry->ent, strhash(cmd)); - err = startfn(entry); if (err) { error("initialization for subprocess '%s' failed", cmd); - subprocess_stop(hashmap, entry); + subprocess_stop_command(entry); return err; } + return 0; +} + +int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, + subprocess_start_fn startfn) +{ + int err; + + err = subprocess_start_command(entry, cmd, startfn); + if (err) + return err; + + hashmap_entry_init(&entry->ent, strhash(cmd)); hashmap_add(hashmap, &entry->ent); return 0; } diff --git a/sub-process.h b/sub-process.h index bfc3959a1b4894..45f1b8e5e3212f 100644 --- a/sub-process.h +++ b/sub-process.h @@ -52,10 +52,17 @@ int cmd2process_cmp(const void *unused_cmp_data, */ typedef int(*subprocess_start_fn)(struct subprocess_entry *entry); -/* Start a subprocess and add it to the subprocess hashmap. */ +/* Start a subprocess and run the startfn (typically handshake). */ +int subprocess_start_command(struct subprocess_entry *entry, const char *cmd, + subprocess_start_fn startfn); + +/* Start a subprocess, run startfn, and add it to the subprocess hashmap. */ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, subprocess_start_fn startfn); +/* Kill a subprocess. */ +void subprocess_stop_command(struct subprocess_entry *entry); + /* Kill a subprocess and remove it from the subprocess hashmap. */ void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry); From 1a2aca2be48f9ea3f54005bbc7a65e087b34937a Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 1 Aug 2026 10:41:51 -0700 Subject: [PATCH 043/259] sub-process: add a gentle status read subprocess_read_status() reads "status=" packets up to a flush with packet_read_line_gently(), which is gentle only about EOF. A malformed length header still dies inside pkt-line, and an empty packet is indistinguishable from the flush that ends the section. A protocol violation in a status section therefore either kills the whole command or silently truncates the section. That posture fits the filter protocol's callers, which treat their process as required infrastructure; the diff process consult added later in this series treats its process as optional, and any protocol error must degrade to the builtin diff rather than abort the command. Add subprocess_read_status_gently(): the same status loop, reading through packet_read_with_status() with the gentle options, returning -1 on a truncated or malformed packet and on an empty packet where a status line or the terminating flush belongs. subprocess_read_status() and its callers are unchanged. The handshake has its gentle counterpart in 061a68e443 (sub-process: use gentle handshake to avoid die() on startup failure, 2026-06-01), which turned truncated handshake reads into error returns for every caller. This series' base includes that commit, so a process that dies during the handshake feeds the same non-fatal fallback as a status failure here, and an optional diff process degrades to the builtin diff on either kind of protocol error. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- sub-process.c | 24 ++++++++++++++++++++++++ sub-process.h | 10 ++++++++++ 2 files changed, 34 insertions(+) diff --git a/sub-process.c b/sub-process.c index 3cef42b0880e3f..33bd78961827bb 100644 --- a/sub-process.c +++ b/sub-process.c @@ -49,6 +49,30 @@ int subprocess_read_status(int fd, struct strbuf *status) return (len < 0) ? len : 0; } +int subprocess_read_status_gently(int fd, struct strbuf *status) +{ + for (;;) { + int pktlen = -1; + enum packet_read_status rs; + const char *value; + + rs = packet_read_with_status(fd, NULL, NULL, packet_buffer, + sizeof(packet_buffer), &pktlen, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF | + PACKET_READ_GENTLE_ON_READ_ERROR); + if (rs == PACKET_READ_FLUSH) + return 0; + if (rs != PACKET_READ_NORMAL || !pktlen) + return -1; + if (skip_prefix(packet_buffer, "status=", &value)) { + /* the last "status=" line wins */ + strbuf_reset(status); + strbuf_addstr(status, value); + } + } +} + void subprocess_stop_command(struct subprocess_entry *entry) { if (!entry) diff --git a/sub-process.h b/sub-process.h index 45f1b8e5e3212f..8655b388972d64 100644 --- a/sub-process.h +++ b/sub-process.h @@ -101,4 +101,14 @@ int subprocess_handshake(struct subprocess_entry *entry, int subprocess_read_status(int fd, struct strbuf *status); +/* + * Like subprocess_read_status(), but a malformed status section fails + * instead of dying: a truncated or malformed packet, and an empty + * packet where a status line or the terminating flush belongs, return + * -1 and leave the stream unusable. subprocess_read_status() cannot + * tell an empty packet from the flush that ends the section, and dies + * on a framing error inside packet_read_line_gently(). + */ +int subprocess_read_status_gently(int fd, struct strbuf *status); + #endif From ffd18fb3ae7ed5010dc6dc9d1f2ad54f2bcb6a39 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 1 Aug 2026 10:41:52 -0700 Subject: [PATCH 044/259] userdiff: add diff..process config Add the process field to struct userdiff_driver and teach the config parser to populate it from diff..process. The field names a long-running hunk provider process. Nothing reads it yet: the consult, the protocol, and the documentation arrive with the next commit, which starts and pools processes keyed by this field's command string. Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- userdiff.c | 7 +++++++ userdiff.h | 2 ++ 2 files changed, 9 insertions(+) diff --git a/userdiff.c b/userdiff.c index b5412e6bc3ecd3..7547874aa2569d 100644 --- a/userdiff.c +++ b/userdiff.c @@ -509,6 +509,13 @@ int userdiff_config(const char *k, const char *v) drv->algorithm = drv->algorithm_owned; return ret; } + if (!strcmp(type, "process")) { + int ret; + FREE_AND_NULL(drv->process_owned); + ret = git_config_string(&drv->process_owned, k, v); + drv->process = drv->process_owned; + return ret; + } return 0; } diff --git a/userdiff.h b/userdiff.h index 827361b0bc9569..51c26e0d4190e5 100644 --- a/userdiff.h +++ b/userdiff.h @@ -31,6 +31,8 @@ struct userdiff_driver { char *textconv_owned; struct notes_cache *textconv_cache; int textconv_want_cache; + const char *process; + char *process_owned; }; enum userdiff_driver_type { USERDIFF_DRIVER_TYPE_BUILTIN = 1<<0, From 91c8989d4e075a7760f2115f2860fbb370813b24 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Sat, 1 Aug 2026 10:41:53 -0700 Subject: [PATCH 045/259] diff: consult oid-only hunk providers via diff..process The provider chain so far holds the diff-hunks store in front of the terminal builtin computation. Open it to external processes: a pair on a path whose driver configures diff..process is answered by a long-running process speaking a pkt-line protocol (following the filter process protocol), registered at the head of the chain and consulted before the store and before any blob is loaded. The protocol starts with the smallest request that can carry an answer: object names alone. A request is the pathname and the pair's old-oid/new-oid, with no content. The process answers with hunk lines, with a zero-hunk success that asserts the blobs equivalent (trailing newlines included), or with status=need-content, on which the pair falls through to the builtin answer. This serves the two shapes that need no content pushed to them: a cache keyed on the blob pair, and a process that fetches the blobs itself (for example over "git cat-file --batch"). A pair whose side is not a stored blob carries a NULL id; the provider sends no request and passes it. Because Git holds no content for the exchange, the answer is used as sent: hunks are validated for order, overlap, lockstep alignment, and magnitude, then replayed without the normalization xdiff applies to diffs it computes itself. The magnitude bound is the blobs' sizes, read from the object database without loading content: a blob of N bytes holds at most N lines. Because the process's answer is authoritative, it outranks the store, and its head-of-chain position says so. A pair the process answers never reaches the store and is never recorded, so nothing it produces enters the store, which holds the builtin answer only. A request it does not answer, whether need-content, a missing capability, or a missing id, passes down the chain to the builtin answer, which is what the store serves, so the store may serve such a pair and a warming run may record it. Entries recorded before a process was configured are not purged; a pair the process answers ignores them, and "git diff-hunks clear" discards them. The provider gates itself per request. The driver is looked up by the old-side path, so a renamed file resolves to the same driver, and by the repository-relative path, so a diff.relative run from a subdirectory names the pair the same way. Options the process is never told about select no process: the whitespace-ignoring options, -I, --anchored, and an algorithm forced by option or configuration (blame routes its algorithm through xdl_opts, so --histogram is covered). The request gains its last field, the path; the consumers change only by filling it, and neither names the process. The provider's state is its repository's pool of running processes, keyed by the configured command, so drivers sharing a command share a process, a submodule speaks to its own, and releasing the provider (from repo_clear()) stops them. The pool owns a copy of each command string, so an entry outlives a config re-read. A command that fails stays as an entry that is not retried: its request and every later one pass, so the store may serve the path for the rest of the command. A protocol error in a response never kills the command. The response is read through a packet reader gentle about framing, so an error takes one path: a single warning, the process stopped and marked failed, and the builtin diff for the rest of the command. That covers garbage bytes, a truncated response, an empty packet, a bare status, and an unrecognized status. Semantically invalid coordinates cost only their pair: the response is drained, the pair is computed, and the process stays alive. A path the protocol cannot carry (an embedded newline, or one too long for a packet) falls back per path rather than costing the command its process. The handshake keeps one fatal check: a process that announces a capability Git did not request aborts the command, as the long-running filter protocol does. Consulting is allowed per command, following the allow_textconv precedent. "git diff", "git log" and "git show", and "git blame" set allow_diff_process; the plumbing diff commands and the interactive-patch machinery never set it, so scripted and staging output stays builtin. The options adjust the flag: - --no-ext-diff clears it and --ext-diff sets it; - --diff-process and --no-diff-process set and clear it alone, leaving external diff drivers as they were; - format-patch clears it unconditionally, so a generated patch applies for recipients without the process; - range-diff passes --no-ext-diff to the "git log" it compares. git blame and the summary formats consult the process. For blame, a pair reported equivalent emits no hunks, so the whole commit passes to its parent. In the stat formats such a pair sums to a zero-count entry, which the "nothing changed" rule omits, as under -w. The subprocess is long-running: one startup cost across a traversal, one round-trip per consulted pair. Answers travel in struct xdl_hunk, new in xdiff-interface.h, holding xdiff's 1-based coordinates; nothing feeds them back to xdiff, since only coordinate consumers consult. A content-carrying request is the natural extension: it would serve sides that are not stored blobs and processes that want content pushed to them, and bring patch output and log -L's range tracking to the same answer. As it stands, a process's answers show in blame and the summary formats while patch output stays builtin. t4080 exercises the protocol, the per-command gate, and the error paths: - each adversarial response shape warns and falls back to builtin, the request log proving which failures disable the process and which keep it alive (a malformed hunk line, coordinates past the blob size, a count overflowing strtol(), overlapping or misaligned hunks, an unrecognized status, a bare status, an empty packet, a mid-response crash, and raw garbage); - a capability-less process and status=abort degrade without noise, and a failed start warns once and returns the path to the store; - a trailing token on a hunk line is ignored, pinning field appendability; - positive consults for git diff, git show, and diff-tree under --ext-diff and --diff-process; textconv output and gitlink sides are never identified; a diff.relative run consults by the repo-relative path; - the equivalence answer is pinned from both consumers, and a warming run past a deferring process records the pair for a later read. Helped-by: Johannes Schindelin Signed-off-by: Michael Montalbo Signed-off-by: Junio C Hamano --- Documentation/config/diff.adoc | 6 + Documentation/diff-algorithm-option.adoc | 3 + Documentation/diff-options.adoc | 15 +- Documentation/gitattributes.adoc | 160 ++++++ Makefile | 2 + blame.c | 10 +- builtin/blame.c | 1 + builtin/diff.c | 1 + builtin/log.c | 11 +- diff-process.c | 669 +++++++++++++++++++++++ diff-provider-internal.h | 1 + diff-provider.c | 12 +- diff-provider.h | 39 +- diff.c | 42 +- diff.h | 9 + meson.build | 1 + range-diff.c | 6 + t/helper/meson.build | 1 + t/helper/test-diff-process-backend.c | 349 ++++++++++++ t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + t/meson.build | 1 + t/t4080-diff-process.sh | 593 ++++++++++++++++++++ xdiff-interface.h | 12 + 24 files changed, 1916 insertions(+), 30 deletions(-) create mode 100644 diff-process.c create mode 100644 t/helper/test-diff-process-backend.c create mode 100755 t/t4080-diff-process.sh diff --git a/Documentation/config/diff.adoc b/Documentation/config/diff.adoc index 1135a62a0ad3de..349bdbe4927b6e 100644 --- a/Documentation/config/diff.adoc +++ b/Documentation/config/diff.adoc @@ -218,6 +218,12 @@ endif::git-diff[] Set this option to `true` to make the diff driver cache the text conversion outputs. See linkgit:gitattributes[5] for details. +`diff..process`:: + The command to run as a long-running process that answers + which line ranges changed between two blobs. See + linkgit:gitattributes[5] for the protocol and when it is + consulted. + `diff.indentHeuristic`:: Set this option to `false` to disable the default heuristics that shift diff hunk boundaries to make patches easier to read. diff --git a/Documentation/diff-algorithm-option.adoc b/Documentation/diff-algorithm-option.adoc index 8e3a0b63d784d8..16e6fc7261d093 100644 --- a/Documentation/diff-algorithm-option.adoc +++ b/Documentation/diff-algorithm-option.adoc @@ -18,3 +18,6 @@ For instance, if you configured the `diff.algorithm` variable to a non-default value and want to use the default one, then you have to use `--diff-algorithm=default` option. ++ +Explicitly choosing a diff algorithm on the command line also +bypasses `diff..process` (see linkgit:gitattributes[5]). diff --git a/Documentation/diff-options.adoc b/Documentation/diff-options.adoc index c8242e24627eef..cf359d88b4ca53 100644 --- a/Documentation/diff-options.adoc +++ b/Documentation/diff-options.adoc @@ -833,7 +833,20 @@ endif::git-format-patch[] to use this option with linkgit:git-log[1] and friends. `--no-ext-diff`:: - Disallow external diff drivers. + Disallow external diff drivers and processes, including + `diff..command` and `diff..process` + (see linkgit:gitattributes[5]). + +`--diff-process`:: +`--no-diff-process`:: + Allow (or forbid) consulting a diff process configured with + ++diff.++____++.process++ (see linkgit:gitattributes[5]), + leaving external diff drivers unaffected. `git diff`, `git log`, + `git show`, and `git blame` allow consulting by default; the + plumbing diff commands forbid it unless this option or + `--ext-diff` is given. linkgit:git-format-patch[1] accepts the + option but ignores it: a generated patch is always based on the + builtin diff. `--textconv`:: `--no-textconv`:: diff --git a/Documentation/gitattributes.adoc b/Documentation/gitattributes.adoc index da773e29247c46..dd4fa0ad2d189a 100644 --- a/Documentation/gitattributes.adoc +++ b/Documentation/gitattributes.adoc @@ -832,6 +832,166 @@ NOTE: If `diff..command` is defined for path with the (see above), and adding `diff..algorithm` has no effect, as the algorithm is not passed to the external diff driver. +Answering diffs from a long-running process +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Unlike `diff..command`, which replaces the textual patch, the +process configured in `diff..process` feeds hunks back into +Git's own machinery: +it answers "which line ranges changed between these two blobs", and +Git's output is produced from that answer. A process is started +lazily, once per configured command string and per repository, and +consulted over a pkt-line protocol (following the long-running filter +process protocol; see the "Long Running Filter Process" section above +for the filter analogue). + +The process is asked by object names alone: a request carries the +pathname and the `old-oid`/`new-oid` of the blob pair, and no content. +The pathname is the repository-relative old-side (preimage) path, not +the shortened display path a `--relative` diff shows, so a driver +scoped to a directory matches whatever directory the command runs from. +This suits a process that keeps a persistent cache keyed on the pair, and +a process that fetches the blobs itself (for example via +`git cat-file --batch`) to compute its own notion of the changed +lines. Pairs with a side that is not a stored blob (a working-tree +file, textconv output) are not sent; Git computes those itself. + +The exchange opens with a handshake: Git announces its role and +version, the process replies in kind, and then Git lists the +capabilities it supports and the process replies with the ones it +implements. A process that announces a capability Git did not list +aborts the command. + +----------------------- +packet: git> git-diff-client +packet: git> version=1 +packet: git> 0000 +packet: git< git-diff-server +packet: git< version=1 +packet: git< 0000 +packet: git> capability=hunks-by-oid +packet: git> 0000 +packet: git< capability=hunks-by-oid +packet: git< 0000 +----------------------- + +After the handshake, each request and response looks like: + +----------------------- +packet: git> command=hunks-by-oid +packet: git> pathname=path/file.c +packet: git> old-oid= +packet: git> new-oid= +packet: git> 0000 +packet: git< hunk +packet: git< 0000 +packet: git< status=success +packet: git< 0000 +----------------------- + +Start values are 1-based and counts are non-negative; a count of 0 +describes a pure insertion or deletion at the 1-based line the change +sits before (a start of 0 is accepted for an empty file side). Hunks +must be listed in order, must not overlap, and must keep the unchanged +runs between them the same length on both sides; Git validates this +and, with a warning, falls back to its builtin diff on a response +that violates these rules. + +A `status=success` response with zero hunks asserts that the blobs are +equivalent, including their trailing newlines. A process that cannot +answer a pair from its object names (or cannot rule out a +trailing-newline-only difference) responds `status=need-content`, and +Git produces that pair's diff itself. An asserted equivalence makes +the pair vanish from the summary formats, but the pair still counts +as changed for `--exit-code`, the same way a whitespace-only pair +does under `-w`. + +The status names the disposition of the whole request. Git +understands three: `success` (the hunk lines are the answer), +`need-content` (Git produces this pair's diff itself), and `abort`, which +withdraws the capability the request used: Git stops sending +`hunks-by-oid` requests to that process for the rest of the command, +while the process stays alive for request forms negotiated under +other capabilities. Any other status is a protocol error: Git warns, +stops the process, and uses the builtin diff for the remainder of the +command. + +Every response has the same shape whatever its status: zero or more +hunk lines, a flush packet, and a status packet terminated with a +flush packet. A response that carries no hunks, `need-content` +included, still begins with the empty hunk section's flush packet; a +bare status packet is a protocol error: + +----------------------- +packet: git< 0000 +packet: git< status=need-content +packet: git< 0000 +----------------------- + +The process must read the entire request before it responds; Git +writes the whole request before it reads the response. + +The protocol extends without breaking deployed processes: a process +must ignore request keys it does not recognize, and Git ignores +trailing space-separated tokens after the last field of a hunk line, +so a later protocol version can append request keys and hunk fields. +New request forms arrive as capabilities, which a process may decline +to announce; announcing a capability Git did not request aborts the +command, as it does under the long-running filter process protocol. + +There is no shutdown handshake: Git's side of the pipes closes when +the command exits, and the process should exit when it reads EOF. No +flush point is guaranteed, so a process that maintains persistent +state (such as a cache) should persist as it answers rather than at +exit. Git applies no timeout to a response; a process that hangs +hangs the command, as with the long-running filter processes. + +`git blame` and the `--stat`, `--numstat`, and `--shortstat` formats +consult the process; the textual patch and `git log -L` range +tracking are produced by the builtin machinery, so a process whose +answers deliberately differ from the builtin diff shows that +difference only in blame and those formats. `--dirstat=lines` routes +through the diffstat path and consults; the other `--dirstat` modes do +not. A merge's `--stat` (including under `--cc`) is computed against +the first parent, so it consults like any other stat; the combined +patch itself compares one merge result against all of its parents at +once, which the pairwise request above does not express, so that patch +uses the builtin diff, and extending the protocol to combined diffs is +left for future work. A content-carrying extension of this +protocol would bring patch output and `git log -L` range tracking to +the same answer. + +Consulting is allowed per command, as with textconv: `git diff`, +`git log` (`git whatchanged` included) and `git show`, and +`git blame` consult a configured process; the plumbing diff commands +do not unless `--ext-diff` or `--diff-process` is given explicitly, +and the interactive-patch commands (`git add -p` and friends), which +build the hunks they present from plumbing output, always stage from +the builtin diff. `git range-diff` generates the patches it +compares with `--no-ext-diff`. `--diff-process` and +`--no-diff-process` allow or forbid only the consulting; +`--no-ext-diff` disables all external diff mechanisms, this one +included. Options the process is never told about never select it: +with the whitespace-ignoring options, `--ignore-matching-lines`, and +`--anchored`, the pair is answered as when no process is configured. +`--diff-algorithm` (or a configured `diff.algorithm`) forces a builtin +algorithm and bypasses the process the same way. A per-path +++diff.++____++.algorithm++ does so for `git diff` and the stat +formats, which build their diff parameters from it; `git blame` builds +its parameters from its own diff options, so a per-driver algorithm +does not by itself keep blame from consulting the process. +`git format-patch` never +consults the process, so generated patches are always based on the +builtin diff and apply for recipients without the process. On a +path whose driver has a process, the process is consulted before the +diff-hunks store (see linkgit:git-diff-hunks[1]): a pair the process +answers is never served from the store and never recorded into it. +A pair the process does not answer, for example with +`status=need-content`, gets the builtin diff, so the store may serve +it and a warming run may record it: the store holds builtin results, +and for such a pair the builtin result is what would be computed +anyway. + Defining a custom hunk-header ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Makefile b/Makefile index 11a06934b3d350..853e3cdacd5731 100644 --- a/Makefile +++ b/Makefile @@ -819,6 +819,7 @@ TEST_BUILTINS_OBJS += test-csprng.o TEST_BUILTINS_OBJS += test-date.o TEST_BUILTINS_OBJS += test-delete-gpgsig.o TEST_BUILTINS_OBJS += test-delta.o +TEST_BUILTINS_OBJS += test-diff-process-backend.o TEST_BUILTINS_OBJS += test-dir-iterator.o TEST_BUILTINS_OBJS += test-drop-caches.o TEST_BUILTINS_OBJS += test-dump-cache-tree.o @@ -1151,6 +1152,7 @@ LIB_OBJS += diff-delta.o LIB_OBJS += diff-merges.o LIB_OBJS += diff-lib.o LIB_OBJS += diff-no-index.o +LIB_OBJS += diff-process.o LIB_OBJS += diff-provider.o LIB_OBJS += diff.o LIB_OBJS += diffcore-break.o diff --git a/blame.c b/blame.c index 7d7671ef5dd128..3189bcdabaa886 100644 --- a/blame.c +++ b/blame.c @@ -2010,17 +2010,25 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, * compute. The working-tree/--contents pseudo-commit (marked by * its null commit id) holds a blob that is not a stored object, * so its pairs withhold identity too: no id may be sent that - * names bytes a provider cannot look up. + * names bytes a process cannot look up. */ provider_usable = !sb->reverse && !ignore_diffs && !is_null_oid(&target->commit->object.oid) && !blame_textconv_active(sb, target->path) && !blame_textconv_active(sb, parent->path); + /* + * Look up the driver by the parent (old) path, as builtin_diff() + * does with name_a, so a renamed file resolves to the same driver + * across diff and blame. A process that reports a pair + * equivalent emits no hunks, so blame passes the whole commit + * through and looks past it. + */ if (provider_usable) { req.old_oid = &parent->blob_oid; req.new_oid = &target->blob_oid; } + req.path = parent->path; req.diffopt = &sb->revs->diffopt; if (diff_provider_emit_hunks(&req, blame_diff_fill, &fill_data, blame_chunk_cb, &d) == DIFF_PROVIDER_ERROR) diff --git a/builtin/blame.c b/builtin/blame.c index 7891d82ae6b093..13e3ff9e36a50e 100644 --- a/builtin/blame.c +++ b/builtin/blame.c @@ -1025,6 +1025,7 @@ int cmd_blame(int argc, repo_init_revisions(the_repository, &revs, NULL); revs.date_mode = blame_date_mode; revs.diffopt.flags.allow_textconv = 1; + revs.diffopt.flags.allow_diff_process = 1; revs.diffopt.flags.follow_renames = 1; save_commit_buffer = 0; diff --git a/builtin/diff.c b/builtin/diff.c index a2ad63ac8c4b3e..a39ffe69a48a5c 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -510,6 +510,7 @@ int cmd_diff(int argc, init_diffstat_widths(&rev.diffopt); rev.diffopt.flags.allow_external = 1; rev.diffopt.flags.allow_textconv = 1; + rev.diffopt.flags.allow_diff_process = 1; /* If this is a no-index diff, just run it and exit there. */ if (no_index) diff --git a/builtin/log.c b/builtin/log.c index 0b72477f56ee2a..59ecd34334b3bd 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -209,6 +209,7 @@ static void cmd_log_init_defaults(struct rev_info *rev, init_diffstat_widths(&rev->diffopt); rev->diffopt.flags.recursive = 1; rev->diffopt.flags.allow_textconv = 1; + rev->diffopt.flags.allow_diff_process = 1; rev->abbrev_commit = cfg->default_abbrev_commit; rev->show_root_diff = cfg->default_show_root; rev->subject_prefix = cfg->fmt_patch_subject_prefix; @@ -2226,11 +2227,15 @@ int cmd_format_patch(int argc, die(_("unrecognized argument: %s"), argv[1]); /* - * A patch generated by format-patch carries the builtin diffstat, - * not one served from a local store, so its counts do not depend - * on whether the sender warmed the store. + * Patches generated by format-patch must be based on the builtin + * diff so recipients without the store or the process can apply + * them, and so the emitted diffstat does not depend on the sender's + * local cache: the precomputed-hunks store is not consulted for the + * diffstat, and the diff process is not consulted even when + * --ext-diff enables the external diff command. */ rev.diffopt.flags.no_precomputed_hunks = 1; + rev.diffopt.flags.allow_diff_process = 0; if (rev.diffopt.output_format & DIFF_FORMAT_NAME) die(_("--name-only does not make sense")); diff --git a/diff-process.c b/diff-process.c new file mode 100644 index 00000000000000..121903a6c80256 --- /dev/null +++ b/diff-process.c @@ -0,0 +1,669 @@ +/* + * The process provider of the hunk provider interface: consult a + * long-running external process via the pkt-line protocol for the + * hunks of a blob pair. The process answers from the pair's object + * names alone: it can serve a persistent cache keyed on the pair, or + * fetch the blobs from the repository itself (e.g. via "git cat-file + * --batch") and compute its own notion of which lines changed. The + * provider sits at the head of its repository's chain and gates + * itself per request; its state is the repository's pool of running + * processes, one per configured command, stopped when the provider + * is released. + * + * Protocol: pkt-line over stdin/stdout, following the pattern of + * the long-running filter process protocol (see convert.c). + * + * Handshake: + * git> git-diff-client / version=1 / flush + * process< git-diff-server / version=1 / flush + * git> capability=hunks-by-oid / flush + * process< capability=hunks-by-oid / flush + * + * Per-pair, when both sides are stored blobs: + * git> command=hunks-by-oid / pathname= + * git> old-oid= / new-oid= / flush + * process< hunk + * process< ... / flush + * process< status=success / flush + * + * No content is sent. Because Git holds no content for the exchange, + * the answer is used as the process sent it: the hunks are not re-run + * through xdiff's compaction, and a status=success response with zero + * hunks asserts that the blobs are equivalent, including their + * trailing newlines. A process that cannot answer from the object names + * (or cannot rule out a trailing-newline-only difference) responds + * status=need-content; the pair then gets the builtin answer, served + * from the diff-hunks store or computed. A later + * protocol extension can define a content-carrying request for such + * processes and for sides that are not stored blobs. + */ + +#include "git-compat-util.h" +#include "diff.h" +#include "diff-provider-internal.h" +#include "gettext.h" +#include "hex.h" +#include "odb.h" +#include "repository.h" +#include "sigchain.h" +#include "userdiff.h" +#include "sub-process.h" +#include "pkt-line.h" +#include "strbuf.h" + +#define CAP_OID_HUNKS (1u << 0) + +/* + * The provider's state: the repository's diff processes, keyed by + * their command string, so drivers that configure the same command + * share one process. An entry whose process failed stays in the + * pool with the failed bit set, so the command is not retried while + * the entry lives; the pool and its entries last until the provider + * is released. + */ +struct diff_process_state { + struct hashmap subprocesses; +}; + +struct diff_subprocess { + struct subprocess_entry subprocess; + /* + * Owns the string subprocess.cmd and the hashmap key borrow: the + * entry outlives the userdiff config a re-read may replace. + */ + char *cmd; + unsigned int supported_capabilities; + unsigned failed : 1; +}; + +static int start_diff_process_fn(struct subprocess_entry *subprocess) +{ + static int versions[] = { 1, 0 }; + static struct subprocess_capability capabilities[] = { + { "hunks-by-oid", CAP_OID_HUNKS }, + { NULL, 0 } + }; + struct diff_subprocess *entry = + container_of(subprocess, struct diff_subprocess, subprocess); + + return subprocess_handshake(subprocess, "git-diff", + versions, NULL, + capabilities, + &entry->supported_capabilities); +} + +/* + * The pool entry for a command, or NULL when its process fails to + * start here: the failure leaves a failed entry in the pool, so only + * the request that observed it maps it to an error and later + * requests pass the provider by. + */ +static struct diff_subprocess *get_or_launch_process( + struct diff_process_state *state, + struct userdiff_driver *drv) +{ + struct subprocess_entry *running; + struct diff_subprocess *entry; + + running = subprocess_find_entry(&state->subprocesses, drv->process); + if (running) { + entry = container_of(running, struct diff_subprocess, + subprocess); + return entry->failed ? NULL : entry; + } + + entry = xcalloc(1, sizeof(*entry)); + entry->cmd = xstrdup(drv->process); + if (subprocess_start_command(&entry->subprocess, entry->cmd, + start_diff_process_fn)) + entry->failed = 1; + hashmap_entry_init(&entry->subprocess.ent, strhash(entry->cmd)); + hashmap_add(&state->subprocesses, &entry->subprocess.ent); + if (entry->failed) { + warning(_("diff process '%s' failed to start;" + " using the builtin diff"), drv->process); + return NULL; + } + return entry; +} + +/* + * A hunk in the diff process's presentation coordinates: the line + * numbering it reports over the protocol. Kept distinct from struct + * xdl_hunk (xdiff's coordinates) so that only translated hunks ever + * reach a consumer; diff_process_hunk_to_xdl() is the single + * crossing point. + */ +struct diff_process_hunk { + long old_start, old_count; + long new_start, new_count; +}; + +/* + * Parse one non-negative decimal field of a hunk line into *out and + * advance *line past it. Fields must be plain decimal with no leading + * whitespace or sign (isdigit() takes an unsigned char to stay defined + * for high-bit bytes). The first three fields are followed by a single + * space; the last (is_last) is followed by end-of-string or a space. + * Trailing space-separated tokens after the last field are allowed and + * ignored, so a future protocol version can append fields (e.g. a + * "moved" marker) without an older Git rejecting the line, mirroring + * the request-side rule that processes ignore unknown keys. + * + * A value that overflows strtol() is not a parse failure: the line is + * well-formed, so the stream stays in protocol sync. It is reported + * through *out_of_range, and the caller skips the pair the same way + * it skips any other out-of-range coordinate. + */ +static int parse_hunk_field(const char **line, long *out, int is_last, + int *out_of_range) +{ + const char *p = *line; + char *end; + + if (!isdigit((unsigned char)*p)) + return -1; + errno = 0; + *out = strtol(p, &end, 10); + if (end == p) + return -1; + if (errno == ERANGE) + *out_of_range = 1; + else if (errno) + return -1; + if (is_last) { + if (*end != '\0' && *end != ' ') + return -1; + } else { + if (*end != ' ') + return -1; + end++; + } + *line = end; + return 0; +} + +static int parse_hunk_line(const char *line, + struct diff_process_hunk *presented, + int *out_of_range) +{ + *out_of_range = 0; + /* Format: "hunk " */ + if (!skip_prefix(line, "hunk ", &line)) + return -1; + if (parse_hunk_field(&line, &presented->old_start, 0, out_of_range) || + parse_hunk_field(&line, &presented->old_count, 0, out_of_range) || + parse_hunk_field(&line, &presented->new_start, 0, out_of_range) || + parse_hunk_field(&line, &presented->new_count, 1, out_of_range)) + return -1; + return 0; +} + +/* + * Translate a hunk from the diff process's presentation coordinates + * into xdiff's. + * + * Protocol starts are already 1-based positions (the line a change + * sits before), the same numbering xdiff uses, so the only adjustment + * is for an empty file side: "git diff" addresses it with a start of 0 + * and a count of 0 (e.g. "0 0 1 5" adds five lines to an empty old + * side), and since xdiff uses start-1 as an array index that 0 becomes + * 1 here. This is NOT the full inverse of xdl_emit_hunk_hdr() + * (xdiff/xutils.c): that emitter shifts a count-0 range to start-1 for + * the displayed "@@" header, but the protocol keeps the unshifted + * 1-based position for a mid-file insert or delete. This is the single + * point where presentation coordinates become xdiff coordinates, so + * any consumer of these coordinates may assume 1-based starts. + * + * Returns -1 for a start of 0 paired with a nonzero count, which names + * no line in either coordinate system. (parse_hunk_line() already + * guarantees non-negative starts and counts.) + */ +static int diff_process_hunk_to_xdl(const struct diff_process_hunk *presented, + struct xdl_hunk *xdl) +{ + long old_start = presented->old_start; + long new_start = presented->new_start; + + if ((!old_start && presented->old_count) || + (!new_start && presented->new_count)) + return -1; + if (!old_start) + old_start = 1; + if (!new_start) + new_start = 1; + + xdl->old_start = old_start; + xdl->old_count = presented->old_count; + xdl->new_start = new_start; + xdl->new_count = presented->new_count; + return 0; +} + +/* + * Validate the process's hunks (already in xdiff coordinates) before they + * bypass the diff algorithm. The content-independent rules (in-order, + * non-overlapping, lockstep-aligned, int32-bounded coordinates) are the + * provider interface's shared rule, diff_provider_check_hunk(); this + * function adds the two checks that need the blobs' line counts (a hunk + * past the end of a file, the run after the last hunk) and the + * per-rule diagnostics naming the process. On a bad response we warn + * and the caller falls back to the builtin diff. Returns 0 if valid, + * -1 (after warning) otherwise. + * + * old_lines/new_lines bound the line count of each side, or are + * negative when no bound is known. An oid-only answer arrives without + * content, so its caller passes upper bounds derived from the blobs' + * byte sizes, which caps coordinate magnitude but cannot support the + * run-after-the-last-hunk check: that one compares exact line counts, + * so it runs only when lines_exact is set, which no caller does today. + * It is kept for a content-carrying request, whose loaded buffers + * would provide exact counts. + */ +static int validate_external_hunks(const struct xdl_hunk *hunks, size_t nr, + long old_lines, long new_lines, + int lines_exact, + const char *process, const char *path) +{ + struct diff_provider_hunks_check c = { 0 }; + size_t i; + + for (i = 0; i < nr; i++) { + const struct xdl_hunk *h = &hunks[i]; + + if (old_lines >= 0 && + (h->old_count > old_lines - h->old_start + 1 || + h->new_count > new_lines - h->new_start + 1)) { + warning(_("diff process '%s' returned a hunk past the " + "end of '%s'; using the builtin diff"), + process, path); + return -1; + } + switch (diff_provider_check_hunk(&c, h->old_start, + h->old_count, h->new_start, + h->new_count)) { + case DIFF_PROVIDER_HUNKS_OK: + break; + case DIFF_PROVIDER_HUNKS_RANGE: + warning(_("diff process '%s' returned out-of-range " + "coordinates for '%s'; using the builtin diff"), + process, path); + return -1; + case DIFF_PROVIDER_HUNKS_OVERLAP: + warning(_("diff process '%s' returned overlapping hunks " + "for '%s'; using the builtin diff"), + process, path); + return -1; + case DIFF_PROVIDER_HUNKS_MISALIGNED: + warning(_("diff process '%s' returned hunks that leave " + "'%s' misaligned; using the builtin diff"), + process, path); + return -1; + } + } + if (lines_exact && + old_lines - c.prev_old_end != new_lines - c.prev_new_end) { + warning(_("diff process '%s' returned hunks that leave '%s' " + "misaligned; using the builtin diff"), + process, path); + return -1; + } + return 0; +} + +/* + * The most lines a blob can hold, from its size alone: every line, + * even an empty one, costs at least one byte, so a blob of N bytes + * holds at most N lines. Returns -1 when the size is unavailable, + * leaving the response bounded only by the shared int32 rule. A size + * beyond INT32_MAX clamps to it, which loses nothing: a coordinate + * that large fails the shared rule anyway. In a partial clone the + * size lookup must not fetch the blob from the promisor remote: + * validating an answer that exists to avoid loading content must not + * itself download that content, so a missing blob reads as size + * unavailable instead. + */ +static long blob_line_cap(struct repository *r, const struct object_id *oid) +{ + unsigned long size; + struct object_info oi = OBJECT_INFO_INIT; + + oi.sizep = &size; + if (odb_read_object_info_extended(r->objects, oid, &oi, + OBJECT_INFO_SKIP_FETCH_OBJECT) < 0) + return -1; + if (size > INT32_MAX) + return INT32_MAX; + return (long)size; +} + +/* + * The driver whose process a consultation for path would ask, or NULL + * when none applies (no driver, process not allowed, or xpp carries + * options the process is never told about). Needs no content, so + * the driver is picked before any blob is loaded. + */ +static struct userdiff_driver *diff_process_driver(struct diff_options *diffopt, + const char *path, + const xpparam_t *xpp) +{ + struct userdiff_driver *drv; + + if (!diffopt || !path) + return NULL; + if (!diffopt->flags.allow_diff_process || diffopt->ignore_driver_algorithm) + return NULL; + /* + * Whitespace-ignoring, regex-ignore (-I) and anchored options + * change which lines count as different, but the process is never + * told about them, so its hunks could not honor them. A forced + * diff algorithm (an option or configured algorithm setting) + * requests a specific builtin computation, which an + * authoritative answer would override. Rather than silently + * override the user's request, fall back to the builtin diff, + * which does honor these flags. Key this off xpp (the + * parameters this diff actually runs with) rather than diffopt, + * so a caller like blame, which keeps its algorithm and + * whitespace flags outside diffopt, is covered without a + * separate guard of its own. + */ + if ((xpp->flags & (XDF_WHITESPACE_FLAGS | XDF_IGNORE_BLANK_LINES | + XDF_DIFF_ALGORITHM_MASK)) || + xpp->ignore_regex_nr || xpp->anchors_nr) + return NULL; + + /* + * A path the protocol cannot carry never selects a process: an + * embedded newline would let the rest of the path forge further + * request keys, and the pathname must fit one packet. Passing + * here keeps the cost local to the path; a failed write would + * instead cost the whole command its process. + */ + if (strchr(path, '\n') || + strlen(path) > LARGE_PACKET_DATA_MAX - strlen("pathname=\n")) + return NULL; + + drv = userdiff_find_by_path(diffopt->repo->index, path); + if (!drv || !drv->process) + return NULL; + return drv; +} + +/* + * Without content there is no size-derived bound on a response, so cap + * accumulation at a constant instead. A response that exceeds the + * cap is a protocol error: the process is disabled for the rest of + * the command and the caller falls back to the builtin diff. + */ +#define OID_HUNKS_MAX (1 << 20) + +enum diff_process_result { + DIFF_PROCESS_ERROR = -1, /* failed; caller falls back to builtin */ + DIFF_PROCESS_OK = 0, /* the process supplied hunks */ + DIFF_PROCESS_SKIP, /* process did not apply: use builtin */ + DIFF_PROCESS_EQUIVALENT, /* process says files are equivalent */ +}; + +/* + * Ask drv's diff process to answer the request from the blob pair's + * object ids alone (the "hunks-by-oid" capability): no content is + * loaded or sent. On DIFF_PROCESS_OK the process's hunks are emitted + * through hunk_cb in 0-based emission coordinates, validated for order, + * overlap, and lockstep alignment first; because Git holds no content, + * the answer is used as the process sent it, without xdiff's compaction. + * DIFF_PROCESS_EQUIVALENT means the process asserts the pair equal. + * DIFF_PROCESS_SKIP covers everything that should fall through to the + * builtin computation: a missing capability, a missing object id, a + * status=need-content answer, or an invalid response. + */ +static enum diff_process_result diff_process_query_hunks( + struct diff_process_state *state, + struct userdiff_driver *drv, + const struct diff_provider_request *req, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data) +{ + const char *path = req->path; + struct diff_subprocess *entry; + struct child_process *process; + int fd_in, fd_out; + struct packet_reader reader; + struct strbuf status = STRBUF_INIT; + struct xdl_hunk *hunks = NULL; + struct diff_process_hunk presented; + struct xdl_hunk hunk; + size_t nr_hunks = 0, alloc_hunks = 0, i; + int bad_coords = 0; + long old_cap, new_cap; + enum diff_process_result res; + + if (!req->old_oid || !req->new_oid) + return DIFF_PROCESS_SKIP; + + entry = get_or_launch_process(state, drv); + if (!entry) + return DIFF_PROCESS_ERROR; + if (!(entry->supported_capabilities & CAP_OID_HUNKS)) + return DIFF_PROCESS_SKIP; + + process = subprocess_get_child_process(&entry->subprocess); + fd_in = process->in; + fd_out = process->out; + + sigchain_push(SIGPIPE, SIG_IGN); + + if (packet_write_fmt_gently(fd_in, "command=hunks-by-oid\n") || + packet_write_fmt_gently(fd_in, "pathname=%s\n", path) || + packet_write_fmt_gently(fd_in, "old-oid=%s\n", + oid_to_hex(req->old_oid)) || + packet_write_fmt_gently(fd_in, "new-oid=%s\n", + oid_to_hex(req->new_oid)) || + packet_flush_gently(fd_in)) + goto comm_error; + + packet_reader_init(&reader, fd_out, NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF | + PACKET_READ_GENTLE_ON_READ_ERROR); + for (;;) { + enum packet_read_status rs = packet_reader_read(&reader); + int out_of_range; + + if (rs == PACKET_READ_FLUSH) + break; + /* + * Only a hunk line may precede the flush. EOF and a + * malformed frame end the session; an empty packet, which + * a length-only read cannot tell from a flush, would + * truncate the hunk section here and leave the status + * section to poison the next request, so it is a protocol + * error too. + */ + if (rs != PACKET_READ_NORMAL || !reader.pktlen) + goto comm_error; + if (parse_hunk_line(reader.line, &presented, + &out_of_range) < 0) + goto comm_error; + if (bad_coords) + continue; + if (out_of_range || + diff_process_hunk_to_xdl(&presented, &hunk) < 0) { + /* + * Semantically invalid coordinates in a well-formed + * response: the stream stays in protocol sync, so + * drain the rest and fall back for this file while + * keeping the process alive, the same treatment + * validate_external_hunks() failures receive. + */ + bad_coords = 1; + continue; + } + if (nr_hunks >= OID_HUNKS_MAX) { + warning(_("diff process '%s' sent too many hunks" + " for '%s'; disabling it for the" + " remainder of this command"), + drv->process, path); + goto disable; + } + ALLOC_GROW(hunks, nr_hunks + 1, alloc_hunks); + hunks[nr_hunks++] = hunk; + } + + if (subprocess_read_status_gently(fd_out, &status)) + goto comm_error; + + if (!strcmp(status.buf, "success")) { + if (bad_coords) { + warning(_("diff process '%s' returned out-of-range " + "coordinates for '%s'; using the builtin diff"), + drv->process, path); + res = DIFF_PROCESS_SKIP; + goto out; + } + if (!nr_hunks) { + res = DIFF_PROCESS_EQUIVALENT; + goto out; + } + /* + * Bound the coordinates by the blobs' sizes, read from the + * object database without loading content. Either both + * bounds hold or neither is applied: a partial bound would + * misclassify a response that the other side's size would + * have caught. + */ + old_cap = blob_line_cap(req->repo, req->old_oid); + new_cap = blob_line_cap(req->repo, req->new_oid); + if (old_cap < 0 || new_cap < 0) + old_cap = new_cap = -1; + if (validate_external_hunks(hunks, nr_hunks, old_cap, new_cap, + 0, drv->process, path) < 0) { + res = DIFF_PROCESS_SKIP; + goto out; + } + /* + * Replay in the coordinates a hunk consumer receives from + * xdiff's emission: 0-based starts. The answer is used as + * the process sent it; with no content in hand it cannot be + * re-run through xdiff's compaction. + */ + for (i = 0; i < nr_hunks; i++) + hunk_cb(hunks[i].old_start - 1, hunks[i].old_count, + hunks[i].new_start - 1, hunks[i].new_count, + cb_data); + res = DIFF_PROCESS_OK; + goto out; + } + if (!strcmp(status.buf, "need-content")) { + /* + * The process cannot answer this pair from its object names; + * the caller computes the diff itself. + */ + res = DIFF_PROCESS_SKIP; + goto out; + } + if (!strcmp(status.buf, "abort")) { + /* The process withdrew: stop asking it for this session. */ + entry->supported_capabilities &= ~CAP_OID_HUNKS; + res = DIFF_PROCESS_SKIP; + goto out; + } + /* + * An unrecognized status is a protocol error, not a per-pair + * failure: this Git did not request anything it does not know, + * so the process is answering some other protocol, and asking + * it again would warn on every pair of the traversal. + */ + warning(_("diff process '%s' sent unrecognized status '%s' for " + "'%s'; disabling it for the remainder of this command"), + drv->process, status.buf, path); + goto disable; +out: + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return res; + +comm_error: + warning(_("diff process '%s' failed for '%s'; disabling it" + " for the remainder of this command"), + drv->process, path); +disable: + subprocess_stop_command(&entry->subprocess); + entry->failed = 1; + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return DIFF_PROCESS_ERROR; +} + +/* + * The process outranks every later provider through its chain + * position: when it answers, the walk ends, so no later provider + * serves the pair, and an answered pair is never recorded. When it + * does not answer (it defers with need-content, lacks the + * capability, or failed), the caller computes the builtin diff for + * that pair. The store holds builtin results and nothing else, so + * an identity answer for such a pair equals what the caller would + * compute. Every non-answer is therefore a pass: a refusal would + * suppress that equal answer, and would keep a warming run from + * recording the builtin result the caller computes anyway. + */ +static enum diff_provider_disposition +diff_process_consult(struct diff_provider *provider, + const struct diff_provider_request *req, + diff_provider_fill_fn fill UNUSED, void *fill_data UNUSED, + xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data) +{ + struct diff_process_state *state = provider->state; + struct userdiff_driver *drv; + struct subprocess_entry *running; + + drv = diff_process_driver(req->diffopt, req->path, req->xpp); + if (!drv) + return DIFF_PROVIDER_DISP_PASS; + running = subprocess_find_entry(&state->subprocesses, drv->process); + if (running && container_of(running, struct diff_subprocess, + subprocess)->failed) + return DIFF_PROVIDER_DISP_PASS; + + switch (diff_process_query_hunks(state, drv, req, + hunk_cb, cb_data)) { + case DIFF_PROCESS_OK: + case DIFF_PROCESS_EQUIVALENT: + return DIFF_PROVIDER_DISP_ANSWERED; + case DIFF_PROCESS_SKIP: + case DIFF_PROCESS_ERROR: + break; + } + return DIFF_PROVIDER_DISP_PASS; +} + +static void diff_process_release(struct diff_provider *provider) +{ + struct diff_process_state *state = provider->state; + struct hashmap_iter iter; + struct diff_subprocess *entry; + + /* A failed entry's process is already stopped or never ran. */ + hashmap_for_each_entry(&state->subprocesses, &iter, entry, + subprocess.ent) { + if (!entry->failed) + subprocess_stop_command(&entry->subprocess); + free(entry->cmd); + } + hashmap_clear_and_free(&state->subprocesses, + struct diff_subprocess, subprocess.ent); + free(state); +} + +struct diff_provider *diff_process_provider_new(void) +{ + struct diff_process_state *state = xcalloc(1, sizeof(*state)); + struct diff_provider *p = xcalloc(1, sizeof(*p)); + + hashmap_init(&state->subprocesses, cmd2process_cmp, NULL, 0); + p->consult = diff_process_consult; + p->release = diff_process_release; + p->state = state; + return p; +} diff --git a/diff-provider-internal.h b/diff-provider-internal.h index 8ad3e481e7bd35..cba1fa271adf6d 100644 --- a/diff-provider-internal.h +++ b/diff-provider-internal.h @@ -93,6 +93,7 @@ struct diff_provider { * diff-provider.c holds itself. Each call returns a fresh provider * for one repository's chain. */ +struct diff_provider *diff_process_provider_new(void); struct diff_provider *diff_hunks_store_provider_new(void); /* diff --git a/diff-provider.c b/diff-provider.c index 66a9909eaa6943..c8aaf8e857a8c0 100644 --- a/diff-provider.c +++ b/diff-provider.c @@ -41,11 +41,11 @@ static struct diff_provider *builtin_provider_new(void) /* * The repository's chain, assembled on first walk. The composition - * is fixed, and the order is the authority resolution: the store is - * consulted before the builtin computation, the terminal provider, - * so the chain always ends in an implementor that can answer. - * Nothing is decided per repository here; each provider gates itself - * per request. + * is fixed, and the order is the authority resolution: the process + * outranks the store, and the builtin computation is the terminal + * provider, so the chain always ends in an implementor that can + * answer. Nothing is decided per repository here; each provider + * gates itself per request. */ static struct diff_provider *provider_chain(struct repository *r) { @@ -53,6 +53,8 @@ static struct diff_provider *provider_chain(struct repository *r) if (*tail) return *tail; + *tail = diff_process_provider_new(); + tail = &(*tail)->next; *tail = diff_hunks_store_provider_new(); tail = &(*tail)->next; *tail = builtin_provider_new(); diff --git a/diff-provider.h b/diff-provider.h index c5478b570025a2..061e1c2f5c2772 100644 --- a/diff-provider.h +++ b/diff-provider.h @@ -11,19 +11,23 @@ * its content is loaded. * * A hunk provider answers a consumer's request from the pair's - * identity (its blob object ids) and the parameters that determine - * the diff; a request no provider answers falls through to the - * consumer's own computation. A provider is either authoritative for - * its requests, meaning its answer may deliberately differ from the - * builtin diff, or not, meaning its answer must reproduce the builtin - * result exactly. The interface resolves that authority through a - * provider chain owned by the repository, built on first consultation - * and released by repo_clear(): chain order is the resolution, and - * the builtin computation itself is the chain's terminal provider. A - * consumer never names a provider; it reads the outcome below. - * Every answer a provider serves from identity passes the shared - * coordinate check (diff-provider-internal.h) before any consumer - * sees it. + * identity, its blob object ids and the settings that determine the + * diff, before any content is loaded; a request no provider answers + * falls through to the consumer's own computation. Two providers implement this + * interface with different authority. The diff-hunks store + * (diff-hunks.h) is in-process and not authoritative: it may only + * reproduce the builtin result, so it never asserts a pair + * equivalent, and it stands aside wherever a process outranks it. A + * process configured in diff..process (diff-process.c) is + * authoritative for its paths: its answer may deliberately differ + * from the builtin diff, including asserting a pair equivalent. The + * interface resolves that authority through a provider chain owned + * by the repository, built on first consultation and released by + * repo_clear(): chain order is the resolution, and the builtin + * computation itself is the chain's terminal provider. A consumer + * never names a provider; it reads the outcome below. Every answer a + * provider serves from identity passes the shared coordinate check + * (diff-provider-internal.h) before any consumer sees it. */ struct diff_options; @@ -95,14 +99,17 @@ enum diff_provider_outcome { * name the blobs whose bytes are diffed; pass NULL for a side whose * bytes are not a stored blob (a working-tree file, textconv output, * a gitlink), so no provider answers from an id it cannot look up. - * diffopt carries the diff settings that live outside xpp; xpp - * carries the parameters the diff runs with. Each provider gates - * itself on the fields that concern it. + * path names the file the pair is diffed as; a provider selected by + * path applies only where it is set. diffopt carries the diff + * settings that live outside xpp; xpp carries the parameters the + * diff runs with. Each provider gates itself on the fields that + * concern it. */ struct diff_provider_request { struct repository *repo; const struct object_id *old_oid; const struct object_id *new_oid; + const char *path; struct diff_options *diffopt; const xpparam_t *xpp; }; diff --git a/diff.c b/diff.c index 0d4cb3fcfd742a..ea79d80c26eb07 100644 --- a/diff.c +++ b/diff.c @@ -4377,6 +4377,13 @@ static int diffstat_from_hunks(struct diff_options *o, &one->oid : NULL, .new_oid = (two->oid_valid && !S_ISGITLINK(two->mode)) ? &two->oid : NULL, + /* + * Attribute lookup and the process protocol need the + * repo-relative path; the display name a caller passes + * around may be stripped of o->prefix and would miss a + * driver scoped to a directory. + */ + .path = one->path, .diffopt = o, .xpp = &xpp, }; @@ -4491,12 +4498,14 @@ static void builtin_diffstat(const char *name_a, const char *name_b, else if (may_differ) { /* - * Serve or record via the diff-hunks store. A "log -L" + * Serve from a hunk provider (the process, then the store), + * or record into the store on a warming run. A "log -L" * range-scoped stat is not the whole-pair diff the store * keys, so it neither reads nor records. Otherwise diff * normally. */ - if (p->line_ranges || !diffstat_from_hunks(o, one, two, data)) { + if (p->line_ranges || + !diffstat_from_hunks(o, one, two, data)) { /* Crazy xdl interfaces.. */ xpparam_t xpp; xdemitconf_t xecfg; @@ -6252,6 +6261,27 @@ static int diff_opt_submodule(const struct option *opt, return 0; } +static int diff_opt_ext_diff(const struct option *opt, + const char *arg, int unset) +{ + struct diff_options *options = opt->value; + + BUG_ON_OPT_ARG(arg); + options->flags.allow_external = !unset; + options->flags.allow_diff_process = !unset; + return 0; +} + +static int diff_opt_diff_process(const struct option *opt, + const char *arg, int unset) +{ + struct diff_options *options = opt->value; + + BUG_ON_OPT_ARG(arg); + options->flags.allow_diff_process = !unset; + return 0; +} + static int diff_opt_textconv(const struct option *opt, const char *arg, int unset) { @@ -6582,8 +6612,12 @@ struct option *add_diff_options(const struct option *opts, N_("exit with 1 if there were differences, 0 otherwise")), OPT_BOOL(0, "quiet", &options->flags.quick, N_("disable all output of the program")), - OPT_BOOL(0, "ext-diff", &options->flags.allow_external, - N_("allow an external diff helper to be executed")), + OPT_CALLBACK_F(0, "ext-diff", options, NULL, + N_("allow an external diff helper to be executed"), + PARSE_OPT_NOARG, diff_opt_ext_diff), + OPT_CALLBACK_F(0, "diff-process", options, NULL, + N_("allow a configured diff process to be consulted"), + PARSE_OPT_NOARG, diff_opt_diff_process), OPT_CALLBACK_F(0, "textconv", options, NULL, N_("run external text conversion filters when comparing binary files"), PARSE_OPT_NOARG, diff_opt_textconv), diff --git a/diff.h b/diff.h index 380a25887839a2..e0b58a8105fe8f 100644 --- a/diff.h +++ b/diff.h @@ -173,6 +173,15 @@ struct diff_flags { */ unsigned allow_external; + /** + * Allows diff..process to be consulted. Set by the + * porcelain commands whose output may reflect a diff process + * (diff, log, show, blame) and by --ext-diff or --diff-process; + * plumbing does not set it by default, so its output stays + * builtin. Cleared by --no-ext-diff or --no-diff-process. + */ + unsigned allow_diff_process; + /** * For communication between the calling program and the options parser; * tell the calling program to signal the presence of difference using diff --git a/meson.build b/meson.build index 391d9da93c0cf8..6d6ac8e7537975 100644 --- a/meson.build +++ b/meson.build @@ -356,6 +356,7 @@ libgit_sources = [ 'diff-merges.c', 'diff-lib.c', 'diff-no-index.c', + 'diff-process.c', 'diff-provider.c', 'diff.c', 'diffcore-break.c', diff --git a/range-diff.c b/range-diff.c index 8e2dd2eb193eb9..3cadbcd7e1d6cb 100644 --- a/range-diff.c +++ b/range-diff.c @@ -52,6 +52,12 @@ static int read_patches(const char *range, struct string_list *list, int ret = -1; strvec_pushl(&cp.args, "log", "--no-color", "-p", + /* + * The patches being compared must be the builtin + * diff's: an external diff command or diff process + * could change either side of the comparison. + */ + "--no-ext-diff", "--reverse", "--date-order", "--decorate=no", "--no-prefix", "--submodule=short", /* diff --git a/t/helper/meson.build b/t/helper/meson.build index 3235f10ab8aae1..6abcda4afb89c0 100644 --- a/t/helper/meson.build +++ b/t/helper/meson.build @@ -12,6 +12,7 @@ test_tool_sources = [ 'test-date.c', 'test-delete-gpgsig.c', 'test-delta.c', + 'test-diff-process-backend.c', 'test-dir-iterator.c', 'test-drop-caches.c', 'test-dump-cache-tree.c', diff --git a/t/helper/test-diff-process-backend.c b/t/helper/test-diff-process-backend.c new file mode 100644 index 00000000000000..b0bb78d9f34fe8 --- /dev/null +++ b/t/helper/test-diff-process-backend.c @@ -0,0 +1,349 @@ +/* + * Test process implementing the diff process protocol (diff..process). + * + * Speaks the long-running process protocol over stdin/stdout and + * answers command=hunks-by-oid requests from the blob object names + * alone; no content is exchanged. The --mode= switch selects the + * response shape: + * + * oid-fixed packet: git< hunk 5 2 5 2 + * oid-equal packet: git< status=success (zero hunks: equivalent) + * oid-need-content packet: git< status=need-content + * oid-empty packet: git< hunk 0 0 1 2 (empty old side) + * + * and the adversarial shapes the protocol error paths are tested + * with: + * + * oid-trailing a hunk line with a trailing token to ignore + * oid-malformed a hunk line that does not parse + * oid-huge coordinates far past the end of any test blob + * oid-erange a count too large for any long + * oid-overlap two hunks out of order + * oid-misaligned two hunks whose unchanged runs differ in length + * oid-badstart a start of 0 paired with a nonzero count + * oid-unknown-status status=frobnicate + * oid-abort status=abort + * oid-bare-status a status packet without the hunk-section flush + * oid-empty-packet an empty packet (0004) inside the hunk section + * oid-crash one hunk line, then exit with no flush or status + * oid-garbage raw non-pkt-line bytes, then exit + * cap-none handshake announcing no capability at all + * + * Success responses end with: + * + * packet: git< 0000 + * packet: git< status=success + * packet: git< 0000 + * + * Each request is logged to --log as: + * + * command= pathname= old-oid= new-oid= + */ + +#include "test-tool.h" +#include "pkt-line.h" +#include "parse-options.h" +#include "strbuf.h" + +static FILE *logfile; + +enum mode { + MODE_OID_FIXED, + MODE_OID_EQUAL, + MODE_OID_NEED_CONTENT, + MODE_OID_EMPTY, + MODE_OID_TRAILING, + MODE_OID_MALFORMED, + MODE_OID_HUGE, + MODE_OID_ERANGE, + MODE_OID_OVERLAP, + MODE_OID_MISALIGNED, + MODE_OID_BADSTART, + MODE_OID_UNKNOWN_STATUS, + MODE_OID_ABORT, + MODE_OID_BARE_STATUS, + MODE_OID_EMPTY_PACKET, + MODE_OID_CRASH, + MODE_OID_GARBAGE, + MODE_CAP_NONE, +}; + +static enum mode parse_mode(const char *s) +{ + if (!strcmp(s, "oid-fixed")) + return MODE_OID_FIXED; + if (!strcmp(s, "oid-equal")) + return MODE_OID_EQUAL; + if (!strcmp(s, "oid-need-content")) + return MODE_OID_NEED_CONTENT; + if (!strcmp(s, "oid-empty")) + return MODE_OID_EMPTY; + if (!strcmp(s, "oid-trailing")) + return MODE_OID_TRAILING; + if (!strcmp(s, "oid-malformed")) + return MODE_OID_MALFORMED; + if (!strcmp(s, "oid-huge")) + return MODE_OID_HUGE; + if (!strcmp(s, "oid-erange")) + return MODE_OID_ERANGE; + if (!strcmp(s, "oid-overlap")) + return MODE_OID_OVERLAP; + if (!strcmp(s, "oid-misaligned")) + return MODE_OID_MISALIGNED; + if (!strcmp(s, "oid-badstart")) + return MODE_OID_BADSTART; + if (!strcmp(s, "oid-unknown-status")) + return MODE_OID_UNKNOWN_STATUS; + if (!strcmp(s, "oid-abort")) + return MODE_OID_ABORT; + if (!strcmp(s, "oid-bare-status")) + return MODE_OID_BARE_STATUS; + if (!strcmp(s, "oid-empty-packet")) + return MODE_OID_EMPTY_PACKET; + if (!strcmp(s, "oid-crash")) + return MODE_OID_CRASH; + if (!strcmp(s, "oid-garbage")) + return MODE_OID_GARBAGE; + if (!strcmp(s, "cap-none")) + return MODE_CAP_NONE; + die("unknown --mode=%s", s); +} + +/* + * Read "key=value" packets up to a flush, capturing "command" and + * "pathname". Returns 1 if a request was read, 0 on EOF. + * + * The first packet uses the gentle variant so that a clean shutdown + * by Git (EOF) does not produce a spurious "the remote end hung up + * unexpectedly" on stderr. Subsequent packets use the non-gentle + * variant: once inside a request, truncation is a protocol violation + * and dying loudly is the correct response. + */ +static int read_request_header(char **command, char **pathname, + char **old_oid, char **new_oid) +{ + int first = 1; + char *line; + + *command = *pathname = *old_oid = *new_oid = NULL; + for (;;) { + const char *value; + + if (first) { + if (packet_read_line_gently(0, NULL, &line) < 0) + return 0; + first = 0; + } else { + line = packet_read_line(0, NULL); + } + if (!line) + break; + if (skip_prefix(line, "command=", &value)) + *command = xstrdup(value); + else if (skip_prefix(line, "pathname=", &value)) + *pathname = xstrdup(value); + else if (skip_prefix(line, "old-oid=", &value)) + *old_oid = xstrdup(value); + else if (skip_prefix(line, "new-oid=", &value)) + *new_oid = xstrdup(value); + } + return 1; +} + +static void send_status(const char *status) +{ + packet_flush(1); + packet_write_fmt(1, "%s\n", status); + packet_flush(1); +} + +static void command_loop(enum mode mode) +{ + for (;;) { + char *command = NULL, *pathname = NULL; + char *old_oid = NULL, *new_oid = NULL; + + if (!read_request_header(&command, &pathname, + &old_oid, &new_oid)) + break; /* EOF: Git closed its end */ + + if (!command || strcmp(command, "hunks-by-oid")) + die("unexpected command: '%s'", + command ? command : "(none)"); + + if (logfile) { + fprintf(logfile, + "command=%s pathname=%s old-oid=%s new-oid=%s\n", + command, + pathname ? pathname : "(none)", + old_oid ? old_oid : "(none)", + new_oid ? new_oid : "(none)"); + fflush(logfile); + } + + switch (mode) { + case MODE_OID_FIXED: + packet_write_fmt(1, "hunk 5 2 5 2\n"); + send_status("status=success"); + break; + case MODE_OID_EQUAL: + send_status("status=success"); + break; + case MODE_OID_EMPTY: + /* + * An empty old side: the "git diff" convention + * addresses it with a start of 0 and a count of 0. + * Claims two lines added, fewer than the builtin + * would show, so the answer is observable. + */ + packet_write_fmt(1, "hunk 0 0 1 2\n"); + send_status("status=success"); + break; + case MODE_OID_TRAILING: + /* + * Git must ignore trailing space-separated tokens + * on a hunk line (the appendability rule), so this + * must behave exactly like oid-fixed. + */ + packet_write_fmt(1, "hunk 5 2 5 2 moved=yes\n"); + send_status("status=success"); + break; + case MODE_OID_MALFORMED: + packet_write_fmt(1, "hunk five two 5 2\n"); + send_status("status=success"); + break; + case MODE_OID_HUGE: + /* + * In-range for int32 (and for a 32-bit long), so + * only the blob-size bound can reject it. + */ + packet_write_fmt(1, "hunk 1 1000000000 1 1000000000\n"); + send_status("status=success"); + break; + case MODE_OID_ERANGE: + /* Overflows strtol() even where long is 64-bit. */ + packet_write_fmt(1, "hunk 1 99999999999999999999 1 1\n"); + send_status("status=success"); + break; + case MODE_OID_OVERLAP: + packet_write_fmt(1, "hunk 3 2 3 2\n"); + packet_write_fmt(1, "hunk 2 2 2 2\n"); + send_status("status=success"); + break; + case MODE_OID_MISALIGNED: + packet_write_fmt(1, "hunk 2 1 2 1\n"); + packet_write_fmt(1, "hunk 5 1 6 1\n"); + send_status("status=success"); + break; + case MODE_OID_BADSTART: + /* + * A start of 0 names an empty side, so a nonzero + * count beside it names no line; the coordinate is + * rejected per pair while the process stays alive. + */ + packet_write_fmt(1, "hunk 0 2 1 2\n"); + send_status("status=success"); + break; + case MODE_OID_UNKNOWN_STATUS: + send_status("status=frobnicate"); + break; + case MODE_OID_ABORT: + send_status("status=abort"); + break; + case MODE_OID_BARE_STATUS: + /* No hunk-section flush: a protocol violation. */ + packet_write_fmt(1, "status=success\n"); + packet_flush(1); + break; + case MODE_OID_EMPTY_PACKET: + /* + * An empty packet is not a flush; inside the hunk + * section it is a protocol violation. + */ + if (write(1, "0004", 4) < 0) + die_errno("write empty packet"); + send_status("status=success"); + break; + case MODE_OID_CRASH: + packet_write_fmt(1, "hunk 5 2 5 2\n"); + exit(0); + case MODE_OID_GARBAGE: + if (write(1, "@@@@ not a pkt-line @@@@", 24) < 0) + die_errno("write garbage"); + exit(0); + default: + send_status("status=need-content"); + break; + } + + free(command); + free(pathname); + free(old_oid); + free(new_oid); + } +} + +static void handshake(enum mode mode) +{ + char *line; + + line = packet_read_line(0, NULL); + if (!line || strcmp(line, "git-diff-client")) + die("bad welcome: '%s'", line ? line : "(eof)"); + line = packet_read_line(0, NULL); + if (!line || strcmp(line, "version=1")) + die("bad version: '%s'", line ? line : "(eof)"); + if (packet_read_line(0, NULL)) + die("expected flush after version"); + + packet_write_fmt(1, "git-diff-server\n"); + packet_write_fmt(1, "version=1\n"); + packet_flush(1); + + /* Drain capabilities advertised by Git */ + while ((line = packet_read_line(0, NULL))) + ; /* drain */ + + if (mode != MODE_CAP_NONE) + packet_write_fmt(1, "capability=hunks-by-oid\n"); + packet_flush(1); +} + +static const char *const usage_str[] = { + "test-tool diff-process-backend --mode= [--log=]", + NULL +}; + +int cmd__diff_process_backend(int argc, const char **argv) +{ + const char *mode_str = NULL, *log_path = NULL; + enum mode mode = MODE_OID_FIXED; + struct option options[] = { + OPT_STRING(0, "mode", &mode_str, "mode", + "response shape (default oid-fixed);" + " see the file header for the full list of modes"), + OPT_STRING(0, "log", &log_path, "path", + "append per-request summary to this file"), + OPT_END() + }; + + argc = parse_options(argc, argv, NULL, options, usage_str, 0); + if (argc) + usage_with_options(usage_str, options); + + if (mode_str) + mode = parse_mode(mode_str); + + if (log_path) { + logfile = fopen(log_path, "a"); + if (!logfile) + die_errno("failed to open log '%s'", log_path); + } + + handshake(mode); + command_loop(mode); + + if (logfile && fclose(logfile)) + die_errno("error closing log"); + return 0; +} diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index b71a22b43bbc9e..3c3f95269c6279 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -22,6 +22,7 @@ static struct test_cmd cmds[] = { { "date", cmd__date }, { "delete-gpgsig", cmd__delete_gpgsig }, { "delta", cmd__delta }, + { "diff-process-backend", cmd__diff_process_backend }, { "dir-iterator", cmd__dir_iterator }, { "drop-caches", cmd__drop_caches }, { "dump-cache-tree", cmd__dump_cache_tree }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index f2885b33d58aa8..a5bb7555162c8e 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -15,6 +15,7 @@ int cmd__csprng(int argc, const char **argv); int cmd__date(int argc, const char **argv); int cmd__delta(int argc, const char **argv); int cmd__delete_gpgsig(int argc, const char **argv); +int cmd__diff_process_backend(int argc, const char **argv); int cmd__dir_iterator(int argc, const char **argv); int cmd__drop_caches(int argc, const char **argv); int cmd__dump_cache_tree(int argc, const char **argv); diff --git a/t/meson.build b/t/meson.build index 3f45b09dd668bd..a76edfbf81db5a 100644 --- a/t/meson.build +++ b/t/meson.build @@ -518,6 +518,7 @@ integration_tests = [ 't4072-diff-max-depth.sh', 't4073-diff-stat-name-width.sh', 't4074-diff-shifted-matched-group.sh', + 't4080-diff-process.sh', 't4100-apply-stat.sh', 't4101-apply-nonl.sh', 't4102-apply-rename.sh', diff --git a/t/t4080-diff-process.sh b/t/t4080-diff-process.sh new file mode 100755 index 00000000000000..a8efa19416b675 --- /dev/null +++ b/t/t4080-diff-process.sh @@ -0,0 +1,593 @@ +#!/bin/sh + +test_description='diff..process: oid-only hunk requests' + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh + +# See t/helper/test-diff-process-backend.c for the process implementation +# and available --mode= options. + +BACKEND="test-tool diff-process-backend" + +test_expect_success 'setup' ' + echo "*.c diff=cdiff" >.gitattributes && + git add .gitattributes && + + # 10 lines, changes at 5-6 and 9-10 between the two commits. + cat >pair.c <<-\EOF && + line1 + line2 + line3 + line4 + original5 + original6 + line7 + line8 + line9 + line10 + EOF + git add pair.c && + git commit -m "add pair.c" && + + cat >pair.c <<-\EOF && + line1 + line2 + line3 + line4 + changed5 + changed6 + line7 + line8 + changed9 + changed10 + EOF + git add pair.c && + git commit -m "change pair.c" +' + +test_expect_success 'an oid-capable process answers blame by object names alone' ' + test_when_finished "rm -f backend.log" && + ORIG=$(git rev-parse --short HEAD~1) && + CHANGE=$(git rev-parse --short HEAD) && + # The process reports only lines 5-6 as changed, so blame attributes + # lines 9-10 to the original commit even though the builtin diff + # would show them as changed. + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + blame pair.c >actual && + sed -n "9p" actual >line9 && + sed -n "10p" actual >line10 && + test_grep "$ORIG" line9 && + test_grep "$ORIG" line10 && + sed -n "5p" actual >line5 && + test_grep "$CHANGE" line5 && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'an oid-capable process answers --numstat by object names alone' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --format= --numstat -- pair.c >actual && + printf "2\t2\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'need-content falls through to the builtin diff' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-need-content --log=backend.log" \ + log -1 --format= --numstat -- pair.c >actual && + printf "4\t4\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'a warmed hunk store does not override process hunks' ' + test_when_finished "git diff-hunks clear" && + ORIG=$(git rev-parse --short HEAD~1) && + GIT_DIFF_HUNKS_WRITE=1 git log -2 --stat -- pair.c >/dev/null && + + # Control: without a process, blame is served from the store. + git blame --show-stats pair.c >stats && + test_grep "num precomputed hits: 1" stats && + + # The store holds the builtin hunks, but a process-capable driver + # makes the process authoritative, so blame must reflect the + # process hunks (only lines 5-6), not a store hit. + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed" \ + blame pair.c >actual && + sed -n "9p" actual >line9 && + test_grep "$ORIG" line9 +' + +test_expect_success 'a worktree side is not asked by object names' ' + test_when_finished "rm -f backend.log && git checkout -- pair.c" && + echo "worktree edit" >>pair.c && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff --numstat -- pair.c >actual && + printf "1\t0\tpair.c\n" >expect && + test_cmp expect actual && + test_path_is_missing backend.log +' + +test_expect_success 'diff process bypassed by --no-ext-diff' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --format= --numstat --no-ext-diff -- pair.c >actual && + printf "4\t4\tpair.c\n" >expect && + test_cmp expect actual && + test_path_is_missing backend.log +' + +test_expect_success 'format-patch keeps its diffstat off the process' ' + test_when_finished "rm -f backend.log" && + # format-patch emits a diffstat, and a diffstat consults the + # process, but the gate keeps it builtin so a generated patch + # applies for recipients without the process. + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + format-patch -1 --stdout --stat -- pair.c >actual && + test_grep "^+changed9" actual && + test_path_is_missing backend.log +' + +test_expect_success 'format-patch --ext-diff keeps its diffstat off the process' ' + test_when_finished "rm -f backend.log" && + # The gate holds even when --ext-diff enables the external command. + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + format-patch -1 --stdout --ext-diff --stat -- pair.c >actual && + test_grep "^+changed9" actual && + test_path_is_missing backend.log +' + +test_expect_success 'diff process not consulted by plumbing diff commands' ' + test_when_finished "rm -f backend.log && git checkout -f HEAD -- pair.c" && + # diff-tree diffs the two commits, a real pair a defeated gate would + # consult the process for. + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff-tree --numstat HEAD >actual && + test_grep "pair.c" actual && + # diff-index needs a change to diff, or there is no pair and a + # missing log proves nothing; stage one and diff it against HEAD. + echo "staged change" >>pair.c && + git add pair.c && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff-index --cached --numstat HEAD -- pair.c >actual && + test_grep "pair.c" actual && + test_path_is_missing backend.log +' + +test_expect_success 'add -p stages from the builtin diff with a process configured' ' + test_when_finished "rm -f backend.log" && + cat >gate.c <<-\EOF && + int gate(void) { return 1; } + EOF + git add gate.c && + git commit -m "add gate.c" && + cat >gate.c <<-\EOF && + int gate(void) { return 2; } + EOF + # add -p builds its hunks from patch text, which is not a provider + # consumer today, so a configured process cannot shape what it + # offers. This pins that interactive patch stays builtin for the + # current consumers, rather than exercising the plumbing gate. + test_write_lines y | + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + add -p gate.c && + git diff --cached -- gate.c >staged && + test_grep "return 2" staged && + test_path_is_missing backend.log && + git commit -m "gate.c v2" +' + +test_expect_success 'blame withholds identity for the working-tree pair' ' + test_when_finished "rm -f backend.log && git checkout -- pair.c" && + echo "uncommitted" >>pair.c && + wt=$(git hash-object pair.c) && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + blame pair.c >actual && + # The dirty working-tree side is not a stored blob: no request + # may name its bytes by object id. + test_grep ! "new-oid=$wt" backend.log +' + +test_expect_success 'a replaced blob makes the process step aside' ' + new_blob=$(git rev-parse HEAD:pair.c) && + test_when_finished "rm -f backend.log && git replace -d $new_blob" && + # Replacing the new-side blob redirects the content the diff reads + # under the id the process would be sent, so a raw-id request would + # name bytes other than the ones diffed. Identity is withheld: the + # process is not consulted and the builtin computes the pair from + # the replaced content. + repl=$(printf "just one line\n" | git hash-object -w --stdin) && + git replace "$new_blob" "$repl" && + git log -1 --format= --numstat -- pair.c >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --format= --numstat -- pair.c >actual && + test_cmp expect actual && + test_path_is_missing backend.log +' + +test_expect_success 'an equivalence answer omits the pair from the stat' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-equal --log=backend.log" \ + log -1 --format= --numstat -- pair.c >actual && + # An equivalent pair sums to a zero-count entry, and the stat + # code omits zero-count modified entries, the same way a + # whitespace-only pair prints nothing under -w. The builtin + # diff would print nonzero counts here, and the log proves the + # process was consulted. + test_must_be_empty actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'blame passes equivalent pairs through to the boundary' ' + ORIG=$(git rev-parse --short ":/add pair.c") && + # The process asserts every consulted pair equal, so no line is + # ever treated as changed: every line passes through to the + # commit that added the file, marked as the blame boundary. + git -c diff.cdiff.process="$BACKEND --mode=oid-equal" \ + blame pair.c >actual && + sed -n "5p" actual >line5 && + test_grep "^\^$ORIG" line5 && + sed -n "10p" actual >line10 && + test_grep "^\^$ORIG" line10 +' + +test_expect_success 'a warming run records a pair the process defers' ' + test_when_finished "git diff-hunks clear" && + git diff-hunks clear && + # The process owns the path but defers this pair with + # need-content, so the pair gets the builtin diff; that is the + # result the store holds, so the warming run records it and a + # later read is served from the store. + GIT_DIFF_HUNKS_WRITE=1 git -c core.diffHunks=true \ + -c diff.cdiff.process="$BACKEND --mode=oid-need-content" \ + log -1 --format= --stat -- pair.c >/dev/null && + git -c core.diffHunks=true blame --show-stats pair.c >stats 2>&1 && + test_grep "num precomputed hits: 1" stats +' + +# The protocol error paths: each adversarial response shape must warn, +# fall back to the builtin output, and either keep the process alive +# (a per-pair rejection) or disable it for the rest of the command (a +# protocol error). The request log tells the two apart: the log walk +# below consults two pairs (gate.c first, then pair.c), so a disabled +# process shows one logged request and a live one shows two. + +test_expect_success 'a malformed hunk line disables the process for the command' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-malformed --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "disabling it for the remainder" err && + test_line_count = 1 backend.log +' + +test_expect_success 'coordinates past the blob size skip the pair, process stays alive' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-huge --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "past the end" err && + test_line_count = 2 backend.log +' + +test_expect_success 'a count that overflows long skips the pair, process stays alive' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-erange --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "out-of-range coordinates" err && + test_line_count = 2 backend.log +' + +test_expect_success 'overlapping hunks are rejected per pair' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-overlap --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "overlapping hunks" err && + test_line_count = 2 backend.log +' + +test_expect_success 'misaligned hunks are rejected per pair' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-misaligned --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "misaligned" err && + test_line_count = 2 backend.log +' + +test_expect_success 'a start of zero with a nonzero count is rejected per pair' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + # A start of 0 names an empty side, so a nonzero count beside it + # names no line; the coordinate is rejected and the pair falls back + # to the builtin diff while the process stays alive. + git -c diff.cdiff.process="$BACKEND --mode=oid-badstart --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "out-of-range coordinates" err && + test_line_count = 2 backend.log +' + +test_expect_success 'an unrecognized status disables the process for the command' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-unknown-status --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "unrecognized status .frobnicate." err && + test_line_count = 1 backend.log +' + +test_expect_success 'status=abort withdraws the capability without a warning' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-abort --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep ! "disabling" err && + test_line_count = 1 backend.log +' + +test_expect_success 'a bare status without the hunk-section flush is a protocol error' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-bare-status --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "disabling it for the remainder" err && + test_line_count = 1 backend.log +' + +test_expect_success 'an empty packet in the hunk section is a protocol error' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-empty-packet --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "disabling it for the remainder" err && + test_line_count = 1 backend.log +' + +test_expect_success 'a process that dies mid-response fails the command over to builtin' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-crash --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "disabling it for the remainder" err && + test_line_count = 1 backend.log +' + +test_expect_success 'garbage bytes on stdout fail the command over to builtin' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-garbage --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "disabling it for the remainder" err && + test_line_count = 1 backend.log +' + +test_expect_success 'a process announcing no capability is never asked' ' + test_when_finished "rm -f backend.log" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=cap-none --log=backend.log" \ + log --format= --numstat -- "*.c" >actual && + test_cmp expect actual && + test_must_be_empty backend.log +' + +test_expect_success 'a trailing token on a hunk line is ignored' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-trailing --log=backend.log" \ + log -1 --format= --numstat -- pair.c >actual && + printf "2\t2\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'a failed start warns once and the store may serve the path' ' + git init failrepo && + ( + cd failrepo && + echo "*.c diff=cdiff" >.gitattributes && + git add .gitattributes && + test_commit f1 f.c "one" && + test_commit f2 f.c "one +two" && + test_commit f3 f.c "one +two +three" && + GIT_DIFF_HUNKS_WRITE=1 git log --format= --stat -- f.c >/dev/null && + # The command has no shell metacharacters, so it fails at + # exec time; a shell-wrapped command would fail at the + # handshake, which the gentle handshake in the base + # (061a68e443) likewise degrades to the builtin diff. + git blame f.c >expect && + git -c diff.cdiff.process=/does-not-exist-diff-backend \ + blame f.c >actual 2>err && + test_cmp expect actual && + # One warning even though the blame consults two pairs. + test $(grep -c "failed to start" err) = 1 && + # A failure is a non-answer like any other: the request + # that observes it and every later request pass to the + # store, so the warmed store serves both pairs. + git -c diff.cdiff.process=/does-not-exist-diff-backend \ + blame --show-stats f.c >stats 2>&1 && + test_grep "num precomputed hits: 2" stats + ) +' + +test_expect_success 'the store serves a pair the process defers' ' + test_when_finished "git diff-hunks clear" && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git log --format= --stat -- pair.c >/dev/null && + git blame pair.c >expect && + # need-content defers the pair to the builtin diff, which is + # what the store holds, so the walk continues past the process + # and the store serves the pair. + git -c diff.cdiff.process="$BACKEND --mode=oid-need-content" \ + blame pair.c >actual && + test_cmp expect actual && + git -c diff.cdiff.process="$BACKEND --mode=oid-need-content" \ + blame --show-stats pair.c >stats 2>&1 && + test_grep "num precomputed hits: 1" stats +' + +test_expect_success 'git diff between commits consults the process' ' + test_when_finished "rm -f backend.log" && + ORIG=$(git rev-parse ":/add pair.c") && + CHANGE=$(git rev-parse ":/change pair.c") && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff --numstat $ORIG $CHANGE -- pair.c >actual && + printf "2\t2\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'git show consults the process' ' + test_when_finished "rm -f backend.log" && + CHANGE=$(git rev-parse ":/change pair.c") && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + show --format= --numstat $CHANGE -- pair.c >actual && + printf "2\t2\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'diff-tree --ext-diff consults the process' ' + test_when_finished "rm -f backend.log" && + CHANGE=$(git rev-parse ":/change pair.c") && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff-tree --ext-diff --no-commit-id --numstat $CHANGE >actual && + printf "2\t2\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success '--no-diff-process forbids consulting alone' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --no-diff-process --format= --numstat -- pair.c >actual && + printf "4\t4\tpair.c\n" >expect && + test_cmp expect actual && + test_path_is_missing backend.log +' + +test_expect_success '--diff-process allows plumbing to consult' ' + test_when_finished "rm -f backend.log" && + CHANGE=$(git rev-parse ":/change pair.c") && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff-tree --diff-process --no-commit-id --numstat $CHANGE >actual && + printf "2\t2\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'a forced blame diff algorithm bypasses the process' ' + test_when_finished "rm -f backend.log" && + CHANGE=$(git rev-parse --short ":/change pair.c") && + # The process would attribute lines 9-10 to the original commit + # (see the oid-fixed blame test above); a forced builtin + # algorithm must produce the builtin attribution and never start + # the process. + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + blame --histogram pair.c >actual && + sed -n "9p" actual >line9 && + test_grep "$CHANGE" line9 && + test_path_is_missing backend.log +' + +test_expect_success 'textconv output is never identified to the process' ' + test_when_finished "rm -f backend.log" && + echo "*.tcv diff=tcv" >>.gitattributes && + git add .gitattributes && + git commit -m tcv-attr && + test_config diff.tcv.textconv cat && + test_commit tcv1 file.tcv "alpha" && + test_commit tcv2 file.tcv "alpha +beta" && + git blame file.tcv >expect && + git -c diff.tcv.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + blame file.tcv >actual && + test_cmp expect actual && + test_path_is_missing backend.log +' + +test_expect_success 'a gitlink side is never identified to the process' ' + test_when_finished "rm -f backend.log" && + echo "sub diff=cdiff" >>.gitattributes && + git add .gitattributes && + git commit -m sub-attr && + C1=$(git rev-parse HEAD) && + C2=$(git rev-parse HEAD~1) && + git update-index --add --cacheinfo 160000,$C1,sub && + git commit -m sub-1 && + git update-index --add --cacheinfo 160000,$C2,sub && + git commit -m sub-2 && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --format= --numstat -- sub >actual && + printf "1\t1\tsub\n" >expect && + test_cmp expect actual && + test_path_is_missing backend.log +' + +test_expect_success 'a relative diff consults by the repo-relative path' ' + test_when_finished "rm -f backend.log" && + echo "reldir/*.rel diff=rdrv" >>.gitattributes && + git add .gitattributes && + git commit -m rel-attr && + mkdir reldir && + test_write_lines line1 line2 line3 line4 original5 original6 \ + line7 line8 line9 line10 >reldir/x.rel && + git add reldir/x.rel && + git commit -m "add x.rel" && + test_write_lines line1 line2 line3 line4 changed5 changed6 \ + line7 line8 changed9 changed10 >reldir/x.rel && + git add reldir/x.rel && + git commit -m "change x.rel" && + # diff.relative strips the prefix from the displayed name; the + # driver lookup and the request pathname must still use the + # repo-relative path, or the directory-scoped attribute above + # would not match and the process would never be consulted. + # The process runs at the repository root, so its log lands there. + ( + cd reldir && + git -c diff.relative=true \ + -c diff.rdrv.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --format= --numstat -- x.rel + ) >actual && + printf "2\t2\tx.rel\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=reldir/x.rel" backend.log +' + +test_expect_success 'an empty file side is answered with a start of zero' ' + test_when_finished "rm -f backend.log" && + >empty.c && + git add empty.c && + git commit -m "add empty.c" && + printf "x\ny\nz\n" >empty.c && + git add empty.c && + git commit -m "fill empty.c" && + # The process addresses the empty old side with a start of 0 and a + # count of 0, and claims two of the three added lines. The answer is + # used as sent, so the stat shows the two lines it named, not the + # three the builtin would. + git -c diff.cdiff.process="$BACKEND --mode=oid-empty --log=backend.log" \ + log -1 --format= --numstat -- empty.c >actual && + printf "2\t0\tempty.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=empty.c" backend.log +' + +test_done diff --git a/xdiff-interface.h b/xdiff-interface.h index 71e5dffefb8ded..23db1d2a46c6cf 100644 --- a/xdiff-interface.h +++ b/xdiff-interface.h @@ -6,6 +6,18 @@ struct object_database; +/* + * Hunk descriptor for externally computed diffs, in xdiff's own + * coordinates: line numbers are 1-based and a hunk's start is the + * first line it covers. A caller translates any external "empty side" + * idiom (such as git diff's start-0/count-0) to a 1-based start before + * storing hunks in this struct. + */ +struct xdl_hunk { + long old_start, old_count; + long new_start, new_count; +}; + /* * xdiff isn't equipped to handle content over a gigabyte; * we make the cutoff 1GB - 1MB to give some breathing From 2e59acb77df349c5aaf06b4194fbbbcf0556d3ba Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Sun, 2 Aug 2026 21:24:19 +0000 Subject: [PATCH 046/259] bisect: let bisect_reset() optionally check out quietly Add a "quiet" parameter to bisect_reset() that passes "--quiet" to the checkout restoring the original HEAD, suppressing its progress and branch-status output. No caller sets the flag yet, so behavior is unchanged. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- builtin/bisect.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/builtin/bisect.c b/builtin/bisect.c index 798e28f5012d31..19bbfbd0ebea41 100644 --- a/builtin/bisect.c +++ b/builtin/bisect.c @@ -234,7 +234,7 @@ static int write_terms(const char *bad, const char *good) return res; } -static int bisect_reset(const char *commit) +static int bisect_reset(const char *commit, bool quiet) { struct strbuf branch = STRBUF_INIT; @@ -255,8 +255,10 @@ static int bisect_reset(const char *commit) struct child_process cmd = CHILD_PROCESS_INIT; cmd.git_cmd = 1; - strvec_pushl(&cmd.args, "checkout", "--ignore-other-worktrees", - branch.buf, "--", NULL); + strvec_pushl(&cmd.args, "checkout", "--ignore-other-worktrees", NULL); + if (quiet) + strvec_push(&cmd.args, "--quiet"); + strvec_pushl(&cmd.args, branch.buf, "--", NULL); if (run_command(&cmd)) { error(_("could not check out original" " HEAD '%s'. Try 'git bisect" @@ -1089,7 +1091,7 @@ static enum bisect_error bisect_replay(struct bisect_terms *terms, const char *f if (is_empty_or_missing_file(filename)) return error(_("cannot read file '%s' for replaying"), filename); - if (bisect_reset(NULL)) + if (bisect_reset(NULL, false)) return BISECT_FAILED; fp = fopen(filename, "r"); @@ -1338,7 +1340,7 @@ static int cmd_bisect__reset(int argc, const char **argv, const char *prefix UNU if (argc > 1) return error(_("'%s' requires either no argument or a commit"), "git bisect reset"); - return bisect_reset(argc ? argv[0] : NULL); + return bisect_reset(argc ? argv[0] : NULL, false); } static int cmd_bisect__terms(int argc, const char **argv, const char *prefix UNUSED, From f70281521a06894e51b5acfa9abbbb884c05d980 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Sun, 2 Aug 2026 21:24:20 +0000 Subject: [PATCH 047/259] bisect: add --reset-when-found to leave when done When a bisection finishes, "git bisect" reports the first bad commit but leaves the session active until "git bisect reset" is run by hand. Add a "--reset-when-found[=]" option, accepted by both "git bisect start" and "git bisect run", that resets as soon as the first bad commit is found. The "original" value returns to the commit checked out before "git bisect start", while "found" leaves the first bad commit checked out; omitting the value defaults to "original". Persist the selected target in a BISECT_RESET_WHEN_FOUND state file and perform the reset quietly. Let the internal first-bad result propagate to cmd_bisect(), which performs the reset using the existing bad bisect ref after the subcommand has returned. For "git bisect run", this means BISECT_RUN has been printed and closed before cleanup, which also works on systems that cannot unlink an open file. Reject this option together with "--no-checkout", since that mode must not check out either target. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- Documentation/git-bisect.adoc | 14 +++- bisect.c | 2 + builtin/bisect.c | 154 ++++++++++++++++++++++++++++++++-- t/t6030-bisect-porcelain.sh | 121 ++++++++++++++++++++++++++ 4 files changed, 280 insertions(+), 11 deletions(-) diff --git a/Documentation/git-bisect.adoc b/Documentation/git-bisect.adoc index d2115b29905f41..aabddd42ca4d31 100644 --- a/Documentation/git-bisect.adoc +++ b/Documentation/git-bisect.adoc @@ -10,7 +10,7 @@ SYNOPSIS -------- [synopsis] git bisect start [--term-(bad|new)= --term-(good|old)=] - [--no-checkout] [--first-parent] [ [...]] [--] [...] + [--no-checkout] [--first-parent] [--reset-when-found[=]] [ [...]] [--] [...] git bisect (bad|new|) [] git bisect (good|old|) [...] git bisect terms [--term-(good|old) | --term-(bad|new)] @@ -20,7 +20,7 @@ git bisect reset [] git bisect (visualize|view) git bisect replay git bisect log -git bisect run [...] +git bisect run [--reset-when-found[=]] [...] git bisect help DESCRIPTION @@ -385,6 +385,16 @@ ignored. This option is particularly useful in avoiding false positives when a merged branch contained broken or non-buildable commits, but the merge itself was OK. +`--reset-when-found[=]`:: + Once the first bad commit is found, report it and clean up the + bisection state. `` may be `original` to return to the commit + checked out before `git bisect start`, or `found` to leave the first + bad commit checked out. If `` is omitted, it defaults to + `original`. ++ +This option may be given to `git bisect start` or to `git bisect run`. It +cannot be used for a bisection started with `--no-checkout`. + EXAMPLES -------- diff --git a/bisect.c b/bisect.c index 94c7028d2a746a..d426fcd5a909e2 100644 --- a/bisect.c +++ b/bisect.c @@ -488,6 +488,7 @@ static GIT_PATH_FUNC(git_path_bisect_start, "BISECT_START") static GIT_PATH_FUNC(git_path_bisect_log, "BISECT_LOG") static GIT_PATH_FUNC(git_path_bisect_terms, "BISECT_TERMS") static GIT_PATH_FUNC(git_path_bisect_first_parent, "BISECT_FIRST_PARENT") +static GIT_PATH_FUNC(git_path_bisect_reset_when_found, "BISECT_RESET_WHEN_FOUND") static void read_bisect_paths(struct strvec *array) { @@ -1211,6 +1212,7 @@ int bisect_clean_state(void) unlink_or_warn(git_path_bisect_run()); unlink_or_warn(git_path_bisect_terms()); unlink_or_warn(git_path_bisect_first_parent()); + unlink_or_warn(git_path_bisect_reset_when_found()); /* * Cleanup BISECT_START last to support the --no-checkout option * introduced in the commit 4796e823a. diff --git a/builtin/bisect.c b/builtin/bisect.c index 19bbfbd0ebea41..c245bbbd5b74ff 100644 --- a/builtin/bisect.c +++ b/builtin/bisect.c @@ -24,11 +24,12 @@ static GIT_PATH_FUNC(git_path_bisect_start, "BISECT_START") static GIT_PATH_FUNC(git_path_bisect_log, "BISECT_LOG") static GIT_PATH_FUNC(git_path_bisect_names, "BISECT_NAMES") static GIT_PATH_FUNC(git_path_bisect_first_parent, "BISECT_FIRST_PARENT") +static GIT_PATH_FUNC(git_path_bisect_reset_when_found, "BISECT_RESET_WHEN_FOUND") static GIT_PATH_FUNC(git_path_bisect_run, "BISECT_RUN") #define BUILTIN_GIT_BISECT_START_USAGE \ N_("git bisect start [--term-(bad|new)= --term-(good|old)=]\n" \ - " [--no-checkout] [--first-parent] [ [...]] [--] [...]") + " [--no-checkout] [--first-parent] [--reset-when-found[=]] [ [...]] [--] [...]") #define BUILTIN_GIT_BISECT_BAD_USAGE \ N_("git bisect (bad|new|) []") #define BUILTIN_GIT_BISECT_GOOD_USAGE \ @@ -48,7 +49,7 @@ static GIT_PATH_FUNC(git_path_bisect_run, "BISECT_RUN") #define BUILTIN_GIT_BISECT_LOG_USAGE \ "git bisect log" #define BUILTIN_GIT_BISECT_RUN_USAGE \ - N_("git bisect run [...]") + N_("git bisect run [--reset-when-found[=]] [...]") #define BUILTIN_GIT_BISECT_HELP_USAGE \ "git bisect help" @@ -68,6 +69,12 @@ static const char * const git_bisect_usage[] = { NULL }; +enum reset_when_found_mode { + RESET_WHEN_FOUND_NONE, + RESET_WHEN_FOUND_TO_ORIGINAL, + RESET_WHEN_FOUND_TO_FOUND, +}; + struct add_bisect_ref_data { struct rev_info *revs; unsigned int object_flags; @@ -269,7 +276,79 @@ static int bisect_reset(const char *commit, bool quiet) } strbuf_release(&branch); - return bisect_clean_state(); + return 0; +} + +static int parse_reset_when_found(const char *value, + enum reset_when_found_mode *mode) +{ + if (!strcmp(value, "original")) + *mode = RESET_WHEN_FOUND_TO_ORIGINAL; + else if (!strcmp(value, "found")) + *mode = RESET_WHEN_FOUND_TO_FOUND; + else + return error(_("invalid value for '--reset-when-found': '%s'"), + value); + + return 0; +} + +static const char *reset_when_found_mode_name(enum reset_when_found_mode mode) +{ + switch (mode) { + case RESET_WHEN_FOUND_TO_ORIGINAL: + return "original"; + case RESET_WHEN_FOUND_TO_FOUND: + return "found"; + case RESET_WHEN_FOUND_NONE: + BUG("no name for unset reset-when-found mode"); + } + BUG("unknown reset-when-found mode %d", mode); +} + +static int read_reset_when_found(enum reset_when_found_mode *mode) +{ + struct strbuf value = STRBUF_INIT; + int res = 0; + + *mode = RESET_WHEN_FOUND_NONE; + if (is_empty_or_missing_file(git_path_bisect_reset_when_found())) + return 0; + + if (strbuf_read_file(&value, git_path_bisect_reset_when_found(), 0) < 0) { + res = error_errno(_("could not read '%s'"), + git_path_bisect_reset_when_found()); + goto out; + } + strbuf_trim(&value); + if (parse_reset_when_found(value.buf, mode)) + res = -1; + +out: + strbuf_release(&value); + return res; +} + +static int bisect_reset_when_found(enum reset_when_found_mode mode) +{ + struct bisect_terms terms = { 0 }; + char *commit = NULL; + int res; + + if (mode == RESET_WHEN_FOUND_TO_FOUND) { + read_bisect_terms(&terms.term_bad, &terms.term_good); + commit = xstrfmt("refs/bisect/%s", terms.term_bad); + } else if (mode == RESET_WHEN_FOUND_NONE) { + BUG("automatic reset requested without a reset mode"); + } + + res = bisect_reset(commit, true); + if (!res) + res = bisect_clean_state(); + + free(commit); + free_terms(&terms); + return res; } static void log_commit(FILE *fp, @@ -677,7 +756,8 @@ static int bisect_successful(struct bisect_terms *terms) return res; } -static enum bisect_error bisect_next(struct bisect_terms *terms, const char *prefix) +static enum bisect_error bisect_next(struct bisect_terms *terms, + const char *prefix) { enum bisect_error res; @@ -700,7 +780,8 @@ static enum bisect_error bisect_next(struct bisect_terms *terms, const char *pre return res; } -static enum bisect_error bisect_auto_next(struct bisect_terms *terms, const char *prefix) +static enum bisect_error bisect_auto_next(struct bisect_terms *terms, + const char *prefix) { if (bisect_next_check(terms, NULL)) { bisect_print_status(terms); @@ -724,6 +805,7 @@ static enum bisect_error bisect_start(struct bisect_terms *terms, int argc, struct strbuf bisect_names = STRBUF_INIT; struct object_id head_oid; struct object_id oid; + enum reset_when_found_mode reset_when_found = RESET_WHEN_FOUND_NONE; const char *head; if (is_bare_repository(the_repository)) @@ -747,6 +829,13 @@ static enum bisect_error bisect_start(struct bisect_terms *terms, int argc, no_checkout = 1; } else if (!strcmp(arg, "--first-parent")) { first_parent_only = 1; + } else if (!strcmp(arg, "--reset-when-found")) { + reset_when_found = RESET_WHEN_FOUND_TO_ORIGINAL; + } else if (skip_prefix(arg, "--reset-when-found=", &arg)) { + if (parse_reset_when_found(arg, &reset_when_found)) { + res = BISECT_FAILED; + goto finish; + } } else if (!strcmp(arg, "--term-good") || !strcmp(arg, "--term-old")) { i++; @@ -784,6 +873,11 @@ static enum bisect_error bisect_start(struct bisect_terms *terms, int argc, break; } } + if (reset_when_found != RESET_WHEN_FOUND_NONE && no_checkout) { + res = error(_("options '%s' and '%s' cannot be used together"), + "--reset-when-found", "--no-checkout"); + goto finish; + } pathspec_pos = i; /* @@ -861,6 +955,10 @@ static enum bisect_error bisect_start(struct bisect_terms *terms, int argc, if (first_parent_only) write_file(git_path_bisect_first_parent(), "\n"); + if (reset_when_found != RESET_WHEN_FOUND_NONE) + write_file(git_path_bisect_reset_when_found(), "%s\n", + reset_when_found_mode_name(reset_when_found)); + if (no_checkout) { if (repo_get_oid(the_repository, start_head.buf, &oid) < 0) { res = error(_("invalid ref: '%s'"), start_head.buf); @@ -1091,7 +1189,7 @@ static enum bisect_error bisect_replay(struct bisect_terms *terms, const char *f if (is_empty_or_missing_file(filename)) return error(_("cannot read file '%s' for replaying"), filename); - if (bisect_reset(NULL, false)) + if (bisect_clean_state()) return BISECT_FAILED; fp = fopen(filename, "r"); @@ -1239,13 +1337,36 @@ static int bisect_run(struct bisect_terms *terms, int argc, const char **argv) { int res = BISECT_OK; struct strbuf command = STRBUF_INIT; + const char *reset_when_found_arg; const char *new_state; int temporary_stdout_fd, saved_stdout; int is_first_run = 1; + enum reset_when_found_mode reset_when_found = RESET_WHEN_FOUND_NONE; if (bisect_next_check(terms, NULL)) return BISECT_FAILED; + if (argc && !strcmp(argv[0], "--reset-when-found")) { + reset_when_found = RESET_WHEN_FOUND_TO_ORIGINAL; + } else if (argc && skip_prefix(argv[0], "--reset-when-found=", + &reset_when_found_arg)) { + if (parse_reset_when_found(reset_when_found_arg, + &reset_when_found)) + return BISECT_FAILED; + } + + if (reset_when_found != RESET_WHEN_FOUND_NONE && + refs_ref_exists(get_main_ref_store(the_repository), "BISECT_HEAD")) + return error(_("options '%s' and '%s' cannot be used together"), + "--reset-when-found", "--no-checkout"); + + if (reset_when_found != RESET_WHEN_FOUND_NONE) { + write_file(git_path_bisect_reset_when_found(), "%s\n", + reset_when_found_mode_name(reset_when_found)); + argc--; + argv++; + } + if (!argc) { error(_("bisect run failed: no command provided.")); return BISECT_FAILED; @@ -1320,7 +1441,6 @@ static int bisect_run(struct bisect_terms *terms, int argc, const char **argv) res = BISECT_OK; } else if (res == BISECT_INTERNAL_SUCCESS_1ST_BAD_FOUND) { printf(_("bisect found first '%s' commit\n"), terms->term_bad); - res = BISECT_OK; } else if (res) { error(_("bisect run failed: 'git bisect %s'" " exited with error code %d"), new_state, res); @@ -1337,10 +1457,15 @@ static int bisect_run(struct bisect_terms *terms, int argc, const char **argv) static int cmd_bisect__reset(int argc, const char **argv, const char *prefix UNUSED, struct repository *repo UNUSED) { + int res; + if (argc > 1) return error(_("'%s' requires either no argument or a commit"), "git bisect reset"); - return bisect_reset(argc ? argv[0] : NULL, false); + res = bisect_reset(argc ? argv[0] : NULL, false); + if (res) + return res; + return bisect_clean_state(); } static int cmd_bisect__terms(int argc, const char **argv, const char *prefix UNUSED, @@ -1482,7 +1607,8 @@ int cmd_bisect(int argc, !one_of(argv[0], terms.term_good, terms.term_bad, NULL)) usage_msg_optf(_("unknown command: '%s'"), git_bisect_usage, options, argv[0]); - res = bisect_state(&terms, argc, argv); + else + res = bisect_state(&terms, argc, argv); free_terms(&terms); } else { argc--; @@ -1490,5 +1616,15 @@ int cmd_bisect(int argc, res = fn(argc, argv, prefix, repo); } + if (res == BISECT_INTERNAL_SUCCESS_1ST_BAD_FOUND) { + enum reset_when_found_mode mode; + + if (read_reset_when_found(&mode)) + res = BISECT_FAILED; + else if (mode != RESET_WHEN_FOUND_NONE && + bisect_reset_when_found(mode)) + res = BISECT_FAILED; + } + return is_bisect_success(res) ? 0 : -res; } diff --git a/t/t6030-bisect-porcelain.sh b/t/t6030-bisect-porcelain.sh index 081116220a4560..456cf4ed36079c 100755 --- a/t/t6030-bisect-porcelain.sh +++ b/t/t6030-bisect-porcelain.sh @@ -43,6 +43,42 @@ test_bisect_usage () { test_cmp expect actual } +test_bisect_state_file () { + local file && + file=$(git rev-parse --git-path "$1") && + test_path_is_file "$file" +} + +test_bisect_state_missing () { + local file && + file=$(git rev-parse --git-path "$1") && + test_path_is_missing "$file" +} + +bisect_start_and_finish () { + git bisect start "$1" $HASH4 $HASH2 && + git bisect bad +} + +bisect_run_reset_when_found () { + write_script test_script.sh <<-\EOF && + ! grep Another hello >/dev/null + EOF + git bisect start $HASH4 $HASH2 && + git bisect run "$1" ./test_script.sh >my_bisect_log.txt && + test_grep "$HASH3 is the first .bad. commit" my_bisect_log.txt && + test_bisect_state_missing BISECT_RUN +} + +test_reset_when_found_fails () { + local pattern="$1" && + local state_file="$2" && + shift 2 && + test_must_fail "$@" 2>err && + test_grep -- "$pattern" err && + test_bisect_state_missing "$state_file" +} + test_expect_success 'bisect usage' " test_bisect_usage 1 git bisect reset extra1 extra2 <<-\EOF && error: 'git bisect reset' requires either no argument or a commit @@ -453,6 +489,91 @@ test_expect_success '"git bisect run" simple case' ' git bisect reset ' +test_expect_success '"git bisect start --reset-when-found" defaults to original' ' + test_when_finished "git bisect reset && git checkout main" && + git checkout main && + bisect_start_and_finish --reset-when-found && + actual=$(git rev-parse HEAD) && + test "$HASH4" = "$actual" && + actual=$(git branch --show-current) && + test main = "$actual" && + test_bisect_state_missing BISECT_START && + + bisect_start_and_finish --reset-when-found=original && + actual=$(git rev-parse HEAD) && + test "$HASH4" = "$actual" && + actual=$(git branch --show-current) && + test main = "$actual" && + test_bisect_state_missing BISECT_START +' + +test_expect_success '"git bisect start --reset-when-found=found" leaves first bad checked out' ' + test_when_finished "git bisect reset && git checkout main" && + bisect_start_and_finish --reset-when-found=found && + actual=$(git rev-parse HEAD) && + test "$HASH3" = "$actual" && + test_bisect_state_missing BISECT_START +' + +test_expect_success '"git bisect run --reset-when-found" defaults to original' ' + test_when_finished "git bisect reset && git checkout main" && + bisect_run_reset_when_found --reset-when-found && + actual=$(git rev-parse HEAD) && + test "$HASH4" = "$actual" && + actual=$(git branch --show-current) && + test main = "$actual" && + test_bisect_state_missing BISECT_START +' + +test_expect_success '"git bisect run --reset-when-found=found" leaves first bad checked out' ' + test_when_finished "git bisect reset && git checkout main" && + bisect_run_reset_when_found --reset-when-found=found && + actual=$(git rev-parse HEAD) && + test "$HASH3" = "$actual" && + test_bisect_state_missing BISECT_START +' + +test_expect_success '--reset-when-found rejects an unknown reset target' ' + test_when_finished "git bisect reset && git checkout main" && + test_reset_when_found_fails \ + "invalid value for.*--reset-when-found.*unknown" BISECT_START \ + git bisect start --reset-when-found=unknown $HASH4 $HASH2 && + + git bisect start $HASH4 $HASH2 && + test_reset_when_found_fails \ + "invalid value for.*--reset-when-found.*unknown" \ + BISECT_RESET_WHEN_FOUND \ + git bisect run --reset-when-found=unknown true +' + +test_expect_success '--reset-when-found cannot be used with --no-checkout' ' + test_when_finished "git bisect reset" && + test_reset_when_found_fails \ + "options .*--reset-when-found.* and .*--no-checkout.* cannot be used together" BISECT_START \ + git bisect start --reset-when-found=original --no-checkout $HASH4 $HASH2 && + + git bisect start --no-checkout $HASH4 $HASH2 && + test_reset_when_found_fails \ + "options .*--reset-when-found.* and .*--no-checkout.* cannot be used together" BISECT_RESET_WHEN_FOUND \ + git bisect run --reset-when-found=found true +' + +test_expect_success 'without --reset-when-found the bisection state is kept' ' + test_when_finished "git bisect reset" && + git bisect start $HASH4 $HASH2 && + git bisect bad && + test_bisect_state_file BISECT_START +' + +test_expect_success '--reset-when-found does not leak into a later bisection' ' + test_when_finished "git bisect reset && git checkout main" && + bisect_start_and_finish --reset-when-found && + + git bisect start $HASH4 $HASH2 && + git bisect bad && + test_bisect_state_file BISECT_START +' + # We want to automatically find the commit that # added "Ciao" into hello. test_expect_success '"git bisect run" with more complex "git bisect start"' ' From fa632be563931fc869aee9a648f50ec64308dc8a Mon Sep 17 00:00:00 2001 From: "brian m. carlson" Date: Wed, 29 Jul 2026 23:32:10 +0000 Subject: [PATCH 048/259] hex: add functionality for lowercase-only hex We currently allow both upper and lower case for all hex values in Git. However, in a future commit, we'll want to change that to allow only lowercase values in some cases. To prepare for that case, provide a table to convert hex values using lowercase only and an enum to let us choose which we want, wiring it up to the hexval function. For now, keep things completely the same by specifying only the variant that accepts both lowercase and uppercase to avoid changing behavior. Signed-off-by: brian m. carlson Signed-off-by: Junio C Hamano --- color.c | 2 +- hex-ll.c | 37 ++++++++++++++++++++++++++++++++++++- hex-ll.h | 14 ++++++++++---- pkt-line.c | 8 ++++---- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/color.c b/color.c index 00b53f97acbcc7..9015d0faf18259 100644 --- a/color.c +++ b/color.c @@ -72,7 +72,7 @@ static int get_hex_color(const char **inp, int width, unsigned char *out) unsigned int val; assert(width == 1 || width == 2); - val = (hexval(in[0]) << 4) | hexval(in[width - 1]); + val = (hexval(in[0], HEX_KIND_MIXED) << 4) | hexval(in[width - 1], HEX_KIND_MIXED); if (val & ~0xff) return -1; *inp += width; diff --git a/hex-ll.c b/hex-ll.c index 4d7ece1de5ed7c..fa85e918279736 100644 --- a/hex-ll.c +++ b/hex-ll.c @@ -36,10 +36,45 @@ const signed char hexval_table[256] = { -1, -1, -1, -1, -1, -1, -1, -1, /* f8-ff */ }; +const signed char hexval_lc_table[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, /* 00-07 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 08-0f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 10-17 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 18-1f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 20-27 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 28-2f */ + 0, 1, 2, 3, 4, 5, 6, 7, /* 30-37 */ + 8, 9, -1, -1, -1, -1, -1, -1, /* 38-3f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 40-47 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 48-4f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 50-57 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 58-5f */ + -1, 10, 11, 12, 13, 14, 15, -1, /* 60-67 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 68-67 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 70-77 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 78-7f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 80-87 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 88-8f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 90-97 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 98-9f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* a0-a7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* a8-af */ + -1, -1, -1, -1, -1, -1, -1, -1, /* b0-b7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* b8-bf */ + -1, -1, -1, -1, -1, -1, -1, -1, /* c0-c7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* c8-cf */ + -1, -1, -1, -1, -1, -1, -1, -1, /* d0-d7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* d8-df */ + -1, -1, -1, -1, -1, -1, -1, -1, /* e0-e7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* e8-ef */ + -1, -1, -1, -1, -1, -1, -1, -1, /* f0-f7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* f8-ff */ +}; + int hex_to_bytes(unsigned char *binary, const char *hex, size_t len) { for (; len; len--, hex += 2) { - unsigned int val = (hexval(hex[0]) << 4) | hexval(hex[1]); + unsigned int val = (hexval(hex[0], HEX_KIND_MIXED) << 4) | hexval(hex[1], HEX_KIND_MIXED); if (val & ~0xff) return -1; diff --git a/hex-ll.h b/hex-ll.h index a381fa85562234..da1b5239b2bc2d 100644 --- a/hex-ll.h +++ b/hex-ll.h @@ -1,10 +1,16 @@ #ifndef HEX_LL_H #define HEX_LL_H +enum hexkind { + HEX_KIND_MIXED = 0, + HEX_KIND_LOWER = 1, +}; + extern const signed char hexval_table[256]; -static inline unsigned int hexval(unsigned char c) +extern const signed char hexval_lc_table[256]; +static inline unsigned int hexval(unsigned char c, enum hexkind kind) { - return hexval_table[c]; + return kind == HEX_KIND_MIXED ? hexval_table[c] : hexval_lc_table[c]; } /* @@ -13,8 +19,8 @@ static inline unsigned int hexval(unsigned char c) */ static inline int hex2chr(const char *s) { - unsigned int val = hexval(s[0]); - return (val & ~0xf) ? val : (val << 4) | hexval(s[1]); + unsigned int val = hexval(s[0], HEX_KIND_MIXED); + return (val & ~0xf) ? val : (val << 4) | hexval(s[1], HEX_KIND_MIXED); } /* diff --git a/pkt-line.c b/pkt-line.c index 3fc3e9ea7059be..338075558c2e9e 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -378,10 +378,10 @@ int packet_length(const char lenbuf_hex[4], size_t size) { if (size < 4) BUG("buffer too small"); - return hexval(lenbuf_hex[0]) << 12 | - hexval(lenbuf_hex[1]) << 8 | - hexval(lenbuf_hex[2]) << 4 | - hexval(lenbuf_hex[3]); + return hexval(lenbuf_hex[0], HEX_KIND_MIXED) << 12 | + hexval(lenbuf_hex[1], HEX_KIND_MIXED) << 8 | + hexval(lenbuf_hex[2], HEX_KIND_MIXED) << 4 | + hexval(lenbuf_hex[3], HEX_KIND_MIXED); } static const char *find_packfile_uri_path(const char *buffer) From f661df10549ca4cfebf53954d6ebd94c353baf4c Mon Sep 17 00:00:00 2001 From: "brian m. carlson" Date: Wed, 29 Jul 2026 23:32:11 +0000 Subject: [PATCH 049/259] hex: allow specifying hex type with hex2chr We have several places where we use hex2chr. One of those is parsing object IDs, but others decode quoted-printable or percent encoding. All of them accept both uppercase and lowercase hex. In a future commit, we'll change some of these cases, so make hex2chr accept the kind of encoding to use: lowercase only hex or any kind of hex. Signed-off-by: brian m. carlson Signed-off-by: Junio C Hamano --- hex-ll.h | 6 +++--- hex.c | 2 +- mailinfo.c | 2 +- ref-filter.c | 2 +- strbuf.c | 2 +- url.c | 2 +- urlmatch.c | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/hex-ll.h b/hex-ll.h index da1b5239b2bc2d..26847c7b2f81bf 100644 --- a/hex-ll.h +++ b/hex-ll.h @@ -17,10 +17,10 @@ static inline unsigned int hexval(unsigned char c, enum hexkind kind) * Convert two consecutive hexadecimal digits into a char. Return a * negative value on error. Don't run over the end of short strings. */ -static inline int hex2chr(const char *s) +static inline int hex2chr(const char *s, enum hexkind kind) { - unsigned int val = hexval(s[0], HEX_KIND_MIXED); - return (val & ~0xf) ? val : (val << 4) | hexval(s[1], HEX_KIND_MIXED); + unsigned int val = hexval(s[0], kind); + return (val & ~0xf) ? val : (val << 4) | hexval(s[1], kind); } /* diff --git a/hex.c b/hex.c index f02832140d2d43..6150bdcbf83de3 100644 --- a/hex.c +++ b/hex.c @@ -9,7 +9,7 @@ static int get_hash_hex_algop(const char *hex, unsigned char *hash, const struct git_hash_algo *algop) { for (size_t i = 0; i < algop->rawsz; i++) { - int val = hex2chr(hex); + int val = hex2chr(hex, HEX_KIND_MIXED); if (val < 0) return -1; *hash++ = val; diff --git a/mailinfo.c b/mailinfo.c index 13949ff31e1769..85c3119048fe5e 100644 --- a/mailinfo.c +++ b/mailinfo.c @@ -396,7 +396,7 @@ static int decode_q_segment(struct strbuf *out, const struct strbuf *q_seg, int ch, d = *in; if (d == '\n' || !d) break; /* drop trailing newline */ - ch = hex2chr(in); + ch = hex2chr(in, HEX_KIND_MIXED); if (ch >= 0) { strbuf_addch(out, ch); in += 2; diff --git a/ref-filter.c b/ref-filter.c index 29aca08ce7b333..884bcd8fc5f593 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -3567,7 +3567,7 @@ static void append_literal(const char *cp, const char *ep, struct ref_formatting if (cp[1] == '%') cp++; else { - int ch = hex2chr(cp + 1); + int ch = hex2chr(cp + 1, HEX_KIND_MIXED); if (0 <= ch) { strbuf_addch(s, ch); cp += 3; diff --git a/strbuf.c b/strbuf.c index 44955669e8c504..88d23f8ac5f5c4 100644 --- a/strbuf.c +++ b/strbuf.c @@ -457,7 +457,7 @@ size_t strbuf_expand_literal(struct strbuf *sb, const char *placeholder) return 1; case 'x': /* %x00 == NUL, %x0a == LF, etc. */ - ch = hex2chr(placeholder + 1); + ch = hex2chr(placeholder + 1, HEX_KIND_MIXED); if (ch < 0) return 0; strbuf_addch(sb, ch); diff --git a/url.c b/url.c index a59818278f49df..b4d72f784a913d 100644 --- a/url.c +++ b/url.c @@ -62,7 +62,7 @@ static char *url_decode_internal(const char **query, int len, } if (c == '%' && (len < 0 || len >= 3)) { - int val = hex2chr(q + 1); + int val = hex2chr(q + 1, HEX_KIND_MIXED); if (0 < val) { strbuf_addch(out, val); q += 3; diff --git a/urlmatch.c b/urlmatch.c index 20bc2d009cd0dd..989f1d794b909f 100644 --- a/urlmatch.c +++ b/urlmatch.c @@ -50,7 +50,7 @@ static int append_normalized_escapes(struct strbuf *buf, if (ch == '%') { if (from_len < 2) return 0; - ch = hex2chr(from); + ch = hex2chr(from, HEX_KIND_MIXED); if (ch < 0) return 0; from += 2; From 8a790e4e8805568d2c5a6de7455edaef1143a7cc Mon Sep 17 00:00:00 2001 From: "brian m. carlson" Date: Wed, 29 Jul 2026 23:32:12 +0000 Subject: [PATCH 050/259] hex: make hex_to_bytes accept kind of hex to use Similarly to the previous commit, introduce an option for hex_to_bytes to allow us to specify the kind of hex to use: lowercase only or not. For now, everything remains the same as before, but we will change things in a future commit. Signed-off-by: brian m. carlson Signed-off-by: Junio C Hamano --- builtin/index-pack.c | 2 +- diagnose.c | 2 +- hex-ll.c | 4 ++-- hex-ll.h | 2 +- http-push.c | 5 +++-- notes.c | 5 +++-- object-file.c | 2 +- 7 files changed, 12 insertions(+), 10 deletions(-) diff --git a/builtin/index-pack.c b/builtin/index-pack.c index bc86925ad04340..9660e5967b0160 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -1866,7 +1866,7 @@ static void repack_local_links(void) while (strbuf_getline_lf(&line, out) != EOF) { unsigned char binary[GIT_MAX_RAWSZ]; if (line.len != the_hash_algo->hexsz || - !hex_to_bytes(binary, line.buf, line.len)) + !hex_to_bytes(binary, line.buf, line.len, HEX_KIND_MIXED)) die(_("index-pack: Expecting full hex object ID lines only from pack-objects.")); /* diff --git a/diagnose.c b/diagnose.c index 5092bf80d35fdd..fc11cea22938b5 100644 --- a/diagnose.c +++ b/diagnose.c @@ -112,7 +112,7 @@ static void loose_objs_stats(struct strbuf *buf, const char *path) while ((e = readdir_skip_dot_and_dotdot(dir)) != NULL) if (get_dtype(e, &count_path, 0) == DT_DIR && strlen(e->d_name) == 2 && - !hex_to_bytes(&c, e->d_name, 1)) { + !hex_to_bytes(&c, e->d_name, 1, HEX_KIND_MIXED)) { strbuf_setlen(&count_path, base_path_len); strbuf_addf(&count_path, "%s/", e->d_name); total += (count = count_files(&count_path)); diff --git a/hex-ll.c b/hex-ll.c index fa85e918279736..b2e9684693057e 100644 --- a/hex-ll.c +++ b/hex-ll.c @@ -71,10 +71,10 @@ const signed char hexval_lc_table[256] = { -1, -1, -1, -1, -1, -1, -1, -1, /* f8-ff */ }; -int hex_to_bytes(unsigned char *binary, const char *hex, size_t len) +int hex_to_bytes(unsigned char *binary, const char *hex, size_t len, enum hexkind kind) { for (; len; len--, hex += 2) { - unsigned int val = (hexval(hex[0], HEX_KIND_MIXED) << 4) | hexval(hex[1], HEX_KIND_MIXED); + unsigned int val = (hexval(hex[0], kind) << 4) | hexval(hex[1], kind); if (val & ~0xff) return -1; diff --git a/hex-ll.h b/hex-ll.h index 26847c7b2f81bf..fe698f0c762ef7 100644 --- a/hex-ll.h +++ b/hex-ll.h @@ -28,6 +28,6 @@ static inline int hex2chr(const char *s, enum hexkind kind) * values to `binary` as `len` bytes. Return 0 on success, or -1 if * the input does not consist of hex digits). */ -int hex_to_bytes(unsigned char *binary, const char *hex, size_t len); +int hex_to_bytes(unsigned char *binary, const char *hex, size_t len, enum hexkind kind); #endif diff --git a/http-push.c b/http-push.c index 94a1fac9ab0fcd..0cc990d395bbd3 100644 --- a/http-push.c +++ b/http-push.c @@ -1030,12 +1030,13 @@ static int get_oid_hex_from_objpath(const char *path, struct object_id *oid) if (strlen(path) != the_hash_algo->hexsz + 1) return -1; - if (hex_to_bytes(oid->hash, path, 1)) + if (hex_to_bytes(oid->hash, path, 1, HEX_KIND_MIXED)) return -1; path += 2; path++; /* skip '/' */ - return hex_to_bytes(oid->hash + 1, path, the_hash_algo->rawsz - 1); + return hex_to_bytes(oid->hash + 1, path, the_hash_algo->rawsz - 1, + HEX_KIND_MIXED); } static void process_ls_object(struct remote_ls_ctx *ls) diff --git a/notes.c b/notes.c index ec9c2cb150d4e3..99b8b15d819214 100644 --- a/notes.c +++ b/notes.c @@ -428,7 +428,7 @@ static void load_subtree(struct notes_tree *t, struct leaf_node *subtree, goto handle_non_note; if (hex_to_bytes(object_oid.hash + prefix_len, entry.path, - hashsz - prefix_len)) + hashsz - prefix_len, HEX_KIND_MIXED)) goto handle_non_note; /* entry.path is not a SHA1 */ memset(object_oid.hash + hashsz, 0, GIT_MAX_RAWSZ - hashsz); @@ -442,7 +442,8 @@ static void load_subtree(struct notes_tree *t, struct leaf_node *subtree, /* internal nodes must be trees */ goto handle_non_note; - if (hex_to_bytes(object_oid.hash + len++, entry.path, 1)) + if (hex_to_bytes(object_oid.hash + len++, entry.path, 1, + HEX_KIND_MIXED)) goto handle_non_note; /* entry.path is not a SHA1 */ /* diff --git a/object-file.c b/object-file.c index ec35c318bc9fe7..5884e726bb36d5 100644 --- a/object-file.c +++ b/object-file.c @@ -1073,7 +1073,7 @@ int for_each_file_in_obj_subdir(unsigned int subdir_nr, strbuf_add(path, de->d_name, namelen); if (namelen == algop->hexsz - 2 && !hex_to_bytes(oid.hash + 1, de->d_name, - algop->rawsz - 1)) { + algop->rawsz - 1, HEX_KIND_MIXED)) { oid_set_algo(&oid, algop); memset(oid.hash + algop->rawsz, 0, GIT_MAX_RAWSZ - algop->rawsz); From 5f2685d25628871fb7477041c16de43592287ca7 Mon Sep 17 00:00:00 2001 From: "brian m. carlson" Date: Wed, 29 Jul 2026 23:32:13 +0000 Subject: [PATCH 051/259] hex: label usages of hex parsing for object IDs In preparation for a future change, label the hex parsing we're doing for object IDs by defining a constant called HEX_KIND_OID. This is currently the same as HEX_KIND_MIXED, so there is no functional change here. Signed-off-by: brian m. carlson Signed-off-by: Junio C Hamano --- diagnose.c | 2 +- hex-ll.h | 2 ++ hex.c | 2 +- http-push.c | 4 ++-- notes.c | 2 +- object-file.c | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/diagnose.c b/diagnose.c index fc11cea22938b5..9c652d36a6649c 100644 --- a/diagnose.c +++ b/diagnose.c @@ -112,7 +112,7 @@ static void loose_objs_stats(struct strbuf *buf, const char *path) while ((e = readdir_skip_dot_and_dotdot(dir)) != NULL) if (get_dtype(e, &count_path, 0) == DT_DIR && strlen(e->d_name) == 2 && - !hex_to_bytes(&c, e->d_name, 1, HEX_KIND_MIXED)) { + !hex_to_bytes(&c, e->d_name, 1, HEX_KIND_OID)) { strbuf_setlen(&count_path, base_path_len); strbuf_addf(&count_path, "%s/", e->d_name); total += (count = count_files(&count_path)); diff --git a/hex-ll.h b/hex-ll.h index fe698f0c762ef7..9da76f17e8a1d7 100644 --- a/hex-ll.h +++ b/hex-ll.h @@ -6,6 +6,8 @@ enum hexkind { HEX_KIND_LOWER = 1, }; +#define HEX_KIND_OID HEX_KIND_MIXED + extern const signed char hexval_table[256]; extern const signed char hexval_lc_table[256]; static inline unsigned int hexval(unsigned char c, enum hexkind kind) diff --git a/hex.c b/hex.c index 6150bdcbf83de3..4e1e81af3f476b 100644 --- a/hex.c +++ b/hex.c @@ -9,7 +9,7 @@ static int get_hash_hex_algop(const char *hex, unsigned char *hash, const struct git_hash_algo *algop) { for (size_t i = 0; i < algop->rawsz; i++) { - int val = hex2chr(hex, HEX_KIND_MIXED); + int val = hex2chr(hex, HEX_KIND_OID); if (val < 0) return -1; *hash++ = val; diff --git a/http-push.c b/http-push.c index 0cc990d395bbd3..132d26d6a1477c 100644 --- a/http-push.c +++ b/http-push.c @@ -1030,13 +1030,13 @@ static int get_oid_hex_from_objpath(const char *path, struct object_id *oid) if (strlen(path) != the_hash_algo->hexsz + 1) return -1; - if (hex_to_bytes(oid->hash, path, 1, HEX_KIND_MIXED)) + if (hex_to_bytes(oid->hash, path, 1, HEX_KIND_OID)) return -1; path += 2; path++; /* skip '/' */ return hex_to_bytes(oid->hash + 1, path, the_hash_algo->rawsz - 1, - HEX_KIND_MIXED); + HEX_KIND_OID); } static void process_ls_object(struct remote_ls_ctx *ls) diff --git a/notes.c b/notes.c index 99b8b15d819214..7e9e3eb2d2cb27 100644 --- a/notes.c +++ b/notes.c @@ -443,7 +443,7 @@ static void load_subtree(struct notes_tree *t, struct leaf_node *subtree, goto handle_non_note; if (hex_to_bytes(object_oid.hash + len++, entry.path, 1, - HEX_KIND_MIXED)) + HEX_KIND_OID)) goto handle_non_note; /* entry.path is not a SHA1 */ /* diff --git a/object-file.c b/object-file.c index 5884e726bb36d5..b44f3c7bf805b2 100644 --- a/object-file.c +++ b/object-file.c @@ -1073,7 +1073,7 @@ int for_each_file_in_obj_subdir(unsigned int subdir_nr, strbuf_add(path, de->d_name, namelen); if (namelen == algop->hexsz - 2 && !hex_to_bytes(oid.hash + 1, de->d_name, - algop->rawsz - 1, HEX_KIND_MIXED)) { + algop->rawsz - 1, HEX_KIND_OID)) { oid_set_algo(&oid, algop); memset(oid.hash + algop->rawsz, 0, GIT_MAX_RAWSZ - algop->rawsz); From 7233f519c7cd58ff09d88814939d72e4fe84b21a Mon Sep 17 00:00:00 2001 From: "brian m. carlson" Date: Wed, 29 Jul 2026 23:32:14 +0000 Subject: [PATCH 052/259] object-name: use hexval We've open-coded a different implementation of parsing hex values here when we already have a perfectly good one in hexval. This implementation will almost certainly be slower because it isn't table-driven, unlike the other one, and since it's not constant time it has no other advantages either. To tidy things up and prepare for future work, switch to hexval in this case. Signed-off-by: brian m. carlson Signed-off-by: Junio C Hamano --- object-name.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/object-name.c b/object-name.c index 83efba0ba668e5..d2d81b35118c7b 100644 --- a/object-name.c +++ b/object-name.c @@ -236,17 +236,10 @@ static int parse_oid_prefix(const char *name, int len, { for (int i = 0; i < len; i++) { unsigned char c = name[i]; - unsigned char val; - if (c >= '0' && c <= '9') { - val = c - '0'; - } else if (c >= 'a' && c <= 'f') { - val = c - 'a' + 10; - } else if (c >= 'A' && c <='F') { - val = c - 'A' + 10; - c -= 'A' - 'a'; - } else { + int val = hexval(c, HEX_KIND_OID); + + if (val < 0) return -1; - } if (hex_out) hex_out[i] = c; From 4a1cd4a2236c19a3a11d4521c1e39a5d98dd7eac Mon Sep 17 00:00:00 2001 From: "brian m. carlson" Date: Wed, 29 Jul 2026 23:32:15 +0000 Subject: [PATCH 053/259] hex: allow only lowercase object IDs in breaking changes mode Git has historically allowed either lowercase or uppercase hex for object IDs, but it has always emitted only lowercase. This has caused people to expect only lowercase and not handle uppercase. As an example, Git's own example hooks look for "[0-9a-f]" in several places, but there are many other Git-adjacent pieces of software, including Gitolite, which make the assumption that object IDs are always lowercase. This is not to criticize the authors of these projects, but rather to point out how common this assumption is. In fact, it's so common that we have only one test in our codebase that fails when we reject uppercase object IDs. More critically, it leads people to make security-based assumptions that an object ID either does not contain uppercase characters or that an object ID can be expressed uniquely in hex form, neither of which are currently true. Git itself normally uses binary object IDs, which avoids many of these problems, but most other projects deal primarily in hex object IDs, so they are more affected. In preparation for Git 3.0, only allow lowercase hex object IDs in breaking changes mode and document this as well. Update the single failing test and add a new one to verify we reject new uppercase object IDs. Note that in t5324, we change the hex character from "A" to "b" because in SHA-256 mode, "a" is the correct value, so our test_must_fail assertion will unexpectedly succeed in that case. Signed-off-by: brian m. carlson Signed-off-by: Junio C Hamano --- Documentation/BreakingChanges.adoc | 5 +++++ hex-ll.h | 4 ++++ t/t1503-rev-parse-verify.sh | 5 +++++ t/t5324-split-commit-graph.sh | 4 ++-- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Documentation/BreakingChanges.adoc b/Documentation/BreakingChanges.adoc index 73bb939359c72e..dbc46d14e33dce 100644 --- a/Documentation/BreakingChanges.adoc +++ b/Documentation/BreakingChanges.adoc @@ -171,6 +171,11 @@ JGit, libgit2 and Gitoxide need to support it. matches the default branch name used in new repositories by many of the big Git forges. +* Git will accept hex object IDs only in lowercase. The fact that Git has + historically allowed uppercase characters in hex object IDs has been the + source of a variety of bugs and security problems in software using Git. We + don't expect most users to notice any change. + * Git will require Rust as a mandatory part of the build process. While Git already started to adopt Rust in Git 2.49, all parts written in Rust are optional for the time being. This includes: diff --git a/hex-ll.h b/hex-ll.h index 9da76f17e8a1d7..2f9c8d7c25fb73 100644 --- a/hex-ll.h +++ b/hex-ll.h @@ -6,7 +6,11 @@ enum hexkind { HEX_KIND_LOWER = 1, }; +#ifdef WITH_BREAKING_CHANGES +#define HEX_KIND_OID HEX_KIND_LOWER +#else #define HEX_KIND_OID HEX_KIND_MIXED +#endif extern const signed char hexval_table[256]; extern const signed char hexval_lc_table[256]; diff --git a/t/t1503-rev-parse-verify.sh b/t/t1503-rev-parse-verify.sh index 87638a4a2c71f9..f07b45de5a3fc1 100755 --- a/t/t1503-rev-parse-verify.sh +++ b/t/t1503-rev-parse-verify.sh @@ -60,6 +60,11 @@ test_expect_success 'works with one good rev' ' test "$rev_head" = "$HASH4" ' +test_expect_success WITH_BREAKING_CHANGES 'rejects uppercase revs' ' + UC_HASH=$(echo "$HASH1" | tr a-f A-F) && + test_must_fail git rev-parse --verify "$UC_HASH" +' + test_expect_success 'fails with any bad rev or many good revs' ' test_must_fail git rev-parse --verify 2>error && test_grep "single revision" error && diff --git a/t/t5324-split-commit-graph.sh b/t/t5324-split-commit-graph.sh index bf7ba0e5580d84..29db815c773a98 100755 --- a/t/t5324-split-commit-graph.sh +++ b/t/t5324-split-commit-graph.sh @@ -349,7 +349,7 @@ test_expect_success 'verify after commit-graph-chain corruption (base)' ' test_must_fail git commit-graph verify 2>test_err && grep -v "^+" test_err >err && test_grep "invalid commit-graph chain" err && - corrupt_file "$graphdir/commit-graph-chain" 30 "A" && + corrupt_file "$graphdir/commit-graph-chain" 30 "a" && test_must_fail git commit-graph verify 2>test_err && grep -v "^+" test_err >err && test_grep "unable to find all commit-graph files" err @@ -364,7 +364,7 @@ test_expect_success 'verify after commit-graph-chain corruption (tip)' ' test_must_fail git commit-graph verify 2>test_err && grep -v "^+" test_err >err && test_grep "invalid commit-graph chain" err && - corrupt_file "$graphdir/commit-graph-chain" 70 "A" && + corrupt_file "$graphdir/commit-graph-chain" 70 "b" && test_must_fail git commit-graph verify 2>test_err && grep -v "^+" test_err >err && test_grep "unable to find all commit-graph files" err From b1b008fa2b3c7591ad0094e45fd6fc9b8fac5bff Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 5 Aug 2026 09:44:45 +0200 Subject: [PATCH 054/259] odb/streaming: track write stream size in the structure When passing around a `struct odb_write_stream` we typically also have to pass the number of bytes that the stream will yield. This is required because the object header itself contains that size, and consequently we cannot write the header without that information. Move this information into the stream itself so that it becomes self- describing. In addition to that, this also brings the `struct odb_write_stream` a bit closer to the `struct odb_read_stream` so that we can eventually merge both stream types. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/unpack-objects.c | 3 ++- object-file.c | 25 +++++++++++-------------- odb.c | 4 ++-- odb.h | 2 +- odb/source-files.c | 3 +-- odb/source-inmemory.c | 11 +++++------ odb/source-loose.c | 7 +++---- odb/source-packed.c | 1 - odb/source.h | 5 ++--- odb/streaming.c | 1 + odb/streaming.h | 1 + odb/transaction.c | 4 ++-- odb/transaction.h | 4 ++-- t/unit-tests/u-odb-inmemory.c | 11 +++++------ 14 files changed, 38 insertions(+), 44 deletions(-) diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index 4263edfbecdd39..f3e0b504f43f13 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -392,13 +392,14 @@ static void stream_blob(unsigned long size, unsigned nr) struct odb_write_stream in_stream = { .read = feed_input_zstream, .data = &data, + .size = size, }; struct obj_info *info = &obj_list[nr]; data.zstream = &zstream; git_inflate_init(&zstream); - if (odb_write_object_stream(the_repository->objects, &in_stream, size, &info->oid)) + if (odb_write_object_stream(the_repository->objects, &in_stream, &info->oid)) die(_("failed to write object in stream")); if (data.status != Z_STREAM_END) diff --git a/object-file.c b/object-file.c index ec35c318bc9fe7..b196abb596e87b 100644 --- a/object-file.c +++ b/object-file.c @@ -704,7 +704,7 @@ static void prepare_packfile_transaction(struct odb_transaction_files *transacti static int hash_blob_stream(struct odb_write_stream *stream, const struct git_hash_algo *hash_algo, - struct object_id *result_oid, size_t size) + struct object_id *result_oid) { unsigned char buf[16384]; struct git_hash_ctx ctx; @@ -712,7 +712,7 @@ static int hash_blob_stream(struct odb_write_stream *stream, size_t bytes_hashed = 0; header_len = format_object_header((char *)buf, sizeof(buf), - OBJ_BLOB, size); + OBJ_BLOB, stream->size); git_hash_init(&ctx, hash_algo); git_hash_update(&ctx, buf, header_len); @@ -727,7 +727,7 @@ static int hash_blob_stream(struct odb_write_stream *stream, bytes_hashed += read_result; } - if (bytes_hashed != size) + if (bytes_hashed != stream->size) return -1; git_hash_final_oid(result_oid, &ctx); @@ -740,7 +740,7 @@ static int hash_blob_stream(struct odb_write_stream *stream, * packfile in state while updating the hash in ctx. */ static void stream_blob_to_pack(struct transaction_packfile *state, - struct git_hash_ctx *ctx, size_t size, + struct git_hash_ctx *ctx, struct odb_write_stream *stream) { git_zstream s; @@ -753,7 +753,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state, git_deflate_init(&s, cfg->pack_compression_level); - hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), OBJ_BLOB, size); + hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), OBJ_BLOB, stream->size); s.next_out = obuf + hdrlen; s.avail_out = sizeof(obuf) - hdrlen; @@ -793,9 +793,9 @@ static void stream_blob_to_pack(struct transaction_packfile *state, } } - if (bytes_read != size) + if (bytes_read != stream->size) die("read %" PRIuMAX " bytes of blob data, but expected %" PRIuMAX " bytes", - (uintmax_t)bytes_read, (uintmax_t)size); + (uintmax_t)bytes_read, (uintmax_t)stream->size); git_deflate_end(&s); } @@ -870,7 +870,6 @@ static void flush_packfile_transaction(struct odb_transaction_files *transaction */ static int odb_transaction_files_write_object_stream(struct odb_transaction *base, struct odb_write_stream *stream, - size_t size, struct object_id *result_oid) { struct odb_transaction_files *transaction = container_of(base, @@ -884,7 +883,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas struct pack_idx_entry *idx; header_len = format_object_header((char *)obuf, sizeof(obuf), - OBJ_BLOB, size); + OBJ_BLOB, stream->size); git_hash_init(&ctx, transaction->base.source->odb->repo->hash_algo); git_hash_update(&ctx, obuf, header_len); @@ -899,7 +898,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas * to zlib compression and is sufficient for this check. */ if (state->nr_written && pack_size_limit_cfg && - pack_size_limit_cfg < state->offset + size) + pack_size_limit_cfg < state->offset + stream->size) flush_packfile_transaction(transaction); CALLOC_ARRAY(idx, 1); @@ -909,7 +908,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas hashfile_checkpoint(state->f, &checkpoint); idx->offset = state->offset; crc32_begin(state->f); - stream_blob_to_pack(state, &ctx, size, stream); + stream_blob_to_pack(state, &ctx, stream); git_hash_final_oid(result_oid, &ctx); idx->crc32 = crc32_end(state->f); @@ -962,14 +961,12 @@ int index_fd(struct index_state *istate, struct object_id *oid, odb_transaction_begin_or_die(odb, &transaction, 0); ret = odb_transaction_write_object_stream(transaction, &stream, - xsize_t(st->st_size), oid); if (!inflight) odb_transaction_commit(transaction); } else { ret = hash_blob_stream(&stream, - the_repository->hash_algo, oid, - xsize_t(st->st_size)); + the_repository->hash_algo, oid); } odb_write_stream_release(&stream); diff --git a/odb.c b/odb.c index dabd481f57dbc4..585b2b2965bb91 100644 --- a/odb.c +++ b/odb.c @@ -1028,10 +1028,10 @@ int odb_write_object_ext(struct object_database *odb, } int odb_write_object_stream(struct object_database *odb, - struct odb_write_stream *stream, size_t len, + struct odb_write_stream *stream, struct object_id *oid) { - return odb_source_write_object_stream(odb->sources, stream, len, oid); + return odb_source_write_object_stream(odb->sources, stream, oid); } struct object_database *odb_new(struct repository *repo, diff --git a/odb.h b/odb.h index cbc2f9ced42338..019d3af3e8d212 100644 --- a/odb.h +++ b/odb.h @@ -629,7 +629,7 @@ static inline int odb_write_object(struct object_database *odb, struct odb_write_stream; int odb_write_object_stream(struct object_database *odb, - struct odb_write_stream *stream, size_t len, + struct odb_write_stream *stream, struct object_id *oid); void parse_alternates(const char *string, diff --git a/odb/source-files.c b/odb/source-files.c index 5e086d266fac4f..f51960bd71bb11 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -175,11 +175,10 @@ static int odb_source_files_write_object(struct odb_source *source, static int odb_source_files_write_object_stream(struct odb_source *source, struct odb_write_stream *stream, - size_t len, struct object_id *oid) { struct odb_source_files *files = odb_source_files_downcast(source); - return odb_source_write_object_stream(&files->loose->base, stream, len, oid); + return odb_source_write_object_stream(&files->loose->base, stream, oid); } static int odb_source_files_begin_transaction(struct odb_source *source, diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index 3e71611b8e0071..398131e194f87c 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -257,7 +257,6 @@ static int odb_source_inmemory_write_object(struct odb_source *source, static int odb_source_inmemory_write_object_stream(struct odb_source *source, struct odb_write_stream *stream, - size_t len, struct object_id *oid) { char buf[16384]; @@ -265,12 +264,12 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source, char *data; int ret; - CALLOC_ARRAY(data, len); + CALLOC_ARRAY(data, stream->size); while (!stream->is_finished) { ssize_t bytes_read; bytes_read = odb_write_stream_read(stream, buf, sizeof(buf)); - if (total_read + bytes_read > len) { + if (total_read + bytes_read > stream->size) { ret = error("object stream yielded more bytes than expected"); goto out; } @@ -279,15 +278,15 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source, total_read += bytes_read; } - if (total_read != len) { + if (total_read != stream->size) { ret = error("object stream yielded less bytes than expected"); goto out; } hash_object_file(source->odb->repo->hash_algo, data, total_read, OBJ_BLOB, oid); - ret = odb_source_inmemory_write_object(source, data, len, OBJ_BLOB, oid, - NULL, NULL, 0); + ret = odb_source_inmemory_write_object(source, data, stream->size, + OBJ_BLOB, oid, NULL, NULL, 0); if (ret < 0) goto out; diff --git a/odb/source-loose.c b/odb/source-loose.c index ef0e9192777c4a..77a2adb52abf47 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -846,7 +846,6 @@ static int odb_source_loose_write_object(struct odb_source *source, static int odb_source_loose_write_object_stream(struct odb_source *source, struct odb_write_stream *in_stream, - size_t len, struct object_id *oid) { struct odb_source_loose *loose = odb_source_loose_downcast(source); @@ -868,7 +867,7 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, /* Since oid is not determined, save tmp file to odb path. */ strbuf_addf(&filename, "%s/", loose->base.path); - hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len); + hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, in_stream->size); /* * Common steps for write_loose_object and stream_loose_object to @@ -916,9 +915,9 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, */ } while (ret == Z_OK || ret == Z_BUF_ERROR); - if (stream.total_in != len + hdrlen) + if (stream.total_in != in_stream->size + hdrlen) die(_("write stream object %"PRIuMAX" != %"PRIuMAX), (uintmax_t)stream.total_in, - (uintmax_t)len + hdrlen); + (uintmax_t)in_stream->size + hdrlen); /* * Common steps for write_loose_object and stream_loose_object to diff --git a/odb/source-packed.c b/odb/source-packed.c index 0890704e76879b..e6ff74833b8bec 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -610,7 +610,6 @@ static int odb_source_packed_write_object(struct odb_source *source UNUSED, static int odb_source_packed_write_object_stream(struct odb_source *source UNUSED, struct odb_write_stream *stream UNUSED, - size_t len UNUSED, struct object_id *oid UNUSED) { return error("packed backend cannot write object streams"); diff --git a/odb/source.h b/odb/source.h index fc04dd5cda8800..0080148ba71078 100644 --- a/odb/source.h +++ b/odb/source.h @@ -221,7 +221,7 @@ struct odb_source { * otherwise. */ int (*write_object_stream)(struct odb_source *source, - struct odb_write_stream *stream, size_t len, + struct odb_write_stream *stream, struct object_id *oid); /* @@ -437,10 +437,9 @@ static inline int odb_source_write_object(struct odb_source *source, */ static inline int odb_source_write_object_stream(struct odb_source *source, struct odb_write_stream *stream, - size_t len, struct object_id *oid) { - return source->write_object_stream(source, stream, len, oid); + return source->write_object_stream(source, stream, oid); } /* diff --git a/odb/streaming.c b/odb/streaming.c index 20531e864c9561..38c2f6687c432d 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -336,5 +336,6 @@ void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd, stream->data = data; stream->read = read_object_fd; + stream->size = size; stream->is_finished = 0; } diff --git a/odb/streaming.h b/odb/streaming.h index c0236717802301..4d7d31b5aa04f6 100644 --- a/odb/streaming.h +++ b/odb/streaming.h @@ -55,6 +55,7 @@ ssize_t odb_read_stream_read(struct odb_read_stream *stream, void *buf, size_t l struct odb_write_stream { ssize_t (*read)(struct odb_write_stream *, unsigned char *, size_t); void *data; + size_t size; int is_finished; }; diff --git a/odb/transaction.c b/odb/transaction.c index dab7da6a9a4f55..6aaf1338127534 100644 --- a/odb/transaction.c +++ b/odb/transaction.c @@ -40,9 +40,9 @@ int odb_transaction_commit(struct odb_transaction *transaction) int odb_transaction_write_object_stream(struct odb_transaction *transaction, struct odb_write_stream *stream, - size_t len, struct object_id *oid) + struct object_id *oid) { - return transaction->write_object_stream(transaction, stream, len, oid); + return transaction->write_object_stream(transaction, stream, oid); } int odb_transaction_env(struct odb_transaction *transaction, struct strvec *env) diff --git a/odb/transaction.h b/odb/transaction.h index 4cb2eafcbf08f5..ffb279314cfd21 100644 --- a/odb/transaction.h +++ b/odb/transaction.h @@ -31,7 +31,7 @@ struct odb_transaction { * otherwise. */ int (*write_object_stream)(struct odb_transaction *transaction, - struct odb_write_stream *stream, size_t len, + struct odb_write_stream *stream, struct object_id *oid); /* @@ -82,7 +82,7 @@ int odb_transaction_commit(struct odb_transaction *transaction); */ int odb_transaction_write_object_stream(struct odb_transaction *transaction, struct odb_write_stream *stream, - size_t len, struct object_id *oid); + struct object_id *oid); /* * Populates the provided strvec with the environment variables that a child diff --git a/t/unit-tests/u-odb-inmemory.c b/t/unit-tests/u-odb-inmemory.c index ddf2db5c811fb8..5ccc52dccc06f9 100644 --- a/t/unit-tests/u-odb-inmemory.c +++ b/t/unit-tests/u-odb-inmemory.c @@ -269,7 +269,6 @@ struct membuf_write_stream { struct odb_write_stream base; const char *buf; size_t offset; - size_t size; }; static ssize_t membuf_write_stream_read(struct odb_write_stream *stream, @@ -280,13 +279,13 @@ static ssize_t membuf_write_stream_read(struct odb_write_stream *stream, if (chunk_size > len) chunk_size = len; - if (chunk_size > s->size - s->offset) - chunk_size = s->size - s->offset; + if (chunk_size > s->base.size - s->offset) + chunk_size = s->base.size - s->offset; memcpy(buf, s->buf + s->offset, chunk_size); s->offset += chunk_size; - if (s->offset == s->size) + if (s->offset == s->base.size) s->base.is_finished = 1; return chunk_size; @@ -298,13 +297,13 @@ void test_odb_inmemory__write_object_stream(void) const char data[] = "foobar"; struct membuf_write_stream stream = { .base.read = membuf_write_stream_read, + .base.size = strlen(data), .buf = data, - .size = strlen(data), }; struct object_id written_oid; cl_must_pass(odb_source_write_object_stream(&source->base, &stream.base, - strlen(data), &written_oid)); + &written_oid)); cl_assert_equal_s(oid_to_hex(&written_oid), FOOBAR_OID); cl_assert_object_info(source, &written_oid, OBJ_BLOB, "foobar"); From 726782254239706bc91276537450b7a21b81f99e Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 5 Aug 2026 09:44:46 +0200 Subject: [PATCH 055/259] odb/streaming: drop `is_finished` field The `is_finished` field is used to track whether a write stream is done writing all of its data. Tracking this field as part of the stream itself shouldn't be required though: callers will already know when the stream is done when the stream's read function returns zero bytes, same as when reading from a file descriptor. There is one exception where it gets a bit more complicated: when consuming data in "builtin/unpack-objects.c" it may happen that we don't yield any new bytes after reading from the pipe. This is addressed by looping until we have produced at least a single byte of output. Drop the field from `struct odb_write_stream`. Again, same as in the preceding commit, this brings the structure a bit closer to its sibling `struct odb_read_stream`. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/unpack-objects.c | 15 ++++++++------- object-file.c | 13 ++++++++----- odb/source-inmemory.c | 9 ++++++++- odb/source-loose.c | 12 ++++++++---- odb/streaming.c | 5 +---- odb/streaming.h | 1 - t/unit-tests/u-odb-inmemory.c | 5 +++-- 7 files changed, 36 insertions(+), 24 deletions(-) diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index f3e0b504f43f13..b7c486ea949995 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -368,20 +368,20 @@ static ssize_t feed_input_zstream(struct odb_write_stream *in_stream, { struct input_zstream_data *data = in_stream->data; git_zstream *zstream = data->zstream; - void *in = fill(1); - if (in_stream->is_finished) + if (data->status != Z_OK) return 0; zstream->next_out = buf; zstream->avail_out = buf_len; - zstream->next_in = in; - zstream->avail_in = len; - data->status = git_inflate(zstream, 0); + while (data->status == Z_OK && zstream->avail_out == buf_len) { + zstream->next_in = fill(1); + zstream->avail_in = len; + data->status = git_inflate(zstream, 0); + use(len - zstream->avail_in); + } - in_stream->is_finished = data->status != Z_OK; - use(len - zstream->avail_in); return buf_len - zstream->avail_out; } @@ -397,6 +397,7 @@ static void stream_blob(unsigned long size, unsigned nr) struct obj_info *info = &obj_list[nr]; data.zstream = &zstream; + data.status = Z_OK; git_inflate_init(&zstream); if (odb_write_object_stream(the_repository->objects, &in_stream, &info->oid)) diff --git a/object-file.c b/object-file.c index b196abb596e87b..317c09dff8653f 100644 --- a/object-file.c +++ b/object-file.c @@ -716,12 +716,13 @@ static int hash_blob_stream(struct odb_write_stream *stream, git_hash_init(&ctx, hash_algo); git_hash_update(&ctx, buf, header_len); - while (!stream->is_finished) { + while (1) { ssize_t read_result = odb_write_stream_read(stream, buf, sizeof(buf)); - if (read_result < 0) return -1; + if (!read_result) + break; git_hash_update(&ctx, buf, read_result); bytes_hashed += read_result; @@ -749,6 +750,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state, unsigned hdrlen; int status = Z_OK; struct repo_config_values *cfg = repo_config_values(the_repository); + bool is_finished = false; size_t bytes_read = 0; git_deflate_init(&s, cfg->pack_compression_level); @@ -758,12 +760,13 @@ static void stream_blob_to_pack(struct transaction_packfile *state, s.avail_out = sizeof(obuf) - hdrlen; while (status != Z_STREAM_END) { - if (!stream->is_finished && !s.avail_in) { + if (!is_finished && !s.avail_in) { ssize_t rsize = odb_write_stream_read(stream, ibuf, sizeof(ibuf)); - if (rsize < 0) die("failed to read blob data"); + if (!rsize) + is_finished = true; git_hash_update(ctx, ibuf, rsize); @@ -772,7 +775,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state, bytes_read += rsize; } - status = git_deflate(&s, stream->is_finished ? Z_FINISH : 0); + status = git_deflate(&s, is_finished ? Z_FINISH : 0); if (!s.avail_out || status == Z_STREAM_END) { size_t written = s.next_out - obuf; diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index 398131e194f87c..01bb81c63cc2a4 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -265,10 +265,17 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source, int ret; CALLOC_ARRAY(data, stream->size); - while (!stream->is_finished) { + while (1) { ssize_t bytes_read; bytes_read = odb_write_stream_read(stream, buf, sizeof(buf)); + if (bytes_read < 0) { + ret = error("failed to read object stream"); + goto out; + } + if (!bytes_read) + break; + if (total_read + bytes_read > stream->size) { ret = error("object stream yielded more bytes than expected"); goto out; diff --git a/odb/source-loose.c b/odb/source-loose.c index 77a2adb52abf47..361b4e2a2a4574 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -859,6 +859,7 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, struct strbuf filename = STRBUF_INIT; unsigned char buf[8192]; int dirlen; + bool is_finished = false; char hdr[MAX_HEADER_LEN]; int hdrlen; @@ -889,7 +890,7 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, do { unsigned char *in0 = stream.next_in; - if (!stream.avail_in && !in_stream->is_finished) { + if (!stream.avail_in && !is_finished) { ssize_t read_len = odb_write_stream_read(in_stream, buf, sizeof(buf)); if (read_len < 0) { @@ -898,12 +899,15 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, goto cleanup; } + /* All data has been read. */ + if (!read_len) { + is_finished = true; + flush = 1; + } + stream.avail_in = read_len; stream.next_in = buf; in0 = buf; - /* All data has been read. */ - if (in_stream->is_finished) - flush = 1; } ret = write_loose_object_common(loose, &c, &compat_c, &stream, flush, in0, fd, compressed, sizeof(compressed)); diff --git a/odb/streaming.c b/odb/streaming.c index 38c2f6687c432d..912e75e682e6a5 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -310,7 +310,7 @@ static ssize_t read_object_fd(struct odb_write_stream *stream, ssize_t read_result; size_t count; - if (stream->is_finished) + if (!data->remaining) return 0; count = data->remaining < len ? data->remaining : len; @@ -319,8 +319,6 @@ static ssize_t read_object_fd(struct odb_write_stream *stream, return -1; data->remaining -= count; - if (!data->remaining) - stream->is_finished = 1; return read_result; } @@ -337,5 +335,4 @@ void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd, stream->data = data; stream->read = read_object_fd; stream->size = size; - stream->is_finished = 0; } diff --git a/odb/streaming.h b/odb/streaming.h index 4d7d31b5aa04f6..5e8e6e532e5660 100644 --- a/odb/streaming.h +++ b/odb/streaming.h @@ -56,7 +56,6 @@ struct odb_write_stream { ssize_t (*read)(struct odb_write_stream *, unsigned char *, size_t); void *data; size_t size; - int is_finished; }; /* diff --git a/t/unit-tests/u-odb-inmemory.c b/t/unit-tests/u-odb-inmemory.c index 5ccc52dccc06f9..4437140ed04e9b 100644 --- a/t/unit-tests/u-odb-inmemory.c +++ b/t/unit-tests/u-odb-inmemory.c @@ -277,6 +277,9 @@ static ssize_t membuf_write_stream_read(struct odb_write_stream *stream, struct membuf_write_stream *s = container_of(stream, struct membuf_write_stream, base); size_t chunk_size = 2; + if (s->offset == s->base.size) + return 0; + if (chunk_size > len) chunk_size = len; if (chunk_size > s->base.size - s->offset) @@ -285,8 +288,6 @@ static ssize_t membuf_write_stream_read(struct odb_write_stream *stream, memcpy(buf, s->buf + s->offset, chunk_size); s->offset += chunk_size; - if (s->offset == s->base.size) - s->base.is_finished = 1; return chunk_size; } From 8a51f6e8c5e1e1b849f41069dabc7bdfc1177d9b Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 5 Aug 2026 09:44:47 +0200 Subject: [PATCH 056/259] odb/streaming: support streaming arbitrary object types The object database supports the ability to write object streams into it. This functionality is used when we encounter a blob that is larger than "core.bigFileThreshold" so that we don't have to soak large files into memory. As we only ever write large files, the infrastructure doesn't support specifying any other object type than "blob". This limitation is quite artificial though: there is no reason why we shouldn't support writing arbitrary large objects with a stream. While it's very unlikely that we encounter a huge object other than a blob, users are known to be creative and sometimes like to inflict pain on themselves by creating commits or trees that are huge. Extend the infrastructure to support streaming arbitrary object types. For now we don't use this functionality anywhere, but it brings us a bit closer to unify `struct odb_read_stream` and `struct odb_write_stream`. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/unpack-objects.c | 1 + object-file.c | 31 +++++++++++++++---------------- odb/source-inmemory.c | 5 +++-- odb/source-loose.c | 2 +- odb/streaming.c | 3 ++- odb/streaming.h | 3 ++- odb/transaction.h | 2 +- t/unit-tests/u-odb-inmemory.c | 7 +++++-- 8 files changed, 30 insertions(+), 24 deletions(-) diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index b7c486ea949995..7439ec53be310d 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -393,6 +393,7 @@ static void stream_blob(unsigned long size, unsigned nr) .read = feed_input_zstream, .data = &data, .size = size, + .type = OBJ_BLOB, }; struct obj_info *info = &obj_list[nr]; diff --git a/object-file.c b/object-file.c index 317c09dff8653f..699a6a008ce5ba 100644 --- a/object-file.c +++ b/object-file.c @@ -702,9 +702,9 @@ static void prepare_packfile_transaction(struct odb_transaction_files *transacti die_errno("unable to write pack header"); } -static int hash_blob_stream(struct odb_write_stream *stream, - const struct git_hash_algo *hash_algo, - struct object_id *result_oid) +static int hash_stream(struct odb_write_stream *stream, + const struct git_hash_algo *hash_algo, + struct object_id *result_oid) { unsigned char buf[16384]; struct git_hash_ctx ctx; @@ -712,7 +712,7 @@ static int hash_blob_stream(struct odb_write_stream *stream, size_t bytes_hashed = 0; header_len = format_object_header((char *)buf, sizeof(buf), - OBJ_BLOB, stream->size); + stream->type, stream->size); git_hash_init(&ctx, hash_algo); git_hash_update(&ctx, buf, header_len); @@ -740,9 +740,9 @@ static int hash_blob_stream(struct odb_write_stream *stream, * Read the contents from the stream provided, streaming it to the * packfile in state while updating the hash in ctx. */ -static void stream_blob_to_pack(struct transaction_packfile *state, - struct git_hash_ctx *ctx, - struct odb_write_stream *stream) +static void stream_to_pack(struct transaction_packfile *state, + struct git_hash_ctx *ctx, + struct odb_write_stream *stream) { git_zstream s; unsigned char ibuf[16384]; @@ -755,7 +755,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state, git_deflate_init(&s, cfg->pack_compression_level); - hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), OBJ_BLOB, stream->size); + hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), stream->type, stream->size); s.next_out = obuf + hdrlen; s.avail_out = sizeof(obuf) - hdrlen; @@ -764,7 +764,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state, ssize_t rsize = odb_write_stream_read(stream, ibuf, sizeof(ibuf)); if (rsize < 0) - die("failed to read blob data"); + die("failed to read object data"); if (!rsize) is_finished = true; @@ -797,7 +797,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state, } if (bytes_read != stream->size) - die("read %" PRIuMAX " bytes of blob data, but expected %" PRIuMAX " bytes", + die("read %" PRIuMAX " bytes of object data, but expected %" PRIuMAX " bytes", (uintmax_t)bytes_read, (uintmax_t)stream->size); git_deflate_end(&s); @@ -868,7 +868,7 @@ static void flush_packfile_transaction(struct odb_transaction_files *transaction * result, which we need to know beforehand when writing a git object. * Since the primary motivation for trying to stream from the working * tree file and to avoid mmaping it in core is to deal with large - * binary blobs, they generally do not want to get any conversion, and + * objects, they generally do not want to get any conversion, and * callers should avoid this code path when filters are requested. */ static int odb_transaction_files_write_object_stream(struct odb_transaction *base, @@ -886,7 +886,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas struct pack_idx_entry *idx; header_len = format_object_header((char *)obuf, sizeof(obuf), - OBJ_BLOB, stream->size); + stream->type, stream->size); git_hash_init(&ctx, transaction->base.source->odb->repo->hash_algo); git_hash_update(&ctx, obuf, header_len); @@ -911,7 +911,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas hashfile_checkpoint(state->f, &checkpoint); idx->offset = state->offset; crc32_begin(state->f); - stream_blob_to_pack(state, &ctx, stream); + stream_to_pack(state, &ctx, stream); git_hash_final_oid(result_oid, &ctx); idx->crc32 = crc32_end(state->f); @@ -953,7 +953,7 @@ int index_fd(struct index_state *istate, struct object_id *oid, type, path, flags); } else { struct odb_write_stream stream; - odb_write_stream_from_fd(&stream, fd, xsize_t(st->st_size)); + odb_write_stream_from_fd(&stream, fd, xsize_t(st->st_size), OBJ_BLOB); if (flags & INDEX_WRITE_OBJECT) { struct object_database *odb = the_repository->objects; @@ -968,8 +968,7 @@ int index_fd(struct index_state *istate, struct object_id *oid, if (!inflight) odb_transaction_commit(transaction); } else { - ret = hash_blob_stream(&stream, - the_repository->hash_algo, oid); + ret = hash_stream(&stream, the_repository->hash_algo, oid); } odb_write_stream_release(&stream); diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index 01bb81c63cc2a4..139618024a6023 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -290,10 +290,11 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source, goto out; } - hash_object_file(source->odb->repo->hash_algo, data, total_read, OBJ_BLOB, oid); + hash_object_file(source->odb->repo->hash_algo, data, total_read, + stream->type, oid); ret = odb_source_inmemory_write_object(source, data, stream->size, - OBJ_BLOB, oid, NULL, NULL, 0); + stream->type, oid, NULL, NULL, 0); if (ret < 0) goto out; diff --git a/odb/source-loose.c b/odb/source-loose.c index 361b4e2a2a4574..5681a38f03d4d5 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -868,7 +868,7 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, /* Since oid is not determined, save tmp file to odb path. */ strbuf_addf(&filename, "%s/", loose->base.path); - hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, in_stream->size); + hdrlen = format_object_header(hdr, sizeof(hdr), in_stream->type, in_stream->size); /* * Common steps for write_loose_object and stream_loose_object to diff --git a/odb/streaming.c b/odb/streaming.c index 912e75e682e6a5..0918cad4267bd3 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -324,7 +324,7 @@ static ssize_t read_object_fd(struct odb_write_stream *stream, } void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd, - size_t size) + size_t size, enum object_type type) { struct read_object_fd_data *data; @@ -335,4 +335,5 @@ void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd, stream->data = data; stream->read = read_object_fd; stream->size = size; + stream->type = type; } diff --git a/odb/streaming.h b/odb/streaming.h index 5e8e6e532e5660..3c8ed551293fd4 100644 --- a/odb/streaming.h +++ b/odb/streaming.h @@ -56,6 +56,7 @@ struct odb_write_stream { ssize_t (*read)(struct odb_write_stream *, unsigned char *, size_t); void *data; size_t size; + enum object_type type; }; /* @@ -92,6 +93,6 @@ int odb_stream_blob_to_fd(struct object_database *odb, * Sets up an ODB write stream that reads from an fd. */ void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd, - size_t size); + size_t size, enum object_type type); #endif /* STREAMING_H */ diff --git a/odb/transaction.h b/odb/transaction.h index ffb279314cfd21..1eb74664c6bb1d 100644 --- a/odb/transaction.h +++ b/odb/transaction.h @@ -24,7 +24,7 @@ struct odb_transaction { /* * This callback is expected to write the given object stream into - * the ODB transaction. Note that for now, only blobs support streaming. + * the ODB transaction. * * The resulting object ID shall be written into the out pointer. The * callback is expected to return 0 on success, a negative error code diff --git a/t/unit-tests/u-odb-inmemory.c b/t/unit-tests/u-odb-inmemory.c index 4437140ed04e9b..1ab07af6d666cd 100644 --- a/t/unit-tests/u-odb-inmemory.c +++ b/t/unit-tests/u-odb-inmemory.c @@ -297,8 +297,11 @@ void test_odb_inmemory__write_object_stream(void) struct odb_source_inmemory *source = odb_source_inmemory_new(odb); const char data[] = "foobar"; struct membuf_write_stream stream = { - .base.read = membuf_write_stream_read, - .base.size = strlen(data), + .base = { + .read = membuf_write_stream_read, + .size = strlen(data), + .type = OBJ_BLOB, + }, .buf = data, }; struct object_id written_oid; From a59e4798f0b3edc59326312b0891521d6886ee7c Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 5 Aug 2026 09:44:48 +0200 Subject: [PATCH 057/259] odb/streaming: rename `struct odb_read_stream` Rename `struct odb_read_stream` to just `struct odb_stream`. This prepares for unification of the two different types of streams, as these provide the same functionality with the preceding refactorings. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- archive-tar.c | 6 ++--- archive-zip.c | 10 ++++---- builtin/index-pack.c | 6 ++--- builtin/pack-objects.c | 14 +++++----- object-file.c | 4 +-- object-file.h | 2 +- object.c | 6 ++--- odb/source-files.c | 2 +- odb/source-inmemory.c | 8 +++--- odb/source-loose.c | 8 +++--- odb/source-packed.c | 2 +- odb/source.h | 6 ++--- odb/streaming.c | 48 +++++++++++++++++------------------ odb/streaming.h | 24 +++++++++--------- pack-check.c | 4 +-- packfile.c | 8 +++--- packfile.h | 4 +-- t/unit-tests/u-odb-inmemory.c | 12 ++++----- 18 files changed, 87 insertions(+), 87 deletions(-) diff --git a/archive-tar.c b/archive-tar.c index 0fc70d13a8807e..df2d7fb8e936a8 100644 --- a/archive-tar.c +++ b/archive-tar.c @@ -129,7 +129,7 @@ static void write_trailer(void) */ static int stream_blocked(struct repository *r, const struct object_id *oid) { - struct odb_read_stream *st; + struct odb_stream *st; char buf[BLOCKSIZE]; ssize_t readlen; @@ -137,12 +137,12 @@ static int stream_blocked(struct repository *r, const struct object_id *oid) if (!st) return error(_("cannot stream blob %s"), oid_to_hex(oid)); for (;;) { - readlen = odb_read_stream_read(st, buf, sizeof(buf)); + readlen = odb_stream_read(st, buf, sizeof(buf)); if (readlen <= 0) break; do_write_blocked(buf, readlen); } - odb_read_stream_close(st); + odb_stream_close(st); if (!readlen) finish_record(); return readlen; diff --git a/archive-zip.c b/archive-zip.c index 97ea8d60d6187b..8095fd04d5b9ba 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -309,7 +309,7 @@ static int write_zip_entry(struct archiver_args *args, enum zip_method method; unsigned char *out; void *deflated = NULL; - struct odb_read_stream *stream = NULL; + struct odb_stream *stream = NULL; unsigned long flags = 0; int is_binary = -1; const char *path_without_prefix = path + args->baselen; @@ -428,7 +428,7 @@ static int write_zip_entry(struct archiver_args *args, ssize_t readlen; for (;;) { - readlen = odb_read_stream_read(stream, buf, sizeof(buf)); + readlen = odb_stream_read(stream, buf, sizeof(buf)); if (readlen <= 0) break; crc = crc32(crc, buf, readlen); @@ -438,7 +438,7 @@ static int write_zip_entry(struct archiver_args *args, buf, readlen); write_or_die(1, buf, readlen); } - odb_read_stream_close(stream); + odb_stream_close(stream); if (readlen) return readlen; @@ -461,7 +461,7 @@ static int write_zip_entry(struct archiver_args *args, zstream.avail_out = sizeof(compressed); for (;;) { - readlen = odb_read_stream_read(stream, buf, sizeof(buf)); + readlen = odb_stream_read(stream, buf, sizeof(buf)); if (readlen <= 0) break; crc = crc32(crc, buf, readlen); @@ -485,7 +485,7 @@ static int write_zip_entry(struct archiver_args *args, } } - odb_read_stream_close(stream); + odb_stream_close(stream); if (readlen) return readlen; diff --git a/builtin/index-pack.c b/builtin/index-pack.c index bc86925ad04340..7226da3e65ad8d 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -763,7 +763,7 @@ static void find_ref_delta_children(const struct object_id *oid, struct compare_data { struct object_entry *entry; - struct odb_read_stream *st; + struct odb_stream *st; unsigned char *buf; unsigned long buf_size; }; @@ -780,7 +780,7 @@ static int compare_objects(const unsigned char *buf, unsigned long size, } while (size) { - ssize_t len = odb_read_stream_read(data->st, data->buf, size); + ssize_t len = odb_stream_read(data->st, data->buf, size); if (len == 0) die(_("SHA1 COLLISION FOUND WITH %s !"), oid_to_hex(&data->entry->idx.oid)); @@ -813,7 +813,7 @@ static int check_collison(struct object_entry *entry) die(_("SHA1 COLLISION FOUND WITH %s !"), oid_to_hex(&entry->idx.oid)); unpack_data(entry, compare_objects, &data); - odb_read_stream_close(data.st); + odb_stream_close(data.st); free(data.buf); return 0; } diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 1ec5b6f206366e..683160c6bbb6ab 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -411,7 +411,7 @@ static unsigned long do_compress(void **pptr, unsigned long size) return stream.total_out; } -static unsigned long write_large_blob_data(struct odb_read_stream *st, struct hashfile *f, +static unsigned long write_large_blob_data(struct odb_stream *st, struct hashfile *f, const struct object_id *oid) { git_zstream stream; @@ -425,7 +425,7 @@ static unsigned long write_large_blob_data(struct odb_read_stream *st, struct ha for (;;) { ssize_t readlen; int zret = Z_OK; - readlen = odb_read_stream_read(st, ibuf, sizeof(ibuf)); + readlen = odb_stream_read(st, ibuf, sizeof(ibuf)); if (readlen == -1) die(_("unable to read %s"), oid_to_hex(oid)); @@ -521,7 +521,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent unsigned hdrlen; enum object_type type; void *buf; - struct odb_read_stream *st = NULL; + struct odb_stream *st = NULL; const unsigned hashsz = the_hash_algo->rawsz; if (!usable_delta) { @@ -589,7 +589,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent dheader[--pos] = 128 | (--ofs & 127); if (limit && hdrlen + sizeof(dheader) - pos + datalen + hashsz >= limit) { if (st) - odb_read_stream_close(st); + odb_stream_close(st); free(buf); return 0; } @@ -603,7 +603,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent */ if (limit && hdrlen + hashsz + datalen + hashsz >= limit) { if (st) - odb_read_stream_close(st); + odb_stream_close(st); free(buf); return 0; } @@ -613,7 +613,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent } else { if (limit && hdrlen + datalen + hashsz >= limit) { if (st) - odb_read_stream_close(st); + odb_stream_close(st); free(buf); return 0; } @@ -621,7 +621,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent } if (st) { datalen = write_large_blob_data(st, f, &entry->idx.oid); - odb_read_stream_close(st); + odb_stream_close(st); } else { hashwrite(f, buf, datalen); free(buf); diff --git a/object-file.c b/object-file.c index 699a6a008ce5ba..5f6d584c356f28 100644 --- a/object-file.c +++ b/object-file.c @@ -122,7 +122,7 @@ int check_object_signature(struct repository *r, const struct object_id *oid, } int stream_object_signature(struct repository *r, - struct odb_read_stream *st, + struct odb_stream *st, const struct object_id *oid) { struct object_id real_oid; @@ -138,7 +138,7 @@ int stream_object_signature(struct repository *r, git_hash_update(&c, hdr, hdrlen); for (;;) { char buf[1024 * 16]; - ssize_t readlen = odb_read_stream_read(st, buf, sizeof(buf)); + ssize_t readlen = odb_stream_read(st, buf, sizeof(buf)); if (readlen < 0) return -1; if (!readlen) diff --git a/object-file.h b/object-file.h index 805f2cfa289661..f44758c4f8ba01 100644 --- a/object-file.h +++ b/object-file.h @@ -101,7 +101,7 @@ int check_object_signature(struct repository *r, const struct object_id *oid, * the streaming interface and rehash it to do the same. */ int stream_object_signature(struct repository *r, - struct odb_read_stream *stream, + struct odb_stream *stream, const struct object_id *oid); enum finalize_object_file_flags { diff --git a/object.c b/object.c index 23b84aa7e29531..37e6efee47ff2a 100644 --- a/object.c +++ b/object.c @@ -345,7 +345,7 @@ struct object *parse_object_with_flags(struct repository *r, if ((!obj || obj->type == OBJ_NONE || obj->type == OBJ_BLOB) && odb_read_object_info(r->objects, oid, NULL) == OBJ_BLOB) { if (!skip_hash) { - struct odb_read_stream *stream = odb_read_stream_open(r->objects, oid, NULL); + struct odb_stream *stream = odb_read_stream_open(r->objects, oid, NULL); if (!stream) { error(_("unable to open object stream for %s"), oid_to_hex(oid)); @@ -354,11 +354,11 @@ struct object *parse_object_with_flags(struct repository *r, if (stream_object_signature(r, stream, repl) < 0) { error(_("hash mismatch %s"), oid_to_hex(oid)); - odb_read_stream_close(stream); + odb_stream_close(stream); return NULL; } - odb_read_stream_close(stream); + odb_stream_close(stream); } parse_blob_buffer(lookup_blob(r, oid)); return lookup_object(r, oid); diff --git a/odb/source-files.c b/odb/source-files.c index f51960bd71bb11..f7b8c76549c393 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -63,7 +63,7 @@ static int odb_source_files_read_object_info(struct odb_source *source, return -1; } -static int odb_source_files_read_object_stream(struct odb_read_stream **out, +static int odb_source_files_read_object_stream(struct odb_stream **out, struct odb_source *source, const struct object_id *oid) { diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index 139618024a6023..485d58703658b0 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -73,12 +73,12 @@ static int odb_source_inmemory_read_object_info(struct odb_source *source, } struct odb_read_stream_inmemory { - struct odb_read_stream base; + struct odb_stream base; const unsigned char *buf; size_t offset; }; -static ssize_t odb_read_stream_inmemory_read(struct odb_read_stream *stream, +static ssize_t odb_read_stream_inmemory_read(struct odb_stream *stream, char *buf, size_t buf_len) { struct odb_read_stream_inmemory *inmemory = @@ -94,12 +94,12 @@ static ssize_t odb_read_stream_inmemory_read(struct odb_read_stream *stream, return bytes; } -static int odb_read_stream_inmemory_close(struct odb_read_stream *stream UNUSED) +static int odb_read_stream_inmemory_close(struct odb_stream *stream UNUSED) { return 0; } -static int odb_source_inmemory_read_object_stream(struct odb_read_stream **out, +static int odb_source_inmemory_read_object_stream(struct odb_stream **out, struct odb_source *source, const struct object_id *oid) { diff --git a/odb/source-loose.c b/odb/source-loose.c index 5681a38f03d4d5..038defd9059408 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -278,7 +278,7 @@ static void *odb_source_loose_map_object(struct odb_source_loose *loose, } struct odb_loose_read_stream { - struct odb_read_stream base; + struct odb_stream base; git_zstream z; enum { ODB_LOOSE_READ_STREAM_INUSE, @@ -292,7 +292,7 @@ struct odb_loose_read_stream { int hdr_used; }; -static ssize_t read_istream_loose(struct odb_read_stream *_st, char *buf, size_t sz) +static ssize_t read_istream_loose(struct odb_stream *_st, char *buf, size_t sz) { struct odb_loose_read_stream *st = container_of(_st, struct odb_loose_read_stream, base); @@ -339,7 +339,7 @@ static ssize_t read_istream_loose(struct odb_read_stream *_st, char *buf, size_t return total_read; } -static int close_istream_loose(struct odb_read_stream *_st) +static int close_istream_loose(struct odb_stream *_st) { struct odb_loose_read_stream *st = container_of(_st, struct odb_loose_read_stream, base); @@ -350,7 +350,7 @@ static int close_istream_loose(struct odb_read_stream *_st) return 0; } -static int odb_source_loose_read_object_stream(struct odb_read_stream **out, +static int odb_source_loose_read_object_stream(struct odb_stream **out, struct odb_source *source, const struct object_id *oid) { diff --git a/odb/source-packed.c b/odb/source-packed.c index e6ff74833b8bec..b3186ca5933ae9 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -70,7 +70,7 @@ static int odb_source_packed_read_object_info(struct odb_source *source, return 0; } -static int odb_source_packed_read_object_stream(struct odb_read_stream **out, +static int odb_source_packed_read_object_stream(struct odb_stream **out, struct odb_source *source, const struct object_id *oid) { diff --git a/odb/source.h b/odb/source.h index 0080148ba71078..89b0c396822c95 100644 --- a/odb/source.h +++ b/odb/source.h @@ -26,7 +26,7 @@ enum odb_source_type { }; struct object_id; -struct odb_read_stream; +struct odb_stream; struct strvec; /* @@ -125,7 +125,7 @@ struct odb_source { * The callback is expected to return a negative error code in case * creating the object stream has failed, 0 otherwise. */ - int (*read_object_stream)(struct odb_read_stream **out, + int (*read_object_stream)(struct odb_stream **out, struct odb_source *source, const struct object_id *oid); @@ -339,7 +339,7 @@ static inline int odb_source_read_object_info(struct odb_source *source, * Create a new read stream for the given object ID. Returns 0 on success, a * negative error code otherwise. */ -static inline int odb_source_read_object_stream(struct odb_read_stream **out, +static inline int odb_source_read_object_stream(struct odb_stream **out, struct odb_source *source, const struct object_id *oid) { diff --git a/odb/streaming.c b/odb/streaming.c index 0918cad4267bd3..98e2152e364741 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -20,8 +20,8 @@ *****************************************************************/ struct odb_filtered_read_stream { - struct odb_read_stream base; - struct odb_read_stream *upstream; + struct odb_stream base; + struct odb_stream *upstream; struct stream_filter *filter; char ibuf[FILTER_BUFFER]; char obuf[FILTER_BUFFER]; @@ -30,14 +30,14 @@ struct odb_filtered_read_stream { int input_finished; }; -static int close_istream_filtered(struct odb_read_stream *_fs) +static int close_istream_filtered(struct odb_stream *_fs) { struct odb_filtered_read_stream *fs = (struct odb_filtered_read_stream *)_fs; free_stream_filter(fs->filter); - return odb_read_stream_close(fs->upstream); + return odb_stream_close(fs->upstream); } -static ssize_t read_istream_filtered(struct odb_read_stream *_fs, char *buf, +static ssize_t read_istream_filtered(struct odb_stream *_fs, char *buf, size_t sz) { struct odb_filtered_read_stream *fs = (struct odb_filtered_read_stream *)_fs; @@ -86,7 +86,7 @@ static ssize_t read_istream_filtered(struct odb_read_stream *_fs, char *buf, /* refill the input from the upstream */ if (!fs->input_finished) { - fs->i_end = odb_read_stream_read(fs->upstream, fs->ibuf, FILTER_BUFFER); + fs->i_end = odb_stream_read(fs->upstream, fs->ibuf, FILTER_BUFFER); if (fs->i_end < 0) return -1; if (fs->i_end) @@ -97,8 +97,8 @@ static ssize_t read_istream_filtered(struct odb_read_stream *_fs, char *buf, return filled; } -static struct odb_read_stream *attach_stream_filter(struct odb_read_stream *st, - struct stream_filter *filter) +static struct odb_stream *attach_stream_filter(struct odb_stream *st, + struct stream_filter *filter) { struct odb_filtered_read_stream *fs; @@ -120,19 +120,19 @@ static struct odb_read_stream *attach_stream_filter(struct odb_read_stream *st, *****************************************************************/ struct odb_incore_read_stream { - struct odb_read_stream base; + struct odb_stream base; char *buf; /* from odb_read_object_info_extended() */ unsigned long read_ptr; }; -static int close_istream_incore(struct odb_read_stream *_st) +static int close_istream_incore(struct odb_stream *_st) { struct odb_incore_read_stream *st = (struct odb_incore_read_stream *)_st; free(st->buf); return 0; } -static ssize_t read_istream_incore(struct odb_read_stream *_st, char *buf, size_t sz) +static ssize_t read_istream_incore(struct odb_stream *_st, char *buf, size_t sz) { struct odb_incore_read_stream *st = (struct odb_incore_read_stream *)_st; size_t read_size = sz; @@ -147,7 +147,7 @@ static ssize_t read_istream_incore(struct odb_read_stream *_st, char *buf, size_ return read_size; } -static int open_istream_incore(struct odb_read_stream **out, +static int open_istream_incore(struct odb_stream **out, struct object_database *odb, const struct object_id *oid) { @@ -178,7 +178,7 @@ static int open_istream_incore(struct odb_read_stream **out, * static helpers variables and functions for users of streaming interface *****************************************************************************/ -static int istream_source(struct odb_read_stream **out, +static int istream_source(struct odb_stream **out, struct object_database *odb, const struct object_id *oid) { @@ -196,23 +196,23 @@ static int istream_source(struct odb_read_stream **out, * Users of streaming interface ****************************************************************/ -int odb_read_stream_close(struct odb_read_stream *st) +int odb_stream_close(struct odb_stream *st) { int r = st->close(st); free(st); return r; } -ssize_t odb_read_stream_read(struct odb_read_stream *st, void *buf, size_t sz) +ssize_t odb_stream_read(struct odb_stream *st, void *buf, size_t sz) { return st->read(st, buf, sz); } -struct odb_read_stream *odb_read_stream_open(struct object_database *odb, - const struct object_id *oid, - struct stream_filter *filter) +struct odb_stream *odb_read_stream_open(struct object_database *odb, + const struct object_id *oid, + struct stream_filter *filter) { - struct odb_read_stream *st; + struct odb_stream *st; const struct object_id *real = lookup_replace_object(odb->repo, oid); int ret = istream_source(&st, odb, real); @@ -221,9 +221,9 @@ struct odb_read_stream *odb_read_stream_open(struct object_database *odb, if (filter) { /* Add "&& !is_null_stream_filter(filter)" for performance */ - struct odb_read_stream *nst = attach_stream_filter(st, filter); + struct odb_stream *nst = attach_stream_filter(st, filter); if (!nst) { - odb_read_stream_close(st); + odb_stream_close(st); return NULL; } st = nst; @@ -248,7 +248,7 @@ int odb_stream_blob_to_fd(struct object_database *odb, struct stream_filter *filter, int can_seek) { - struct odb_read_stream *st; + struct odb_stream *st; ssize_t kept = 0; int result = -1; @@ -263,7 +263,7 @@ int odb_stream_blob_to_fd(struct object_database *odb, for (;;) { char buf[1024 * 16]; ssize_t wrote, holeto; - ssize_t readlen = odb_read_stream_read(st, buf, sizeof(buf)); + ssize_t readlen = odb_stream_read(st, buf, sizeof(buf)); if (readlen < 0) goto close_and_exit; @@ -294,7 +294,7 @@ int odb_stream_blob_to_fd(struct object_database *odb, result = 0; close_and_exit: - odb_read_stream_close(st); + odb_stream_close(st); return result; } diff --git a/odb/streaming.h b/odb/streaming.h index 3c8ed551293fd4..037954c2315895 100644 --- a/odb/streaming.h +++ b/odb/streaming.h @@ -8,19 +8,19 @@ #include "odb.h" struct object_database; -struct odb_read_stream; +struct odb_stream; struct stream_filter; -typedef int (*odb_read_stream_close_fn)(struct odb_read_stream *); -typedef ssize_t (*odb_read_stream_read_fn)(struct odb_read_stream *, char *, size_t); +typedef int (*odb_stream_close_fn)(struct odb_stream *); +typedef ssize_t (*odb_stream_read_fn)(struct odb_stream *, char *, size_t); /* * A stream that can be used to read an object from the object database without * loading all of it into memory. */ -struct odb_read_stream { - odb_read_stream_close_fn close; - odb_read_stream_read_fn read; +struct odb_stream { + odb_stream_close_fn close; + odb_stream_read_fn read; enum object_type type; size_t size; /* inflated size of full object */ }; @@ -31,22 +31,22 @@ struct odb_read_stream { * * Returns the stream on success, a `NULL` pointer otherwise. */ -struct odb_read_stream *odb_read_stream_open(struct object_database *odb, - const struct object_id *oid, - struct stream_filter *filter); +struct odb_stream *odb_read_stream_open(struct object_database *odb, + const struct object_id *oid, + struct stream_filter *filter); /* - * Close the given read stream and release all resources associated with it. + * Close the given object stream and release all resources associated with it. * Returns 0 on success, a negative error code otherwise. */ -int odb_read_stream_close(struct odb_read_stream *stream); +int odb_stream_close(struct odb_stream *stream); /* * Read data from the stream into the buffer. Returns 0 on EOF and the number * of bytes read on success. Returns a negative error code in case reading from * the stream fails. */ -ssize_t odb_read_stream_read(struct odb_read_stream *stream, void *buf, size_t len); +ssize_t odb_stream_read(struct odb_stream *stream, void *buf, size_t len); /* * A stream that provides an object to be written to the object database without diff --git a/pack-check.c b/pack-check.c index c3b8db7c5c41a6..1b5e26847d0b2a 100644 --- a/pack-check.c +++ b/pack-check.c @@ -106,7 +106,7 @@ static int verify_packfile(struct repository *r, QSORT(entries, nr_objects, compare_entries); for (i = 0; i < nr_objects; i++) { - struct odb_read_stream *stream = NULL; + struct odb_stream *stream = NULL; void *data; struct object_id oid; enum object_type type; @@ -171,7 +171,7 @@ static int verify_packfile(struct repository *r, display_progress(progress, base_count + i); if (stream) - odb_read_stream_close(stream); + odb_stream_close(stream); free(data); } diff --git a/packfile.c b/packfile.c index 0eee45055f833e..70254573a3f4dc 100644 --- a/packfile.c +++ b/packfile.c @@ -2115,7 +2115,7 @@ int parse_pack_header_option(const char *in, unsigned char *out, unsigned int *l } struct odb_packed_read_stream { - struct odb_read_stream base; + struct odb_stream base; struct packed_git *pack; git_zstream z; enum { @@ -2127,7 +2127,7 @@ struct odb_packed_read_stream { off_t pos; }; -static ssize_t read_istream_pack_non_delta(struct odb_read_stream *_st, char *buf, +static ssize_t read_istream_pack_non_delta(struct odb_stream *_st, char *buf, size_t sz) { struct odb_packed_read_stream *st = (struct odb_packed_read_stream *)_st; @@ -2187,7 +2187,7 @@ static ssize_t read_istream_pack_non_delta(struct odb_read_stream *_st, char *bu return total_read; } -static int close_istream_pack_non_delta(struct odb_read_stream *_st) +static int close_istream_pack_non_delta(struct odb_stream *_st) { struct odb_packed_read_stream *st = (struct odb_packed_read_stream *)_st; if (st->z_state == ODB_PACKED_READ_STREAM_INUSE) @@ -2195,7 +2195,7 @@ static int close_istream_pack_non_delta(struct odb_read_stream *_st) return 0; } -int packfile_read_object_stream(struct odb_read_stream **out, +int packfile_read_object_stream(struct odb_stream **out, const struct object_id *oid, struct packed_git *pack, off_t offset) diff --git a/packfile.h b/packfile.h index e1f77152b5c4bf..f913cb3d0c589c 100644 --- a/packfile.h +++ b/packfile.h @@ -12,7 +12,7 @@ /* in odb.h */ struct object_info; -struct odb_read_stream; +struct odb_stream; struct packed_git { struct pack_window *windows; @@ -306,7 +306,7 @@ off_t get_delta_base(struct packed_git *p, struct pack_window **w_curs, off_t *curpos, enum object_type type, off_t delta_obj_offset); -int packfile_read_object_stream(struct odb_read_stream **out, +int packfile_read_object_stream(struct odb_stream **out, const struct object_id *oid, struct packed_git *pack, off_t offset); diff --git a/t/unit-tests/u-odb-inmemory.c b/t/unit-tests/u-odb-inmemory.c index 1ab07af6d666cd..839a0fd3b753b2 100644 --- a/t/unit-tests/u-odb-inmemory.c +++ b/t/unit-tests/u-odb-inmemory.c @@ -100,7 +100,7 @@ void test_odb_inmemory__read_written_object(void) void test_odb_inmemory__read_stream_object(void) { struct odb_source_inmemory *source = odb_source_inmemory_new(odb); - struct odb_read_stream *stream; + struct odb_stream *stream; struct object_id written_oid; const char data[] = "foobar"; char buf[3] = { 0 }; @@ -112,15 +112,15 @@ void test_odb_inmemory__read_stream_object(void) cl_assert_equal_i(stream->type, OBJ_BLOB); cl_assert_equal_u(stream->size, 6); - cl_assert_equal_i(odb_read_stream_read(stream, buf, 2), 2); + cl_assert_equal_i(odb_stream_read(stream, buf, 2), 2); cl_assert_equal_s(buf, "fo"); - cl_assert_equal_i(odb_read_stream_read(stream, buf, 2), 2); + cl_assert_equal_i(odb_stream_read(stream, buf, 2), 2); cl_assert_equal_s(buf, "ob"); - cl_assert_equal_i(odb_read_stream_read(stream, buf, 2), 2); + cl_assert_equal_i(odb_stream_read(stream, buf, 2), 2); cl_assert_equal_s(buf, "ar"); - cl_assert_equal_i(odb_read_stream_read(stream, buf, 2), 0); + cl_assert_equal_i(odb_stream_read(stream, buf, 2), 0); - odb_read_stream_close(stream); + odb_stream_close(stream); odb_source_free(&source->base); } From e837b812fe3521cadab2923ec7457c1ebcaeda43 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 5 Aug 2026 09:44:49 +0200 Subject: [PATCH 058/259] odb/streaming: consolidate read and write streams The `struct odb_read_stream` and `struct odb_write_stream` both provide the same functionality: they allow a caller to read object data from an arbitrary source. Historically, the only difference was that the read stream was used to read data out of the object database, whereas the write stream was used to write data into the object database, but the interfaces were mostly the same. Over the preceding commits we have refactored the write stream to have almost exactly the same interface as the read stream. With these refactorings we can now easily merge those two streams into a single interface that's used for both use cases. While most of the changes are mechanical, there are two sites that need special mention: - "builtin/unpack-objects.c" creates a write stream from compressed object data. - "odb/streaming.c" creates a write stream from a file descriptor. Adapting these sites to yield the new stream type requires a couple more changes. Most importantly, instead of embedding the pointer to the data in `struct odb_write_stream`, we now allocate a structure that wraps the new `struct odb_stream` base. Other than that though, the changes are rather straight forward. Some of the structures and functions are now somewhat misnamed. These will be fixed in subsequent commits. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/unpack-objects.c | 31 ++++++++++++++++--------------- object-file.c | 25 ++++++++++++------------- odb.c | 2 +- odb.h | 4 ++-- odb/source-files.c | 2 +- odb/source-inmemory.c | 4 ++-- odb/source-loose.c | 6 +++--- odb/source-packed.c | 2 +- odb/source.h | 4 ++-- odb/streaming.c | 35 ++++++++++++++++------------------- odb/streaming.h | 31 +++---------------------------- odb/transaction.c | 2 +- odb/transaction.h | 4 ++-- t/unit-tests/u-odb-inmemory.c | 6 +++--- 14 files changed, 65 insertions(+), 93 deletions(-) diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index 7439ec53be310d..05a2d48011fc73 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -359,20 +359,21 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size, } struct input_zstream_data { + struct odb_stream base; git_zstream *zstream; int status; }; -static ssize_t feed_input_zstream(struct odb_write_stream *in_stream, - unsigned char *buf, size_t buf_len) +static ssize_t feed_input_zstream(struct odb_stream *in_stream, + char *buf, size_t buf_len) { - struct input_zstream_data *data = in_stream->data; + struct input_zstream_data *data = container_of(in_stream, struct input_zstream_data, base); git_zstream *zstream = data->zstream; if (data->status != Z_OK) return 0; - zstream->next_out = buf; + zstream->next_out = (unsigned char *) buf; zstream->avail_out = buf_len; while (data->status == Z_OK && zstream->avail_out == buf_len) { @@ -388,24 +389,24 @@ static ssize_t feed_input_zstream(struct odb_write_stream *in_stream, static void stream_blob(unsigned long size, unsigned nr) { git_zstream zstream = { 0 }; - struct input_zstream_data data = { 0 }; - struct odb_write_stream in_stream = { - .read = feed_input_zstream, - .data = &data, - .size = size, - .type = OBJ_BLOB, + struct input_zstream_data in_stream = { + .base = { + .read = feed_input_zstream, + .size = size, + .type = OBJ_BLOB, + }, + .zstream = &zstream, + .status = Z_OK, }; struct obj_info *info = &obj_list[nr]; - data.zstream = &zstream; - data.status = Z_OK; git_inflate_init(&zstream); - if (odb_write_object_stream(the_repository->objects, &in_stream, &info->oid)) + if (odb_write_object_stream(the_repository->objects, &in_stream.base, &info->oid)) die(_("failed to write object in stream")); - if (data.status != Z_STREAM_END) - die(_("inflate returned (%d)"), data.status); + if (in_stream.status != Z_STREAM_END) + die(_("inflate returned (%d)"), in_stream.status); git_inflate_end(&zstream); if (strict) { diff --git a/object-file.c b/object-file.c index 5f6d584c356f28..068c6e56726db9 100644 --- a/object-file.c +++ b/object-file.c @@ -702,7 +702,7 @@ static void prepare_packfile_transaction(struct odb_transaction_files *transacti die_errno("unable to write pack header"); } -static int hash_stream(struct odb_write_stream *stream, +static int hash_stream(struct odb_stream *stream, const struct git_hash_algo *hash_algo, struct object_id *result_oid) { @@ -717,8 +717,8 @@ static int hash_stream(struct odb_write_stream *stream, git_hash_update(&ctx, buf, header_len); while (1) { - ssize_t read_result = odb_write_stream_read(stream, buf, - sizeof(buf)); + ssize_t read_result = odb_stream_read(stream, buf, + sizeof(buf)); if (read_result < 0) return -1; if (!read_result) @@ -742,7 +742,7 @@ static int hash_stream(struct odb_write_stream *stream, */ static void stream_to_pack(struct transaction_packfile *state, struct git_hash_ctx *ctx, - struct odb_write_stream *stream) + struct odb_stream *stream) { git_zstream s; unsigned char ibuf[16384]; @@ -761,8 +761,8 @@ static void stream_to_pack(struct transaction_packfile *state, while (status != Z_STREAM_END) { if (!is_finished && !s.avail_in) { - ssize_t rsize = odb_write_stream_read(stream, ibuf, - sizeof(ibuf)); + ssize_t rsize = odb_stream_read(stream, ibuf, + sizeof(ibuf)); if (rsize < 0) die("failed to read object data"); if (!rsize) @@ -872,7 +872,7 @@ static void flush_packfile_transaction(struct odb_transaction_files *transaction * callers should avoid this code path when filters are requested. */ static int odb_transaction_files_write_object_stream(struct odb_transaction *base, - struct odb_write_stream *stream, + struct odb_stream *stream, struct object_id *result_oid) { struct odb_transaction_files *transaction = container_of(base, @@ -952,8 +952,8 @@ int index_fd(struct index_state *istate, struct object_id *oid, ret = index_core(istate, oid, fd, xsize_t(st->st_size), type, path, flags); } else { - struct odb_write_stream stream; - odb_write_stream_from_fd(&stream, fd, xsize_t(st->st_size), OBJ_BLOB); + struct odb_stream *stream = odb_write_stream_from_fd(fd, xsize_t(st->st_size), + OBJ_BLOB); if (flags & INDEX_WRITE_OBJECT) { struct object_database *odb = the_repository->objects; @@ -963,15 +963,14 @@ int index_fd(struct index_state *istate, struct object_id *oid, if (!inflight) odb_transaction_begin_or_die(odb, &transaction, 0); ret = odb_transaction_write_object_stream(transaction, - &stream, - oid); + stream, oid); if (!inflight) odb_transaction_commit(transaction); } else { - ret = hash_stream(&stream, the_repository->hash_algo, oid); + ret = hash_stream(stream, the_repository->hash_algo, oid); } - odb_write_stream_release(&stream); + odb_stream_close(stream); } close(fd); diff --git a/odb.c b/odb.c index 585b2b2965bb91..eec4cc53022e9d 100644 --- a/odb.c +++ b/odb.c @@ -1028,7 +1028,7 @@ int odb_write_object_ext(struct object_database *odb, } int odb_write_object_stream(struct object_database *odb, - struct odb_write_stream *stream, + struct odb_stream *stream, struct object_id *oid) { return odb_source_write_object_stream(odb->sources, stream, oid); diff --git a/odb.h b/odb.h index 019d3af3e8d212..fbe75c5a811a55 100644 --- a/odb.h +++ b/odb.h @@ -626,10 +626,10 @@ static inline int odb_write_object(struct object_database *odb, return odb_write_object_ext(odb, buf, len, type, oid, NULL, 0); } -struct odb_write_stream; +struct odb_stream; int odb_write_object_stream(struct object_database *odb, - struct odb_write_stream *stream, + struct odb_stream *stream, struct object_id *oid); void parse_alternates(const char *string, diff --git a/odb/source-files.c b/odb/source-files.c index f7b8c76549c393..6defe5ac4f94ca 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -174,7 +174,7 @@ static int odb_source_files_write_object(struct odb_source *source, } static int odb_source_files_write_object_stream(struct odb_source *source, - struct odb_write_stream *stream, + struct odb_stream *stream, struct object_id *oid) { struct odb_source_files *files = odb_source_files_downcast(source); diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index 485d58703658b0..795672adf255c6 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -256,7 +256,7 @@ static int odb_source_inmemory_write_object(struct odb_source *source, } static int odb_source_inmemory_write_object_stream(struct odb_source *source, - struct odb_write_stream *stream, + struct odb_stream *stream, struct object_id *oid) { char buf[16384]; @@ -268,7 +268,7 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source, while (1) { ssize_t bytes_read; - bytes_read = odb_write_stream_read(stream, buf, sizeof(buf)); + bytes_read = odb_stream_read(stream, buf, sizeof(buf)); if (bytes_read < 0) { ret = error("failed to read object stream"); goto out; diff --git a/odb/source-loose.c b/odb/source-loose.c index 038defd9059408..ff1bede7fef24a 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -845,7 +845,7 @@ static int odb_source_loose_write_object(struct odb_source *source, } static int odb_source_loose_write_object_stream(struct odb_source *source, - struct odb_write_stream *in_stream, + struct odb_stream *in_stream, struct object_id *oid) { struct odb_source_loose *loose = odb_source_loose_downcast(source); @@ -891,8 +891,8 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, unsigned char *in0 = stream.next_in; if (!stream.avail_in && !is_finished) { - ssize_t read_len = odb_write_stream_read(in_stream, buf, - sizeof(buf)); + ssize_t read_len = odb_stream_read(in_stream, buf, + sizeof(buf)); if (read_len < 0) { close(fd); err = -1; diff --git a/odb/source-packed.c b/odb/source-packed.c index b3186ca5933ae9..630d9555856d7c 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -609,7 +609,7 @@ static int odb_source_packed_write_object(struct odb_source *source UNUSED, } static int odb_source_packed_write_object_stream(struct odb_source *source UNUSED, - struct odb_write_stream *stream UNUSED, + struct odb_stream *stream UNUSED, struct object_id *oid UNUSED) { return error("packed backend cannot write object streams"); diff --git a/odb/source.h b/odb/source.h index 89b0c396822c95..0b99c698b5d625 100644 --- a/odb/source.h +++ b/odb/source.h @@ -221,7 +221,7 @@ struct odb_source { * otherwise. */ int (*write_object_stream)(struct odb_source *source, - struct odb_write_stream *stream, + struct odb_stream *stream, struct object_id *oid); /* @@ -436,7 +436,7 @@ static inline int odb_source_write_object(struct odb_source *source, * out pointer for the object ID. */ static inline int odb_source_write_object_stream(struct odb_source *source, - struct odb_write_stream *stream, + struct odb_stream *stream, struct object_id *oid) { return source->write_object_stream(source, stream, oid); diff --git a/odb/streaming.c b/odb/streaming.c index 98e2152e364741..1a267e6b90702a 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -232,16 +232,6 @@ struct odb_stream *odb_read_stream_open(struct object_database *odb, return st; } -ssize_t odb_write_stream_read(struct odb_write_stream *st, void *buf, size_t sz) -{ - return st->read(st, buf, sz); -} - -void odb_write_stream_release(struct odb_write_stream *st) -{ - free(st->data); -} - int odb_stream_blob_to_fd(struct object_database *odb, int fd, const struct object_id *oid, @@ -299,14 +289,15 @@ int odb_stream_blob_to_fd(struct object_database *odb, } struct read_object_fd_data { + struct odb_stream base; int fd; size_t remaining; }; -static ssize_t read_object_fd(struct odb_write_stream *stream, - unsigned char *buf, size_t len) +static ssize_t read_object_fd(struct odb_stream *stream, + char *buf, size_t len) { - struct read_object_fd_data *data = stream->data; + struct read_object_fd_data *data = container_of(stream, struct read_object_fd_data, base); ssize_t read_result; size_t count; @@ -323,17 +314,23 @@ static ssize_t read_object_fd(struct odb_write_stream *stream, return read_result; } -void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd, - size_t size, enum object_type type) +static int close_object_fd(struct odb_stream *stream UNUSED) +{ + /* The file descriptor is owned by the caller for now. */ + return 0; +} + +struct odb_stream *odb_write_stream_from_fd(int fd, size_t size, enum object_type type) { struct read_object_fd_data *data; CALLOC_ARRAY(data, 1); + data->base.read = read_object_fd; + data->base.close = close_object_fd; + data->base.size = size; + data->base.type = type; data->fd = fd; data->remaining = size; - stream->data = data; - stream->read = read_object_fd; - stream->size = size; - stream->type = type; + return &data->base; } diff --git a/odb/streaming.h b/odb/streaming.h index 037954c2315895..60b98031908f35 100644 --- a/odb/streaming.h +++ b/odb/streaming.h @@ -15,8 +15,8 @@ typedef int (*odb_stream_close_fn)(struct odb_stream *); typedef ssize_t (*odb_stream_read_fn)(struct odb_stream *, char *, size_t); /* - * A stream that can be used to read an object from the object database without - * loading all of it into memory. + * A stream that can be used to read an object from or write an object into the + * object database without loading all of it into memory. */ struct odb_stream { odb_stream_close_fn close; @@ -48,30 +48,6 @@ int odb_stream_close(struct odb_stream *stream); */ ssize_t odb_stream_read(struct odb_stream *stream, void *buf, size_t len); -/* - * A stream that provides an object to be written to the object database without - * loading all of it into memory. - */ -struct odb_write_stream { - ssize_t (*read)(struct odb_write_stream *, unsigned char *, size_t); - void *data; - size_t size; - enum object_type type; -}; - -/* - * Read data from the stream into the buffer. Returns 0 when finished and the - * number of bytes read on success. Returns a negative error code in case - * reading from the stream fails. - */ -ssize_t odb_write_stream_read(struct odb_write_stream *stream, void *buf, - size_t len); - -/* - * Releases memory allocated for underlying stream data. - */ -void odb_write_stream_release(struct odb_write_stream *stream); - /* * Look up the object by its ID and write the full contents to the file * descriptor. The object must be a blob, or the function will fail. When @@ -92,7 +68,6 @@ int odb_stream_blob_to_fd(struct object_database *odb, /* * Sets up an ODB write stream that reads from an fd. */ -void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd, - size_t size, enum object_type type); +struct odb_stream *odb_write_stream_from_fd(int fd, size_t size, enum object_type type); #endif /* STREAMING_H */ diff --git a/odb/transaction.c b/odb/transaction.c index 6aaf1338127534..69d71b9e97c61d 100644 --- a/odb/transaction.c +++ b/odb/transaction.c @@ -39,7 +39,7 @@ int odb_transaction_commit(struct odb_transaction *transaction) } int odb_transaction_write_object_stream(struct odb_transaction *transaction, - struct odb_write_stream *stream, + struct odb_stream *stream, struct object_id *oid) { return transaction->write_object_stream(transaction, stream, oid); diff --git a/odb/transaction.h b/odb/transaction.h index 1eb74664c6bb1d..65248a409c820d 100644 --- a/odb/transaction.h +++ b/odb/transaction.h @@ -31,7 +31,7 @@ struct odb_transaction { * otherwise. */ int (*write_object_stream)(struct odb_transaction *transaction, - struct odb_write_stream *stream, + struct odb_stream *stream, struct object_id *oid); /* @@ -81,7 +81,7 @@ int odb_transaction_commit(struct odb_transaction *transaction); * error code otherwise. */ int odb_transaction_write_object_stream(struct odb_transaction *transaction, - struct odb_write_stream *stream, + struct odb_stream *stream, struct object_id *oid); /* diff --git a/t/unit-tests/u-odb-inmemory.c b/t/unit-tests/u-odb-inmemory.c index 839a0fd3b753b2..b8b331b37d3beb 100644 --- a/t/unit-tests/u-odb-inmemory.c +++ b/t/unit-tests/u-odb-inmemory.c @@ -266,13 +266,13 @@ void test_odb_inmemory__freshen_object(void) } struct membuf_write_stream { - struct odb_write_stream base; + struct odb_stream base; const char *buf; size_t offset; }; -static ssize_t membuf_write_stream_read(struct odb_write_stream *stream, - unsigned char *buf, size_t len) +static ssize_t membuf_write_stream_read(struct odb_stream *stream, + char *buf, size_t len) { struct membuf_write_stream *s = container_of(stream, struct membuf_write_stream, base); size_t chunk_size = 2; From 93dd24603b76ccc26322e48ba6c3e1dfc95ef30e Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 5 Aug 2026 09:44:50 +0200 Subject: [PATCH 059/259] odb/streaming: rename `struct read_object_fd_data` With the preceding refactorings the `struct read_object_fd_data` is now somewhat misnamed, as it doesn't only contain the data anymore, but also the stream itself. Rename the structure to `struct fd_stream` to better match the new structure. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb/streaming.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/odb/streaming.c b/odb/streaming.c index 1a267e6b90702a..c436b18d39acdf 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -288,33 +288,33 @@ int odb_stream_blob_to_fd(struct object_database *odb, return result; } -struct read_object_fd_data { +struct fd_stream { struct odb_stream base; int fd; size_t remaining; }; -static ssize_t read_object_fd(struct odb_stream *stream, +static ssize_t fd_stream_read(struct odb_stream *stream, char *buf, size_t len) { - struct read_object_fd_data *data = container_of(stream, struct read_object_fd_data, base); + struct fd_stream *fds = container_of(stream, struct fd_stream, base); ssize_t read_result; size_t count; - if (!data->remaining) + if (!fds->remaining) return 0; - count = data->remaining < len ? data->remaining : len; - read_result = read_in_full(data->fd, buf, count); + count = fds->remaining < len ? fds->remaining : len; + read_result = read_in_full(fds->fd, buf, count); if (read_result < 0 || (size_t)read_result != count) return -1; - data->remaining -= count; + fds->remaining -= count; return read_result; } -static int close_object_fd(struct odb_stream *stream UNUSED) +static int fd_stream_close(struct odb_stream *stream UNUSED) { /* The file descriptor is owned by the caller for now. */ return 0; @@ -322,15 +322,15 @@ static int close_object_fd(struct odb_stream *stream UNUSED) struct odb_stream *odb_write_stream_from_fd(int fd, size_t size, enum object_type type) { - struct read_object_fd_data *data; + struct fd_stream *fds; - CALLOC_ARRAY(data, 1); - data->base.read = read_object_fd; - data->base.close = close_object_fd; - data->base.size = size; - data->base.type = type; - data->fd = fd; - data->remaining = size; + CALLOC_ARRAY(fds, 1); + fds->base.read = fd_stream_read; + fds->base.close = fd_stream_close; + fds->base.size = size; + fds->base.type = type; + fds->fd = fd; + fds->remaining = size; - return &data->base; + return &fds->base; } From 3f8290ea8552bb028c14e5be75315c2b90221802 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 5 Aug 2026 09:44:51 +0200 Subject: [PATCH 060/259] odb/streaming: rename `struct input_zstream_data` With the preceding refactorings the `struct input_zstream_data` is now somewhat misnamed, as it doesn't only contain the data anymore, but also the stream itself. Rename the structure to `struct zlib_stream` to better match the new structure. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/unpack-objects.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index 05a2d48011fc73..3392a3b87ddb0c 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -358,16 +358,16 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size, write_object(nr, type, buf, size); } -struct input_zstream_data { +struct zlib_stream { struct odb_stream base; git_zstream *zstream; int status; }; -static ssize_t feed_input_zstream(struct odb_stream *in_stream, - char *buf, size_t buf_len) +static ssize_t zlib_stream_read(struct odb_stream *in_stream, + char *buf, size_t buf_len) { - struct input_zstream_data *data = container_of(in_stream, struct input_zstream_data, base); + struct zlib_stream *data = container_of(in_stream, struct zlib_stream, base); git_zstream *zstream = data->zstream; if (data->status != Z_OK) @@ -389,9 +389,9 @@ static ssize_t feed_input_zstream(struct odb_stream *in_stream, static void stream_blob(unsigned long size, unsigned nr) { git_zstream zstream = { 0 }; - struct input_zstream_data in_stream = { + struct zlib_stream in_stream = { .base = { - .read = feed_input_zstream, + .read = zlib_stream_read, .size = size, .type = OBJ_BLOB, }, From ebdd7e10d6935c510154bcfff03b92cd7e972830 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 5 Aug 2026 09:44:52 +0200 Subject: [PATCH 061/259] odb/streaming: unify function names to create new streams Unify the function names to create new streams from different sources so that they follow a common schema. While at it, document the ownership of the file descriptor passed to `odb_stream_from_fd()`. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- archive-tar.c | 2 +- archive-zip.c | 2 +- builtin/index-pack.c | 2 +- builtin/pack-objects.c | 4 ++-- object-file.c | 4 ++-- object.c | 2 +- odb/streaming.c | 10 +++++----- odb/streaming.h | 23 +++++++++++++---------- 8 files changed, 26 insertions(+), 23 deletions(-) diff --git a/archive-tar.c b/archive-tar.c index df2d7fb8e936a8..a1c66024d4dff4 100644 --- a/archive-tar.c +++ b/archive-tar.c @@ -133,7 +133,7 @@ static int stream_blocked(struct repository *r, const struct object_id *oid) char buf[BLOCKSIZE]; ssize_t readlen; - st = odb_read_stream_open(r->objects, oid, NULL); + st = odb_stream_from_object(r->objects, oid, NULL); if (!st) return error(_("cannot stream blob %s"), oid_to_hex(oid)); for (;;) { diff --git a/archive-zip.c b/archive-zip.c index 8095fd04d5b9ba..1a948c2f83c919 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -347,7 +347,7 @@ static int write_zip_entry(struct archiver_args *args, method = ZIP_METHOD_DEFLATE; if (!buffer) { - stream = odb_read_stream_open(args->repo->objects, oid, NULL); + stream = odb_stream_from_object(args->repo->objects, oid, NULL); if (!stream) return error(_("cannot stream blob %s"), oid_to_hex(oid)); diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 7226da3e65ad8d..d1761282db8915 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -806,7 +806,7 @@ static int check_collison(struct object_entry *entry) memset(&data, 0, sizeof(data)); data.entry = entry; - data.st = odb_read_stream_open(the_repository->objects, &entry->idx.oid, NULL); + data.st = odb_stream_from_object(the_repository->objects, &entry->idx.oid, NULL); if (!data.st) return -1; if (data.st->size != entry->size || data.st->type != entry->type) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 683160c6bbb6ab..10d00ca7922260 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -528,8 +528,8 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent if (oe_type(entry) == OBJ_BLOB && oe_size_greater_than(&to_pack, entry, repo_settings_get_big_file_threshold(the_repository)) && - (st = odb_read_stream_open(the_repository->objects, &entry->idx.oid, - NULL)) != NULL) { + (st = odb_stream_from_object(the_repository->objects, &entry->idx.oid, + NULL)) != NULL) { buf = NULL; type = st->type; size = st->size; diff --git a/object-file.c b/object-file.c index 068c6e56726db9..11d1af342e8086 100644 --- a/object-file.c +++ b/object-file.c @@ -952,8 +952,8 @@ int index_fd(struct index_state *istate, struct object_id *oid, ret = index_core(istate, oid, fd, xsize_t(st->st_size), type, path, flags); } else { - struct odb_stream *stream = odb_write_stream_from_fd(fd, xsize_t(st->st_size), - OBJ_BLOB); + struct odb_stream *stream = odb_stream_from_fd(fd, xsize_t(st->st_size), + OBJ_BLOB); if (flags & INDEX_WRITE_OBJECT) { struct object_database *odb = the_repository->objects; diff --git a/object.c b/object.c index 37e6efee47ff2a..97f7fc0e87a1db 100644 --- a/object.c +++ b/object.c @@ -345,7 +345,7 @@ struct object *parse_object_with_flags(struct repository *r, if ((!obj || obj->type == OBJ_NONE || obj->type == OBJ_BLOB) && odb_read_object_info(r->objects, oid, NULL) == OBJ_BLOB) { if (!skip_hash) { - struct odb_stream *stream = odb_read_stream_open(r->objects, oid, NULL); + struct odb_stream *stream = odb_stream_from_object(r->objects, oid, NULL); if (!stream) { error(_("unable to open object stream for %s"), oid_to_hex(oid)); diff --git a/odb/streaming.c b/odb/streaming.c index c436b18d39acdf..9c85ec54f59bb1 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -208,9 +208,9 @@ ssize_t odb_stream_read(struct odb_stream *st, void *buf, size_t sz) return st->read(st, buf, sz); } -struct odb_stream *odb_read_stream_open(struct object_database *odb, - const struct object_id *oid, - struct stream_filter *filter) +struct odb_stream *odb_stream_from_object(struct object_database *odb, + const struct object_id *oid, + struct stream_filter *filter) { struct odb_stream *st; const struct object_id *real = lookup_replace_object(odb->repo, oid); @@ -242,7 +242,7 @@ int odb_stream_blob_to_fd(struct object_database *odb, ssize_t kept = 0; int result = -1; - st = odb_read_stream_open(odb, oid, filter); + st = odb_stream_from_object(odb, oid, filter); if (!st) { if (filter) free_stream_filter(filter); @@ -320,7 +320,7 @@ static int fd_stream_close(struct odb_stream *stream UNUSED) return 0; } -struct odb_stream *odb_write_stream_from_fd(int fd, size_t size, enum object_type type) +struct odb_stream *odb_stream_from_fd(int fd, size_t size, enum object_type type) { struct fd_stream *fds; diff --git a/odb/streaming.h b/odb/streaming.h index 60b98031908f35..b522ff513f26d2 100644 --- a/odb/streaming.h +++ b/odb/streaming.h @@ -26,14 +26,22 @@ struct odb_stream { }; /* - * Create a new object stream for the given object database. An optional filter - * can be used to transform the object's content. + * Create a new object stream for the given object. An optional filter can be + * used to transform the object's content. * * Returns the stream on success, a `NULL` pointer otherwise. */ -struct odb_stream *odb_read_stream_open(struct object_database *odb, - const struct object_id *oid, - struct stream_filter *filter); +struct odb_stream *odb_stream_from_object(struct object_database *odb, + const struct object_id *oid, + struct stream_filter *filter); + +/* + * Create a new object stream for the given file descriptor. This can be used + * to, for example, stream an object into the object database. This function + * does _not_ take ownership of the file descriptor. It's the responsibility of + * the caller to close it after the stream has been closed. + */ +struct odb_stream *odb_stream_from_fd(int fd, size_t size, enum object_type type); /* * Close the given object stream and release all resources associated with it. @@ -65,9 +73,4 @@ int odb_stream_blob_to_fd(struct object_database *odb, struct stream_filter *filter, int can_seek); -/* - * Sets up an ODB write stream that reads from an fd. - */ -struct odb_stream *odb_write_stream_from_fd(int fd, size_t size, enum object_type type); - #endif /* STREAMING_H */ From 876029beca8495a44e3b6c335b7fac5208e8da86 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Wed, 5 Aug 2026 14:24:31 +0000 Subject: [PATCH 062/259] branch: add --forked filter for --list mode Add a --forked option to "git branch" list mode that lists only branches whose configured upstream matches . The argument can be a ref (e.g. "origin/main", "master"), a remote name like "origin" for the branch its origin/HEAD points at, or a shell glob (e.g. "origin/*"), and may be repeated to widen the filter. It is an ordinary list filter, so it combines with the others: git branch --merged origin/main --forked 'origin/*' lists branches forked from origin that are already merged into origin/main, and --no-merged inverts the question. This is the building block for --delete-merged, which deletes the listed branches once they have landed on their upstream. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- Documentation/git-branch.adoc | 12 +++- builtin/branch.c | 18 ++++- ref-filter.c | 70 +++++++++++++++++++ ref-filter.h | 10 +++ t/t3200-branch.sh | 127 ++++++++++++++++++++++++++++++++++ 5 files changed, 234 insertions(+), 3 deletions(-) diff --git a/Documentation/git-branch.adoc b/Documentation/git-branch.adoc index c0afddc424d610..b0d66a6deb8b8c 100644 --- a/Documentation/git-branch.adoc +++ b/Documentation/git-branch.adoc @@ -13,6 +13,7 @@ git branch [--color[=] | --no-color] [--show-current] [--column[=] | --no-column] [--sort=] [--merged []] [--no-merged []] [--contains []] [--no-contains []] + [(--forked )...] [--points-at ] [--format=] [(-r|--remotes) | (-a|--all)] [--list] [...] @@ -51,7 +52,8 @@ merged into the named commit (i.e. the branches whose tip commits are reachable from the named commit) will be listed. With `--no-merged` only branches not merged into the named commit will be listed. If the __ argument is missing it defaults to `HEAD` (i.e. the tip of the current -branch). +branch). With `--forked`, only branches whose configured upstream matches +the given branch or pattern will be listed. The command's second form creates a new branch head named __ which points to the current `HEAD`, or __ if given. As a @@ -311,6 +313,14 @@ superproject's "origin/main", but tracks the submodule's "origin/main". Only list branches whose tips are not reachable from __ (`HEAD` if not specified). Implies `--list`. +`--forked `:: + Only list branches whose configured upstream matches + __. The argument can be a ref (e.g. `origin/main`, + `master`), a remote name like `origin` for the branch its + `origin/HEAD` points at, or a shell-style glob (e.g. + `'origin/*'`). The option can be repeated to widen the + filter. Implies `--list`. + `--points-at `:: Only list branches of __. diff --git a/builtin/branch.c b/builtin/branch.c index 031a4a9d055558..1ab4356188fc1e 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -30,7 +30,7 @@ #include "commit-reach.h" static const char * const builtin_branch_usage[] = { - N_("git branch [] [-r | -a] [--merged] [--no-merged]"), + N_("git branch [] [-r | -a] [--merged] [--no-merged] [(--forked )...]"), N_("git branch [] [-f] [--recurse-submodules] []"), N_("git branch [] [-l] [...]"), N_("git branch [] [-r] (-d | -D) ..."), @@ -674,6 +674,16 @@ static void copy_or_rename_branch(const char *oldname, const char *newname, int free_worktrees(worktrees); } +static int parse_opt_forked(const struct option *opt, const char *arg, int unset) +{ + struct ref_filter *filter = opt->value; + + BUG_ON_OPT_NEG(unset); + if (ref_filter_forked_add(filter, arg) < 0) + die(_("'%s' is not a valid branch or pattern"), arg); + return 0; +} + static GIT_PATH_FUNC(edit_description, "EDIT_DESCRIPTION") static int edit_branch_description(const char *branch_name) @@ -794,6 +804,9 @@ int cmd_branch(int argc, OPT__FORCE(&force, N_("force creation, move/rename, deletion"), PARSE_OPT_NOCOMPLETE), OPT_MERGED(&filter, N_("print only branches that are merged")), OPT_NO_MERGED(&filter, N_("print only branches that are not merged")), + OPT_CALLBACK_F(0, "forked", &filter, N_("branch"), + N_("print only branches whose upstream matches (repeatable)"), + PARSE_OPT_NONEG, parse_opt_forked), OPT_COLUMN(0, "column", &colopts, N_("list branches in columns")), OPT_REF_SORT(&sorting_options), OPT_CALLBACK(0, "points-at", &filter.points_at, N_("object"), @@ -839,7 +852,8 @@ int cmd_branch(int argc, list = 1; if (filter.with_commit || filter.no_commit || - filter.reachable_from || filter.unreachable_from || filter.points_at.nr) + filter.reachable_from || filter.unreachable_from || + filter.points_at.nr || filter.forked.nr) list = 1; noncreate_actions = !!delete + !!rename + !!copy + !!new_upstream + diff --git a/ref-filter.c b/ref-filter.c index 29aca08ce7b333..bdf54f6f592499 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -2744,6 +2744,72 @@ static int filter_exclude_match(struct ref_filter *filter, const char *refname) return match_pattern(filter->exclude.v, refname, filter->ignore_case); } +static const char *short_upstream_name(const char *full_ref) +{ + const char *short_name = full_ref; + + if (!skip_prefix(short_name, "refs/heads/", &short_name)) + skip_prefix(short_name, "refs/remotes/", &short_name); + return short_name; +} + +/* + * Match the configured upstream of a branch against the registered + * --forked patterns. Exact patterns are compared against the full + * upstream refname so they are unambiguous; glob patterns are matched + * against the abbreviated upstream so that a glob such as origin/... + * works as typed. + */ +static int filter_forked_match(struct ref_filter *filter, const char *refname) +{ + const char *short_name; + struct branch *branch; + const char *upstream; + + if (!skip_prefix(refname, "refs/heads/", &short_name)) + return 0; + branch = branch_get(short_name); + if (!branch) + return 0; + upstream = branch_get_upstream(branch, NULL); + if (!upstream) + return 0; + + for (size_t i = 0; i < filter->forked.nr; i++) { + const char *pattern = filter->forked.v[i]; + if (has_glob_specials(pattern)) { + if (!wildmatch(pattern, short_upstream_name(upstream), + WM_PATHNAME)) + return 1; + } else if (!strcmp(pattern, upstream)) { + return 1; + } + } + return 0; +} + +int ref_filter_forked_add(struct ref_filter *filter, const char *arg) +{ + struct object_id oid; + char *full_ref = NULL; + + if (has_glob_specials(arg)) { + strvec_push(&filter->forked, arg); + return 0; + } + + if (repo_dwim_ref(the_repository, arg, strlen(arg), &oid, + &full_ref, 0) == 1 && + (starts_with(full_ref, "refs/heads/") || + starts_with(full_ref, "refs/remotes/"))) { + strvec_push(&filter->forked, full_ref); + free(full_ref); + return 0; + } + free(full_ref); + return -1; +} + /* * We need to seek to the reference right after a given marker but excluding any * matching references. So we seek to the lexicographically next reference. @@ -2979,6 +3045,9 @@ static struct ref_array_item *apply_ref_filter(const struct reference *ref, if (filter->points_at.nr && !match_points_at(&filter->points_at, ref->oid, ref->name)) return NULL; + if (filter->forked.nr && !filter_forked_match(filter, ref->name)) + return NULL; + /* * A merge filter is applied on refs pointing to commits. Hence * obtain the commit using the 'oid' available and discard all @@ -3764,6 +3833,7 @@ void ref_filter_init(struct ref_filter *filter) void ref_filter_clear(struct ref_filter *filter) { strvec_clear(&filter->exclude); + strvec_clear(&filter->forked); oid_array_clear(&filter->points_at); commit_list_free(filter->with_commit); commit_list_free(filter->no_commit); diff --git a/ref-filter.h b/ref-filter.h index 120221b47fa30d..9361296e2a7440 100644 --- a/ref-filter.h +++ b/ref-filter.h @@ -67,6 +67,7 @@ struct ref_filter { const char **name_patterns; const char *start_after; struct strvec exclude; + struct strvec forked; struct oid_array points_at; struct commit_list *with_commit; struct commit_list *no_commit; @@ -110,6 +111,7 @@ struct ref_format { #define REF_FILTER_INIT { \ .points_at = OID_ARRAY_INIT, \ .exclude = STRVEC_INIT, \ + .forked = STRVEC_INIT, \ } #define REF_FORMAT_INIT { \ .use_color = GIT_COLOR_UNKNOWN, \ @@ -172,6 +174,14 @@ void ref_sorting_release(struct ref_sorting *); struct ref_sorting *ref_sorting_options(struct string_list *); /* Function to parse --merged and --no-merged options */ int parse_opt_merge_filter(const struct option *opt, const char *arg, int unset); +/* + * Register a --forked pattern on the filter. The argument is + * either a ref, which is resolved to its full refname, or a shell-style + * glob. Branches are kept only when their configured upstream matches + * one of the registered patterns. Returns -1 if the argument is not a + * valid ref or pattern. + */ +int ref_filter_forked_add(struct ref_filter *filter, const char *arg); /* Get the current HEAD's description */ char *get_head_description(void); /* Set up translated strings in the output. */ diff --git a/t/t3200-branch.sh b/t/t3200-branch.sh index 1ecbafbee18e03..84940951651e67 100755 --- a/t/t3200-branch.sh +++ b/t/t3200-branch.sh @@ -1755,4 +1755,131 @@ test_expect_success 'errors if given a bad branch name' ' test_cmp expect actual ' +test_expect_success '--forked: setup' ' + test_create_repo forked-upstream && + ( + cd forked-upstream && + test_commit base && + git branch one base && + git branch two base + ) && + + test_create_repo forked-other && + ( + cd forked-other && + test_commit other-base && + git branch foreign other-base + ) && + + git clone forked-upstream forked && + ( + cd forked && + git remote add -f other ../forked-other && + git branch local-base && + git branch --track local-one origin/one && + git branch --track local-two origin/two && + git branch --track local-foreign other/foreign && + git branch --track local-onbase local-base && + + git checkout local-one && + test_commit --no-tag local-one-work local-one.t && + git checkout local-foreign && + test_commit --no-tag local-foreign-work local-foreign.t + ) +' + +test_expect_success '--forked filters by upstream' ' + git -C forked branch --forked origin/one \ + --format="%(refname:short)" >actual && + echo local-one >expect && + test_cmp expect actual +' + +test_expect_success '--forked filters by wildmatch' ' + git -C forked branch --forked "origin/*" \ + --format="%(refname:short)" >actual && + cat >expect <<-\EOF && + local-one + local-two + main + EOF + test_cmp expect actual +' + +test_expect_success '--forked matches branches with local upstream' ' + git -C forked branch --forked local-base \ + --format="%(refname:short)" >actual && + echo local-onbase >expect && + test_cmp expect actual +' + +test_expect_success '--forked can be repeated to widen the filter' ' + git -C forked branch --forked origin/one \ + --forked other/foreign \ + --format="%(refname:short)" >actual && + cat >expect <<-\EOF && + local-foreign + local-one + EOF + test_cmp expect actual +' + +test_expect_success '--forked combines literal and glob arguments' ' + git -C forked branch --forked local-base \ + --forked "other/*" \ + --format="%(refname:short)" >actual && + cat >expect <<-\EOF && + local-foreign + local-onbase + EOF + test_cmp expect actual +' + +test_expect_success '--forked "*/*" covers every remote-tracking upstream' ' + git -C forked branch --forked "*/*" \ + --format="%(refname:short)" >actual && + cat >expect <<-\EOF && + local-foreign + local-one + local-two + main + EOF + test_cmp expect actual +' + +test_expect_success '--forked composes with --no-merged' ' + git -C forked branch --forked "origin/*" \ + --no-merged origin/one \ + --format="%(refname:short)" >actual && + echo local-one >expect && + test_cmp expect actual +' + +test_expect_success '--forked uses the branch /HEAD points at' ' + git -C forked branch --forked origin \ + --format="%(refname:short)" >actual && + echo main >expect && + test_cmp expect actual +' + +test_expect_success '--forked narrows a argument' ' + git -C forked branch --forked "origin/*" "local-*" \ + --format="%(refname:short)" >actual && + cat >expect <<-\EOF && + local-one + local-two + EOF + test_cmp expect actual +' + +test_expect_success '--forked rejects unknown branch/pattern' ' + test_must_fail git -C forked branch --forked nope 2>err && + test_grep "not a valid branch or pattern" err +' + +test_expect_success '--forked requires a value' ' + test_must_fail git -C forked branch --forked 2>err && + test_grep "requires a value" err +' + test_done From 7969faaf9b4e6e6cfff85ba9a0c592971264596b Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Wed, 5 Aug 2026 14:24:32 +0000 Subject: [PATCH 063/259] branch: convert delete_branches() to a flags argument delete_branches() takes separate force and quiet parameters, while check_branch_commit() takes force. The next commits would grow this collection further. Replace them with a single unsigned flags argument and an enum. Test the FORCE and QUIET bits directly from flags at each use site so that mutating or forwarding flags cannot leave cached values stale. No change in behavior. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- builtin/branch.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/builtin/branch.c b/builtin/branch.c index 1ab4356188fc1e..db7cb011901a42 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -189,16 +189,22 @@ static int branch_merged(int kind, const char *name, return merged; } +enum delete_branch_flags { + DELETE_BRANCH_FORCE = (1 << 0), + DELETE_BRANCH_QUIET = (1 << 1), +}; + static int check_branch_commit(const char *branchname, const char *refname, const struct object_id *oid, struct commit *head_rev, - int kinds, int force) + int kinds, unsigned int flags) { struct commit *rev = lookup_commit_reference(the_repository, oid); - if (!force && !rev) { + if (!(flags & DELETE_BRANCH_FORCE) && !rev) { error(_("couldn't look up commit object for '%s'"), refname); return -1; } - if (!force && !branch_merged(kinds, branchname, rev, head_rev)) { + if (!(flags & DELETE_BRANCH_FORCE) && + !branch_merged(kinds, branchname, rev, head_rev)) { error(_("the branch '%s' is not fully merged"), branchname); advise_if_enabled(ADVICE_FORCE_DELETE_BRANCH, _("If you are sure you want to delete it, " @@ -217,8 +223,8 @@ static void delete_branch_config(const char *branchname) strbuf_release(&buf); } -static int delete_branches(int argc, const char **argv, int force, int kinds, - int quiet) +static int delete_branches(int argc, const char **argv, int kinds, + unsigned int flags) { struct commit *head_rev = NULL; struct object_id oid; @@ -241,7 +247,7 @@ static int delete_branches(int argc, const char **argv, int force, int kinds, remote_branch = 1; allowed_interpret = INTERPRET_BRANCH_REMOTE; - force = 1; + flags |= DELETE_BRANCH_FORCE; break; case FILTER_REFS_BRANCHES: fmt = "refs/heads/%s"; @@ -252,12 +258,12 @@ static int delete_branches(int argc, const char **argv, int force, int kinds, } branch_name_pos = strcspn(fmt, "%"); - if (!force) + if (!(flags & DELETE_BRANCH_FORCE)) head_rev = lookup_commit_reference(the_repository, &head_oid); for (i = 0; i < argc; i++, strbuf_reset(&bname)) { char *target = NULL; - int flags = 0; + int ref_flags = 0; copy_branchname(the_repository, &bname, argv[i], allowed_interpret); @@ -280,7 +286,7 @@ static int delete_branches(int argc, const char **argv, int force, int kinds, RESOLVE_REF_READING | RESOLVE_REF_NO_RECURSE | RESOLVE_REF_ALLOW_BAD_NAME, - &oid, &flags); + &oid, &ref_flags); if (!target) { if (remote_branch) { error(_("remote-tracking branch '%s' not found"), bname.buf); @@ -292,7 +298,7 @@ static int delete_branches(int argc, const char **argv, int force, int kinds, | RESOLVE_REF_NO_RECURSE | RESOLVE_REF_ALLOW_BAD_NAME, &oid, - &flags); + &ref_flags); FREE_AND_NULL(virtual_name); if (virtual_target) @@ -307,16 +313,16 @@ static int delete_branches(int argc, const char **argv, int force, int kinds, continue; } - if (!(flags & (REF_ISSYMREF|REF_ISBROKEN)) && + if (!(ref_flags & (REF_ISSYMREF|REF_ISBROKEN)) && check_branch_commit(bname.buf, name, &oid, head_rev, kinds, - force)) { + flags)) { ret = 1; goto next; } item = string_list_append(&refs_to_delete, name); - item->util = xstrdup((flags & REF_ISBROKEN) ? "broken" - : (flags & REF_ISSYMREF) ? target + item->util = xstrdup((ref_flags & REF_ISBROKEN) ? "broken" + : (ref_flags & REF_ISSYMREF) ? target : repo_find_unique_abbrev(the_repository, &oid, DEFAULT_ABBREV)); next: @@ -331,7 +337,7 @@ static int delete_branches(int argc, const char **argv, int force, int kinds, char *name = item->string; if (!refs_ref_exists(get_main_ref_store(the_repository), name)) { char *refname = name + branch_name_pos; - if (!quiet) + if (!(flags & DELETE_BRANCH_QUIET)) printf(remote_branch ? _("Deleted remote-tracking branch %s (was %s).\n") : _("Deleted branch %s (was %s).\n"), @@ -896,7 +902,9 @@ int cmd_branch(int argc, if (delete) { if (!argc) die(_("branch name required")); - ret = delete_branches(argc, argv, delete > 1, filter.kind, quiet); + ret = delete_branches(argc, argv, filter.kind, + (delete > 1 ? DELETE_BRANCH_FORCE : 0) | + (quiet ? DELETE_BRANCH_QUIET : 0)); goto out; } else if (show_current) { print_current_branch_name(); From cdbcde91be1a4110a6d75b3f24fa2767c06187f6 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Wed, 5 Aug 2026 14:24:33 +0000 Subject: [PATCH 064/259] branch: let delete_branches skip unmerged branches on bulk refusal Add a skip-unmerged mode to delete_branches() and check_branch_commit() so a bulk caller can silently skip branches that are not fully merged and carry on, rather than erroring with the "use 'git branch -D'" advice that the plain "git branch -d" path emits. Existing callers are unaffected. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- builtin/branch.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/builtin/branch.c b/builtin/branch.c index db7cb011901a42..c44f710a48eb98 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -192,6 +192,7 @@ static int branch_merged(int kind, const char *name, enum delete_branch_flags { DELETE_BRANCH_FORCE = (1 << 0), DELETE_BRANCH_QUIET = (1 << 1), + DELETE_BRANCH_SKIP_UNMERGED = (1 << 2), }; static int check_branch_commit(const char *branchname, const char *refname, @@ -205,10 +206,13 @@ static int check_branch_commit(const char *branchname, const char *refname, } if (!(flags & DELETE_BRANCH_FORCE) && !branch_merged(kinds, branchname, rev, head_rev)) { - error(_("the branch '%s' is not fully merged"), branchname); - advise_if_enabled(ADVICE_FORCE_DELETE_BRANCH, - _("If you are sure you want to delete it, " - "run 'git branch -D %s'"), branchname); + if (!(flags & DELETE_BRANCH_SKIP_UNMERGED)) { + error(_("the branch '%s' is not fully merged"), + branchname); + advise_if_enabled(ADVICE_FORCE_DELETE_BRANCH, + _("If you are sure you want to delete it, " + "run 'git branch -D %s'"), branchname); + } return -1; } return 0; @@ -316,7 +320,8 @@ static int delete_branches(int argc, const char **argv, int kinds, if (!(ref_flags & (REF_ISSYMREF|REF_ISBROKEN)) && check_branch_commit(bname.buf, name, &oid, head_rev, kinds, flags)) { - ret = 1; + if (!(flags & DELETE_BRANCH_SKIP_UNMERGED)) + ret = 1; goto next; } From 5e528a36d243f6152dbc5e04ab2314b32f30efcb Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Wed, 5 Aug 2026 14:24:34 +0000 Subject: [PATCH 065/259] branch: prepare delete_branches for a bulk caller Teach delete_branches() a new mode for the upcoming --delete-merged caller that checks whether a branch is merged into its upstream without falling back to HEAD when there is no upstream. Existing callers keep their current behavior. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- builtin/branch.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/builtin/branch.c b/builtin/branch.c index c44f710a48eb98..7b0aa685728ea5 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -168,10 +168,13 @@ static int branch_merged(int kind, const char *name, * upstream, if any, otherwise with HEAD", we should just * return the result of the repo_in_merge_bases() above without * any of the following code, but during the transition period, - * a gentle reminder is in order. + * a gentle reminder is in order. Callers that opt out of the + * HEAD fallback by passing head_rev=NULL are not interested in + * the reminder either: they have already established that the + * branch has an upstream, so HEAD is irrelevant to the decision. */ - if (head_rev != reference_rev) { - int expect = head_rev ? repo_in_merge_bases(the_repository, rev, head_rev) : 0; + if (head_rev && head_rev != reference_rev) { + int expect = repo_in_merge_bases(the_repository, rev, head_rev); if (expect < 0) exit(128); if (expect == merged) @@ -193,6 +196,7 @@ enum delete_branch_flags { DELETE_BRANCH_FORCE = (1 << 0), DELETE_BRANCH_QUIET = (1 << 1), DELETE_BRANCH_SKIP_UNMERGED = (1 << 2), + DELETE_BRANCH_NO_HEAD_FALLBACK = (1 << 3), }; static int check_branch_commit(const char *branchname, const char *refname, @@ -262,7 +266,8 @@ static int delete_branches(int argc, const char **argv, int kinds, } branch_name_pos = strcspn(fmt, "%"); - if (!(flags & DELETE_BRANCH_FORCE)) + if (!(flags & DELETE_BRANCH_FORCE) && + !(flags & DELETE_BRANCH_NO_HEAD_FALLBACK)) head_rev = lookup_commit_reference(the_repository, &head_oid); for (i = 0; i < argc; i++, strbuf_reset(&bname)) { From 15bcdf43bbe2206998f7c044bd44479d47749809 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Wed, 5 Aug 2026 14:24:35 +0000 Subject: [PATCH 066/259] branch: add --delete-merged git branch (--delete-merged )... [...] deletes local branches matching the optional branch patterns when their configured upstream matches one of the --delete-merged arguments and their tip is reachable from that upstream. The work has already landed on the upstream they track, so the local copy is no longer needed. Each may name a ref, a remote, or a shell glob. The option can be repeated to widen the upstream match. Keeping the candidate patterns as positional arguments lets users bound the set of local branches that may be deleted independently of the upstream selection. A branch is not deleted when: * it is checked out in any worktree * its configured upstream ref no longer exists, since a missing upstream is not by itself a sign of integration * pushing it to the remote configured by branch..remote would update its upstream, as determined by that remote's configured push and fetch refspecs. For example, a local "main" that tracks "origin/main" is kept even when remote.pushDefault names a fork. Right after a pull it merely looks fully merged. * it is the local upstream of a branch that is not being deleted, so no branch is deleted out from under stacked work. A branch whose work is not yet merged into its upstream is silently skipped, so one unmerged topic does not abort the whole sweep. Collect protected local upstreams without changing the candidate set during ref iteration, then remove them after iteration. This makes the result independent of ref iteration order. If a protected branch's own upstream is deleted by the same sweep, clear its upstream configuration. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- Documentation/git-branch.adoc | 33 +++++ builtin/branch.c | 195 +++++++++++++++++++++++++++++- t/t3200-branch.sh | 219 ++++++++++++++++++++++++++++++++++ 3 files changed, 445 insertions(+), 2 deletions(-) diff --git a/Documentation/git-branch.adoc b/Documentation/git-branch.adoc index b0d66a6deb8b8c..47661782045e99 100644 --- a/Documentation/git-branch.adoc +++ b/Documentation/git-branch.adoc @@ -25,6 +25,7 @@ git branch (-m|-M) [] git branch (-c|-C) [] git branch (-d|-D) [-r] ... git branch --edit-description [] +git branch (--delete-merged )... [...] DESCRIPTION ----------- @@ -201,6 +202,38 @@ This option is only applicable in non-verbose mode. Print the name of the current branch. In detached `HEAD` state, nothing is printed. +`--delete-merged `:: + Delete local branches whose configured upstream matches + __, but only when their tip is reachable from that + upstream. In other words, the work on the branch has already + landed on the upstream it tracks, so the local copy is no longer + needed. __ may name a ref, a remote (using the branch its + `HEAD` points at), or a shell-style glob. The option can be + repeated to widen the upstream match. + Optional __ arguments limit which local branches + are considered, e.g. `git branch --delete-merged 'origin/*' + 'topic-*'`. ++ +A branch is not deleted when: ++ +-- +* its configured upstream ref no longer exists, +* it is checked out in any worktree, +* pushing it to the remote configured by + `branch..remote` would update its upstream, so it cannot be + distinguished from a branch that just looks fully merged right + after a pull; this is determined by the remote's configured push and + fetch refspecs, +* it is the local upstream of a branch that is not being deleted. +-- ++ +When such a local upstream branch has its own upstream deleted by the +same operation, its upstream configuration is cleared. ++ +A branch whose work has not yet been merged into its upstream is +silently skipped. Delete it with `git branch -D` if you want to +remove it anyway. + `-v`:: `-vv`:: `--verbose`:: diff --git a/builtin/branch.c b/builtin/branch.c index 7b0aa685728ea5..f1a73bcea1cee1 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -21,6 +21,7 @@ #include "branch.h" #include "path.h" #include "string-list.h" +#include "strmap.h" #include "column.h" #include "utf8.h" #include "ref-filter.h" @@ -38,6 +39,8 @@ static const char * const builtin_branch_usage[] = { N_("git branch [] (-c | -C) [] "), N_("git branch [] [-r | -a] [--points-at]"), N_("git branch [] [-r | -a] [--format]"), + N_("git branch [] (--delete-merged )... " + "[...]"), NULL }; @@ -700,6 +703,184 @@ static int parse_opt_forked(const struct option *opt, const char *arg, int unset return 0; } +struct stacked_branch_data { + struct strset *deletable_branch_names; + struct strset *protected_branch_names; +}; + +static int collect_stacked_branch_base(const struct reference *ref, + void *cb_data) +{ + struct stacked_branch_data *data = cb_data; + const char *branch_name; + struct branch *branch; + const char *upstream_refname; + const char *upstream_branch_name; + + if (!skip_prefix(ref->name, "refs/heads/", &branch_name)) + BUG("expected local branch ref, got '%s'", ref->name); + if (strset_contains(data->deletable_branch_names, branch_name)) + return 0; + + branch = branch_get(branch_name); + upstream_refname = branch_get_upstream(branch, NULL); + if (!upstream_refname || + !skip_prefix(upstream_refname, "refs/heads/", + &upstream_branch_name) || + !strset_contains(data->deletable_branch_names, + upstream_branch_name)) + return 0; + + strset_add(data->protected_branch_names, upstream_branch_name); + return 0; +} + +static void protect_stacked_branch_bases(struct ref_store *refs, + struct strset *deletable_branch_names, + struct strset *protected_branch_names) +{ + struct stacked_branch_data data = { + .deletable_branch_names = deletable_branch_names, + .protected_branch_names = protected_branch_names, + }; + struct refs_for_each_ref_options opts = { + .prefix = "refs/heads/", + }; + struct hashmap_iter iter; + struct strmap_entry *entry; + + refs_for_each_ref_ext(refs, collect_stacked_branch_base, &data, &opts); + + strset_for_each_entry(protected_branch_names, &iter, entry) + strset_remove(deletable_branch_names, entry->key); +} + +static void clear_deleted_upstreams(struct strset *protected_branch_names, + struct strset *deletable_branch_names) +{ + struct strbuf key = STRBUF_INIT; + struct hashmap_iter iter; + struct strmap_entry *entry; + + strset_for_each_entry(protected_branch_names, &iter, entry) { + struct branch *branch = branch_get(entry->key); + const char *upstream_refname = branch_get_upstream(branch, NULL); + const char *upstream_branch_name; + + if (!upstream_refname || + !skip_prefix(upstream_refname, "refs/heads/", + &upstream_branch_name) || + !strset_contains(deletable_branch_names, + upstream_branch_name)) + continue; + + strbuf_addf(&key, "branch.%s.merge", branch->name); + repo_config_set_gently(the_repository, key.buf, NULL); + strbuf_reset(&key); + strbuf_addf(&key, "branch.%s.remote", branch->name); + repo_config_set_gently(the_repository, key.buf, NULL); + strbuf_reset(&key); + } + + strbuf_release(&key); +} + +static int branch_pushes_to_upstream(struct branch *branch, + const char *upstream) +{ + struct remote *remote = remote_get(remote_for_branch(branch, NULL)); + char *push_refname = NULL; + char *tracking = NULL; + int ret = 0; + + if (!remote) + return 0; + if (remote->push.nr) + push_refname = apply_refspecs(&remote->push, branch->refname); + else + push_refname = xstrdup(branch->refname); + if (push_refname) + tracking = apply_refspecs(&remote->fetch, push_refname); + if (tracking && !strcmp(tracking, upstream)) + ret = 1; + + free(push_refname); + free(tracking); + return ret; +} + +static int delete_merged_branches(const struct strvec *upstreams, + const char **argv, unsigned int flags) +{ + struct ref_store *refs = get_main_ref_store(the_repository); + struct ref_filter filter = REF_FILTER_INIT; + struct ref_array candidates = { 0 }; + struct strset deletable_branch_names = STRSET_INIT; + struct strset protected_branch_names = STRSET_INIT; + struct strvec branches_to_delete = STRVEC_INIT; + struct hashmap_iter iter; + struct strmap_entry *entry; + int ret = 0; + + for (size_t i = 0; i < upstreams->nr; i++) + if (ref_filter_forked_add(&filter, upstreams->v[i]) < 0) + die(_("'%s' is not a valid branch or pattern"), + upstreams->v[i]); + + filter.kind = FILTER_REFS_BRANCHES; + filter.name_patterns = argv; + filter_refs(&candidates, &filter, filter.kind); + + for (int i = 0; i < candidates.nr; i++) { + const char *branch_refname = candidates.items[i]->refname; + const char *branch_name; + struct branch *branch; + const char *upstream_refname; + + if (!skip_prefix(branch_refname, "refs/heads/", &branch_name)) + BUG("filter returned non-branch ref '%s'", branch_refname); + if (branch_checked_out(branch_refname)) + continue; + + branch = branch_get(branch_name); + upstream_refname = branch_get_upstream(branch, NULL); + if (!upstream_refname || !refs_ref_exists(refs, upstream_refname)) + continue; + if (branch_pushes_to_upstream(branch, upstream_refname)) + continue; + if (check_branch_commit(branch_name, branch_name, + &candidates.items[i]->objectname, NULL, + FILTER_REFS_BRANCHES, DELETE_BRANCH_SKIP_UNMERGED)) + continue; + + strset_add(&deletable_branch_names, branch_name); + } + + protect_stacked_branch_bases(refs, &deletable_branch_names, + &protected_branch_names); + + strset_for_each_entry(&deletable_branch_names, &iter, entry) + strvec_push(&branches_to_delete, entry->key); + + if (branches_to_delete.nr) + ret = delete_branches(branches_to_delete.nr, branches_to_delete.v, + FILTER_REFS_BRANCHES, + DELETE_BRANCH_SKIP_UNMERGED | + DELETE_BRANCH_NO_HEAD_FALLBACK | + flags); + + if (!ret) + clear_deleted_upstreams(&protected_branch_names, + &deletable_branch_names); + + strvec_clear(&branches_to_delete); + strset_clear(&protected_branch_names); + strset_clear(&deletable_branch_names); + ref_array_clear(&candidates); + ref_filter_clear(&filter); + return ret; +} + static GIT_PATH_FUNC(edit_description, "EDIT_DESCRIPTION") static int edit_branch_description(const char *branch_name) @@ -764,6 +945,7 @@ int cmd_branch(int argc, /* possible actions */ int delete = 0, rename = 0, copy = 0, list = 0, unset_upstream = 0, show_current = 0, edit_description = 0; + struct strvec delete_merged = STRVEC_INIT; const char *new_upstream = NULL; int noncreate_actions = 0; /* possible options */ @@ -817,6 +999,9 @@ int cmd_branch(int argc, OPT_BOOL(0, "create-reflog", &reflog, N_("create the branch's reflog")), OPT_BOOL(0, "edit-description", &edit_description, N_("edit the description for the branch")), + OPT_CALLBACK_F(0, "delete-merged", &delete_merged, N_("pattern"), + N_("delete merged branches whose upstream matches (repeatable)"), + PARSE_OPT_NONEG, parse_opt_strvec), OPT__FORCE(&force, N_("force creation, move/rename, deletion"), PARSE_OPT_NOCOMPLETE), OPT_MERGED(&filter, N_("print only branches that are merged")), OPT_NO_MERGED(&filter, N_("print only branches that are not merged")), @@ -864,7 +1049,8 @@ int cmd_branch(int argc, 0); if (!delete && !rename && !copy && !edit_description && !new_upstream && - !show_current && !unset_upstream && argc == 0) + !show_current && !unset_upstream && !delete_merged.nr && + argc == 0) list = 1; if (filter.with_commit || filter.no_commit || @@ -874,7 +1060,7 @@ int cmd_branch(int argc, noncreate_actions = !!delete + !!rename + !!copy + !!new_upstream + !!show_current + !!list + !!edit_description + - !!unset_upstream; + !!unset_upstream + !!delete_merged.nr; if (noncreate_actions > 1) usage_with_options(builtin_branch_usage, options); @@ -916,6 +1102,10 @@ int cmd_branch(int argc, (delete > 1 ? DELETE_BRANCH_FORCE : 0) | (quiet ? DELETE_BRANCH_QUIET : 0)); goto out; + } else if (delete_merged.nr) { + ret = delete_merged_branches(&delete_merged, argv, + quiet ? DELETE_BRANCH_QUIET : 0); + goto out; } else if (show_current) { print_current_branch_name(); ret = 0; @@ -1087,6 +1277,7 @@ int cmd_branch(int argc, ret = 0; out: + strvec_clear(&delete_merged); string_list_clear(&sorting_options, 0); return ret; } diff --git a/t/t3200-branch.sh b/t/t3200-branch.sh index 84940951651e67..79bb56b1bc2e73 100755 --- a/t/t3200-branch.sh +++ b/t/t3200-branch.sh @@ -1882,4 +1882,223 @@ test_expect_success '--forked requires a value' ' test_grep "requires a value" err ' +test_expect_success '--delete-merged: setup' ' + git init -b main upstream && + ( + cd upstream && + test_commit base && + git checkout -b next && + test_commit next-work && + git checkout main + ) && + git init -b main other && + test_commit -C other other-base && + git init -b main fork +' + +setup_repo_for_delete_merged () { + rm -rf repo && + git clone upstream repo && + ( + cd repo && + git remote add fork ../fork && + git remote add other ../other && + git config push.default current && + git fetch other + ) +} + +create_merged_branch () { + ( + cd repo && + git checkout -b "$1" --track origin/next && + git commit --allow-empty -m "$1 work" && + git push origin "$1:next" + ) +} + +check_branches () { + git for-each-ref --format="%(refname:short)" refs/heads/ >actual && + cat >expect && + test_cmp expect actual +} + +test_expect_success '--delete-merged keeps cloned main without explicit push configuration' ' + setup_repo_for_delete_merged && + ( + cd repo && + test_cmp_config origin branch.main.remote && + test_cmp_config refs/heads/main branch.main.merge && + git checkout --detach && + + git branch --delete-merged */* && + + check_branches <<-\EOF + main + EOF + ) +' + +test_expect_success '--delete-merged deletes only selected merged branches' ' + setup_repo_for_delete_merged && + create_merged_branch also-merged && + create_merged_branch merged && + ( + cd repo && + git checkout -b unmerged --track origin/next && + git commit --allow-empty -m "unmerged work" && + git checkout -b tracks-other --track other/main && + sha=$(git rev-parse --short merged) && + + git branch --delete-merged origin/next merged >actual 2>&1 && + echo "Deleted branch merged (was $sha)." >expect && + test_cmp expect actual && + + check_branches <<-\EOF + also-merged + main + tracks-other + unmerged + EOF + ) +' + +test_expect_success '--delete-merged keeps main despite a different default push remote' ' + setup_repo_for_delete_merged && + create_merged_branch on-next && + create_merged_branch checked-out && + create_merged_branch upstream-gone && + ( + cd repo && + git config remote.pushDefault fork && + git checkout -b local-to-delete --track main && + git config branch.upstream-gone.merge refs/heads/topic && + git checkout -b tracks-other --track other/main && + git checkout checked-out && + + git branch --delete-merged origin/* --delete-merged main && + + check_branches <<-\EOF + checked-out + main + tracks-other + upstream-gone + EOF + ) +' + +test_expect_success '--delete-merged maps push refspecs to upstreams' ' + setup_repo_for_delete_merged && + ( + cd repo && + git checkout -b topic && + git config remote.origin.push \ + "refs/heads/topic:refs/heads/published" && + git push origin && + git branch --set-upstream-to=origin/published topic && + git checkout -b other-topic --track origin/published && + git checkout --detach && + + git branch --delete-merged origin/published && + + check_branches <<-\EOF + main + topic + EOF + ) +' + +test_expect_success '--delete-merged keeps the upstream of a surviving branch' ' + setup_repo_for_delete_merged && + create_merged_branch feature && + ( + cd repo && + git checkout -b topic --track feature && + git commit --allow-empty -m "topic work" && + + git branch --delete-merged origin/next 2>err && + + test_must_be_empty err && + check_branches <<-\EOF && + feature + main + topic + EOF + + pattern="branch\\.(feature|topic)\\.(merge|remote)" && + git config --local --get-regexp "$pattern" >actual && + cat >expect <<-\EOF && + branch.feature.remote origin + branch.feature.merge refs/heads/next + branch.topic.remote . + branch.topic.merge refs/heads/feature + EOF + test_cmp expect actual + ) +' + +test_expect_success '--delete-merged clears the deleted upstream of a protected branch' ' + setup_repo_for_delete_merged && + ( + cd repo && + git branch --track lower origin/next && + git branch --track mid lower && + git checkout -b tip --track mid && + git commit --allow-empty -m "tip work" && + sha=$(git rev-parse --short lower) && + + git branch --delete-merged origin/next \ + --delete-merged lower >actual 2>&1 && + echo "Deleted branch lower (was $sha)." >expect && + test_cmp expect actual && + + check_branches <<-\EOF && + main + mid + tip + EOF + + pattern="branch\\.(mid|tip)\\.(merge|remote)" && + git config --local --get-regexp "$pattern" >actual && + cat >expect <<-\EOF && + branch.tip.remote . + branch.tip.merge refs/heads/mid + EOF + test_cmp expect actual + ) +' + +test_expect_success '--delete-merged result is independent of stacked branch names' ' + setup_repo_for_delete_merged && + ( + cd repo && + git branch --track c-lower origin/next && + git branch --track b-mid c-lower && + git checkout -b a-tip --track b-mid && + git commit --allow-empty -m "tip work" && + + git branch --delete-merged origin/next --delete-merged "c-*" && + + check_branches <<-\EOF && + a-tip + b-mid + main + EOF + + git branch --delete-merged origin/next \ + --delete-merged "c-*" >actual 2>&1 && + test_must_be_empty actual && + + check_branches <<-\EOF + a-tip + b-mid + main + EOF + ) +' + +test_expect_success '--delete-merged requires a value' ' + test_must_fail git -C forked branch --delete-merged 2>err && + test_grep "requires a value" err +' test_done From 3d1f0df6e4ebcb8de0e8b3d968763cbbca6967a5 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Wed, 5 Aug 2026 14:24:36 +0000 Subject: [PATCH 067/259] branch: add branch..deleteMerged opt-out Setting branch..deleteMerged=false exempts that branch from "git branch --delete-merged", which is useful for a topic you want to keep developing after an early round of it has been merged upstream. Unless --quiet is given, each skip is reported so the user knows why their topic was kept. Explicit deletion with "git branch -d" still uses the normal merge check and ignores this setting. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- Documentation/config/branch.adoc | 7 +++++++ Documentation/git-branch.adoc | 3 ++- builtin/branch.c | 14 +++++++++++++ t/t3200-branch.sh | 36 ++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/Documentation/config/branch.adoc b/Documentation/config/branch.adoc index a4db9fa5c87eab..d8483acb4f9b98 100644 --- a/Documentation/config/branch.adoc +++ b/Documentation/config/branch.adoc @@ -102,3 +102,10 @@ for details). `git branch --edit-description`. Branch description is automatically added to the `format-patch` cover letter or `request-pull` summary. + +`branch..deleteMerged`:: + If set to `false`, branch __ is exempt from + `git branch --delete-merged`. Useful for a topic branch you + intend to develop further after an initial round has been + merged upstream. Defaults to true. Explicit deletion via + `git branch -d` is unaffected. diff --git a/Documentation/git-branch.adoc b/Documentation/git-branch.adoc index 47661782045e99..cfaac4b90f9692 100644 --- a/Documentation/git-branch.adoc +++ b/Documentation/git-branch.adoc @@ -224,7 +224,8 @@ A branch is not deleted when: distinguished from a branch that just looks fully merged right after a pull; this is determined by the remote's configured push and fetch refspecs, -* it is the local upstream of a branch that is not being deleted. +* it is the local upstream of a branch that is not being deleted, or +* `branch..deleteMerged` is set to `false`. -- + When such a local upstream branch has its own upstream deleted by the diff --git a/builtin/branch.c b/builtin/branch.c index f1a73bcea1cee1..2d0c4f51ea49d6 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -818,6 +818,7 @@ static int delete_merged_branches(const struct strvec *upstreams, struct strset deletable_branch_names = STRSET_INIT; struct strset protected_branch_names = STRSET_INIT; struct strvec branches_to_delete = STRVEC_INIT; + struct strbuf key = STRBUF_INIT; struct hashmap_iter iter; struct strmap_entry *entry; int ret = 0; @@ -836,6 +837,7 @@ static int delete_merged_branches(const struct strvec *upstreams, const char *branch_name; struct branch *branch; const char *upstream_refname; + int opt_out; if (!skip_prefix(branch_refname, "refs/heads/", &branch_name)) BUG("filter returned non-branch ref '%s'", branch_refname); @@ -853,6 +855,17 @@ static int delete_merged_branches(const struct strvec *upstreams, FILTER_REFS_BRANCHES, DELETE_BRANCH_SKIP_UNMERGED)) continue; + strbuf_reset(&key); + strbuf_addf(&key, "branch.%s.deletemerged", branch_name); + if (!repo_config_get_bool(the_repository, key.buf, &opt_out) && + !opt_out) { + if (!(flags & DELETE_BRANCH_QUIET)) + fprintf(stderr, + _("Skipping '%s' (branch.%s.deleteMerged is false)\n"), + branch_name, branch_name); + continue; + } + strset_add(&deletable_branch_names, branch_name); } @@ -873,6 +886,7 @@ static int delete_merged_branches(const struct strvec *upstreams, clear_deleted_upstreams(&protected_branch_names, &deletable_branch_names); + strbuf_release(&key); strvec_clear(&branches_to_delete); strset_clear(&protected_branch_names); strset_clear(&deletable_branch_names); diff --git a/t/t3200-branch.sh b/t/t3200-branch.sh index 79bb56b1bc2e73..829bbfef4e1e05 100755 --- a/t/t3200-branch.sh +++ b/t/t3200-branch.sh @@ -2101,4 +2101,40 @@ test_expect_success '--delete-merged requires a value' ' test_must_fail git -C forked branch --delete-merged 2>err && test_grep "requires a value" err ' + +test_expect_success '--delete-merged honours branch..deleteMerged=false' ' + setup_repo_for_delete_merged && + create_merged_branch deleted && + create_merged_branch kept && + ( + cd repo && + git config branch.kept.deleteMerged false && + git checkout --detach && + + git branch --delete-merged origin/next 2>err && + + test_grep "Skipping .kept." err && + check_branches <<-\EOF + kept + main + EOF + ) +' + +test_expect_success "branch -d still deletes a deleteMerged=false branch" ' + setup_repo_for_delete_merged && + create_merged_branch kept && + ( + cd repo && + git config branch.kept.deleteMerged false && + git checkout --detach && + + git branch -d kept && + + check_branches <<-\EOF + main + EOF + ) +' + test_done From 25285a6763543242514f3a58c504fd80e9996df2 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Wed, 5 Aug 2026 14:24:37 +0000 Subject: [PATCH 068/259] branch: add --dry-run for --delete-merged "git branch --dry-run --delete-merged ..." prints one line per ref that would be deleted without modifying refs or branch configuration. --dry-run is only meaningful together with --delete-merged and is rejected otherwise. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- Documentation/git-branch.adoc | 8 ++++++- builtin/branch.c | 23 ++++++++++++++++---- t/t3200-branch.sh | 41 ++++++++++++++++++++++++++++++++++- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/Documentation/git-branch.adoc b/Documentation/git-branch.adoc index cfaac4b90f9692..bfdf4593298631 100644 --- a/Documentation/git-branch.adoc +++ b/Documentation/git-branch.adoc @@ -25,7 +25,7 @@ git branch (-m|-M) [] git branch (-c|-C) [] git branch (-d|-D) [-r] ... git branch --edit-description [] -git branch (--delete-merged )... [...] +git branch [--dry-run] (--delete-merged )... [...] DESCRIPTION ----------- @@ -235,6 +235,12 @@ A branch whose work has not yet been merged into its upstream is silently skipped. Delete it with `git branch -D` if you want to remove it anyway. +`--dry-run`:: + With `--delete-merged`, print which branches would be + deleted and exit without touching any ref. Useful for + sanity-checking a wide pattern like `'origin/*'` before + committing to the deletion. + `-v`:: `-vv`:: `--verbose`:: diff --git a/builtin/branch.c b/builtin/branch.c index 2d0c4f51ea49d6..57ee384d2320e5 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -200,6 +200,7 @@ enum delete_branch_flags { DELETE_BRANCH_QUIET = (1 << 1), DELETE_BRANCH_SKIP_UNMERGED = (1 << 2), DELETE_BRANCH_NO_HEAD_FALLBACK = (1 << 3), + DELETE_BRANCH_DRY_RUN = (1 << 4), }; static int check_branch_commit(const char *branchname, const char *refname, @@ -342,13 +343,20 @@ static int delete_branches(int argc, const char **argv, int kinds, free(target); } - if (refs_delete_refs(get_main_ref_store(the_repository), NULL, &refs_to_delete, REF_NO_DEREF)) + if (!(flags & DELETE_BRANCH_DRY_RUN) && + refs_delete_refs(get_main_ref_store(the_repository), NULL, &refs_to_delete, REF_NO_DEREF)) ret = 1; for_each_string_list_item(item, &refs_to_delete) { char *describe_ref = item->util; char *name = item->string; - if (!refs_ref_exists(get_main_ref_store(the_repository), name)) { + if (flags & DELETE_BRANCH_DRY_RUN) { + if (!(flags & DELETE_BRANCH_QUIET)) + printf(remote_branch + ? _("Would delete remote-tracking branch %s (was %s).\n") + : _("Would delete branch %s (was %s).\n"), + name + branch_name_pos, describe_ref); + } else if (!refs_ref_exists(get_main_ref_store(the_repository), name)) { char *refname = name + branch_name_pos; if (!(flags & DELETE_BRANCH_QUIET)) printf(remote_branch @@ -882,7 +890,7 @@ static int delete_merged_branches(const struct strvec *upstreams, DELETE_BRANCH_NO_HEAD_FALLBACK | flags); - if (!ret) + if (!ret && !(flags & DELETE_BRANCH_DRY_RUN)) clear_deleted_upstreams(&protected_branch_names, &deletable_branch_names); @@ -960,6 +968,7 @@ int cmd_branch(int argc, int delete = 0, rename = 0, copy = 0, list = 0, unset_upstream = 0, show_current = 0, edit_description = 0; struct strvec delete_merged = STRVEC_INIT; + int dry_run = 0; const char *new_upstream = NULL; int noncreate_actions = 0; /* possible options */ @@ -1016,6 +1025,8 @@ int cmd_branch(int argc, OPT_CALLBACK_F(0, "delete-merged", &delete_merged, N_("pattern"), N_("delete merged branches whose upstream matches (repeatable)"), PARSE_OPT_NONEG, parse_opt_strvec), + OPT_BOOL(0, "dry-run", &dry_run, + N_("with --delete-merged, only print which branches would be deleted")), OPT__FORCE(&force, N_("force creation, move/rename, deletion"), PARSE_OPT_NOCOMPLETE), OPT_MERGED(&filter, N_("print only branches that are merged")), OPT_NO_MERGED(&filter, N_("print only branches that are not merged")), @@ -1078,6 +1089,9 @@ int cmd_branch(int argc, if (noncreate_actions > 1) usage_with_options(builtin_branch_usage, options); + if (dry_run && !delete_merged.nr) + die(_("--dry-run requires --delete-merged")); + if (recurse_submodules_explicit) { if (!submodule_propagate_branches) die(_("branch with --recurse-submodules can only be used if submodule.propagateBranches is enabled")); @@ -1118,7 +1132,8 @@ int cmd_branch(int argc, goto out; } else if (delete_merged.nr) { ret = delete_merged_branches(&delete_merged, argv, - quiet ? DELETE_BRANCH_QUIET : 0); + (quiet ? DELETE_BRANCH_QUIET : 0) | + (dry_run ? DELETE_BRANCH_DRY_RUN : 0)); goto out; } else if (show_current) { print_current_branch_name(); diff --git a/t/t3200-branch.sh b/t/t3200-branch.sh index 829bbfef4e1e05..0bf6b3e42e0a82 100755 --- a/t/t3200-branch.sh +++ b/t/t3200-branch.sh @@ -1950,6 +1950,20 @@ test_expect_success '--delete-merged deletes only selected merged branches' ' git checkout -b tracks-other --track other/main && sha=$(git rev-parse --short merged) && + git branch --dry-run --delete-merged origin/next merged \ + >actual 2>&1 && + echo "Would delete branch merged (was $sha)." >expect && + test_cmp expect actual && + git rev-parse --verify refs/heads/merged && + + check_branches <<-\EOF && + also-merged + main + merged + tracks-other + unmerged + EOF + git branch --delete-merged origin/next merged >actual 2>&1 && echo "Deleted branch merged (was $sha)." >expect && test_cmp expect actual && @@ -2016,9 +2030,12 @@ test_expect_success '--delete-merged keeps the upstream of a surviving branch' ' git checkout -b topic --track feature && git commit --allow-empty -m "topic work" && - git branch --delete-merged origin/next 2>err && + git branch --dry-run --delete-merged origin/next >out && + test_grep ! "feature" out && + git branch --delete-merged origin/next 2>err && test_must_be_empty err && + check_branches <<-\EOF && feature main @@ -2047,6 +2064,23 @@ test_expect_success '--delete-merged clears the deleted upstream of a protected git commit --allow-empty -m "tip work" && sha=$(git rev-parse --short lower) && + git branch --dry-run --delete-merged origin/next \ + --delete-merged lower >actual 2>&1 && + echo "Would delete branch lower (was $sha)." >expect && + test_cmp expect actual && + + pattern="branch\\.(lower|mid|tip)\\.(merge|remote)" && + git config --local --get-regexp "$pattern" >actual && + cat >expect <<-\EOF && + branch.lower.remote origin + branch.lower.merge refs/heads/next + branch.mid.remote . + branch.mid.merge refs/heads/lower + branch.tip.remote . + branch.tip.merge refs/heads/mid + EOF + test_cmp expect actual && + git branch --delete-merged origin/next \ --delete-merged lower >actual 2>&1 && echo "Deleted branch lower (was $sha)." >expect && @@ -2137,4 +2171,9 @@ test_expect_success "branch -d still deletes a deleteMerged=false branch" ' ) ' +test_expect_success '--dry-run without --delete-merged is rejected' ' + test_must_fail git -C forked branch --dry-run 2>err && + test_grep "requires --delete-merged" err +' + test_done From 4cc9039ff094a99aa2754c7b98ba6621079f0ca1 Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Thu, 6 Aug 2026 08:20:21 +0200 Subject: [PATCH 069/259] doc: refs: put ref migration warning under the command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I have to scroll down at least three screens in man(1) from the `migrate` description in order to see the “known limitations” for it. This is important information since the text says that concurrent writes can lead to an inconsistent migrated state. Let’s move that text up to the command description and put it inside a Warning admonition. This section made sense when it was added in 25a0023f (builtin/refs: new command to migrate ref storage formats, 2024-06-06); `migrate` was the only subcommand, and this section was visible from the command description. A one-page man page. But that is not the case anymore now that the command has nine subcommands to describe. Acked-by: Patrick Steinhardt Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-refs.adoc | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Documentation/git-refs.adoc b/Documentation/git-refs.adoc index ce278c59bfc1dc..3b5af936ed614b 100644 --- a/Documentation/git-refs.adoc +++ b/Documentation/git-refs.adoc @@ -35,6 +35,21 @@ COMMANDS `migrate`:: Migrate ref store between different formats. ++ +[WARNING] +-- +The ref format migration has several known limitations in its current form: + +* It is not possible to migrate repositories that have worktrees. + +* There is no way to block concurrent writes to the repository during an + ongoing migration. Concurrent writes can lead to an inconsistent migrated + state. Users are expected to block writes on a higher level. If your + repository is registered for scheduled maintenance, it is recommended to + unregister it first with git-maintenance(1). + +These limitations may eventually be lifted. +-- `verify`:: Verify reference database consistency. @@ -130,21 +145,6 @@ The following options are specific to commands which write references: Operate on itself rather than the reference it points to via a symbolic ref. -KNOWN LIMITATIONS ------------------ - -The ref format migration has several known limitations in its current form: - -* It is not possible to migrate repositories that have worktrees. - -* There is no way to block concurrent writes to the repository during an - ongoing migration. Concurrent writes can lead to an inconsistent migrated - state. Users are expected to block writes on a higher level. If your - repository is registered for scheduled maintenance, it is recommended to - unregister it first with git-maintenance(1). - -These limitations may eventually be lifted. - GIT --- Part of the linkgit:git[1] suite From 0dc68f404af778338a4090a857d51f16b9ed54b8 Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Thu, 6 Aug 2026 08:20:22 +0200 Subject: [PATCH 070/259] doc: refs: linkgit to git-maintenance(1) Acked-by: Patrick Steinhardt Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-refs.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/git-refs.adoc b/Documentation/git-refs.adoc index 3b5af936ed614b..9063892651e478 100644 --- a/Documentation/git-refs.adoc +++ b/Documentation/git-refs.adoc @@ -46,7 +46,7 @@ The ref format migration has several known limitations in its current form: ongoing migration. Concurrent writes can lead to an inconsistent migrated state. Users are expected to block writes on a higher level. If your repository is registered for scheduled maintenance, it is recommended to - unregister it first with git-maintenance(1). + unregister it first with linkgit:git-maintenance[1]. These limitations may eventually be lifted. -- From f85ffdfba1fa0991de374d961d2be437822c4aae Mon Sep 17 00:00:00 2001 From: K Jayatheerth Date: Thu, 6 Aug 2026 15:45:50 +0530 Subject: [PATCH 071/259] repo: add path.toplevel with absolute and relative suffix formatting Scripts frequently need to find the root directory of a repository's working tree. Currently, this requires using `git rev-parse --show-toplevel` or inferring it from other repository information. Introduce `path.toplevel.absolute` and `path.toplevel.relative` keys to `git repo info`. This allows scripts to retrieve the top-level working tree path in a predictable, strictly formatted manner without relying on `rev-parse`. If requested in a bare repository where no working tree exists, the command returns an empty string. Mentored-by: Justin Tobler Mentored-by: Lucas Seiki Oshiro Signed-off-by: K Jayatheerth Signed-off-by: Junio C Hamano --- Documentation/git-repo.adoc | 10 ++++++++++ builtin/repo.c | 24 ++++++++++++++++++++++++ t/t1900-repo-info.sh | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index ed7d80c690c720..e34abe5feae134 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -119,6 +119,16 @@ values that they return: `path.gitdir.relative`:: The path to the Git repository directory relative to the current working directory. +`path.toplevel.absolute`:: + The canonical absolute path to the top-level directory of the + repository's working tree. Outputs an empty string if the repository + is bare. + +`path.toplevel.relative`:: + The path to the top-level directory of the repository's working + tree relative to the current working directory. Outputs an empty + string if the repository is bare. + `references.format`:: The reference storage format. The valid values are: + diff --git a/builtin/repo.c b/builtin/repo.c index 042d6de558e930..15f899c7c7d502 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -121,6 +121,28 @@ static int get_path_gitdir_relative(struct repository *repo, struct strbuf *buf) return 0; } +static int get_path_toplevel_absolute(struct repository *repo, struct strbuf *buf) +{ + const char *work_tree = repo_get_work_tree(repo); + + if (!work_tree) + return 0; + + format_path(buf, work_tree, "", PATH_FORMAT_CANONICAL); + return 0; +} + +static int get_path_toplevel_relative(struct repository *repo, struct strbuf *buf) +{ + const char *work_tree = repo_get_work_tree(repo); + + if (!work_tree) + return 0; + + format_path(buf, work_tree, repo->prefix, PATH_FORMAT_RELATIVE); + return 0; +} + static int get_references_format(struct repository *repo, struct strbuf *buf) { strbuf_addstr(buf, @@ -137,6 +159,8 @@ static const struct repo_info_field repo_info_field[] = { { "path.commondir.relative", get_path_commondir_relative }, { "path.gitdir.absolute", get_path_gitdir_absolute }, { "path.gitdir.relative", get_path_gitdir_relative }, + { "path.toplevel.absolute", get_path_toplevel_absolute }, + { "path.toplevel.relative", get_path_toplevel_relative }, { "references.format", get_references_format }, }; diff --git a/t/t1900-repo-info.sh b/t/t1900-repo-info.sh index ae8c22c81749ba..46b5ae3dbcd3cb 100755 --- a/t/t1900-repo-info.sh +++ b/t/t1900-repo-info.sh @@ -213,4 +213,39 @@ test_repo_info_path 'gitdir with explicit GIT_DIR' 'gitdir' \ '.git' \ 'GIT_DIR="../.git" && export GIT_DIR' +test_expect_success 'path.toplevel absolute and relative' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + mkdir -p repo/sub && + cd repo/sub && + + ROOT="$(test-tool path-utils real_path ..)" && + + echo "path.toplevel.absolute=$ROOT" >expect.abs && + git repo info path.toplevel.absolute >actual.abs && + test_cmp expect.abs actual.abs && + + echo "path.toplevel.relative=../" >expect.rel && + git repo info path.toplevel.relative >actual.rel && + test_cmp expect.rel actual.rel + ) +' + +test_expect_success 'path.toplevel absolute and relative in a bare repository' ' + test_when_finished "rm -rf bare.git" && + git init --bare bare.git && + ( + cd bare.git && + + echo "path.toplevel.absolute=" >expect.abs && + git repo info path.toplevel.absolute >actual.abs && + test_cmp expect.abs actual.abs && + + echo "path.toplevel.relative=" >expect.rel && + git repo info path.toplevel.relative >actual.rel && + test_cmp expect.rel actual.rel + ) +' + test_done From 8ca70bf5a58a5bebd3552e2c77a82b12c87af89e Mon Sep 17 00:00:00 2001 From: K Jayatheerth Date: Thu, 6 Aug 2026 15:45:51 +0530 Subject: [PATCH 072/259] repo: add path.superproject-root with absolute and relative suffixes Scripts working in multi-repository setups often need to identify the top-level working tree of a superproject from within a submodule. Currently, this is only exposed via `git rev-parse --show-superproject-working-tree`. Introduce `path.superproject-root.absolute` and `path.superproject-root.relative` keys to `git repo info`. This exposes the core submodule context via a scriptable config-like key using standard format rules. If requested when not inside a submodule, the command returns an empty string. Mentored-by: Justin Tobler Mentored-by: Lucas Seiki Oshiro Signed-off-by: K Jayatheerth Signed-off-by: Junio C Hamano --- Documentation/git-repo.adoc | 10 ++++++++++ builtin/repo.c | 31 +++++++++++++++++++++++++++++ t/t1900-repo-info.sh | 39 +++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index e34abe5feae134..e524a07f53668b 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -119,6 +119,16 @@ values that they return: `path.gitdir.relative`:: The path to the Git repository directory relative to the current working directory. +`path.superproject-root.absolute`:: + The canonical absolute path to the working tree root of the superproject + if the current repository is an initialized submodule. Outputs an empty + string if not in a submodule. + +`path.superproject-root.relative`:: + The path to the working tree root of the superproject relative to the + current working directory if the current repository is an initialized + submodule. Outputs an empty string if not in a submodule. + `path.toplevel.absolute`:: The canonical absolute path to the top-level directory of the repository's working tree. Outputs an empty string if the repository diff --git a/builtin/repo.c b/builtin/repo.c index 15f899c7c7d502..144526512e5fe5 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -18,6 +18,7 @@ #include "strbuf.h" #include "string-list.h" #include "shallow.h" +#include "submodule.h" #include "tree.h" #include "tree-walk.h" #include "utf8.h" @@ -121,6 +122,34 @@ static int get_path_gitdir_relative(struct repository *repo, struct strbuf *buf) return 0; } +static int get_path_superproject_absolute(struct repository *repo UNUSED, struct strbuf *buf) +{ + struct strbuf superproject = STRBUF_INIT; + + if (!get_superproject_working_tree(&superproject)) { + strbuf_release(&superproject); + return 0; + } + + format_path(buf, superproject.buf, "", PATH_FORMAT_CANONICAL); + strbuf_release(&superproject); + return 0; +} + +static int get_path_superproject_relative(struct repository *repo, struct strbuf *buf) +{ + struct strbuf superproject = STRBUF_INIT; + + if (!get_superproject_working_tree(&superproject)) { + strbuf_release(&superproject); + return 0; + } + + format_path(buf, superproject.buf, repo->prefix, PATH_FORMAT_RELATIVE); + strbuf_release(&superproject); + return 0; +} + static int get_path_toplevel_absolute(struct repository *repo, struct strbuf *buf) { const char *work_tree = repo_get_work_tree(repo); @@ -159,6 +188,8 @@ static const struct repo_info_field repo_info_field[] = { { "path.commondir.relative", get_path_commondir_relative }, { "path.gitdir.absolute", get_path_gitdir_absolute }, { "path.gitdir.relative", get_path_gitdir_relative }, + { "path.superproject-root.absolute", get_path_superproject_absolute }, + { "path.superproject-root.relative", get_path_superproject_relative }, { "path.toplevel.absolute", get_path_toplevel_absolute }, { "path.toplevel.relative", get_path_toplevel_relative }, { "references.format", get_references_format }, diff --git a/t/t1900-repo-info.sh b/t/t1900-repo-info.sh index 46b5ae3dbcd3cb..30037a49044e2a 100755 --- a/t/t1900-repo-info.sh +++ b/t/t1900-repo-info.sh @@ -213,6 +213,45 @@ test_repo_info_path 'gitdir with explicit GIT_DIR' 'gitdir' \ '.git' \ 'GIT_DIR="../.git" && export GIT_DIR' +test_expect_success 'path.superproject-root absolute and relative' ' + test_when_finished "rm -rf sub super" && + git init sub && + test_commit -C sub initial && + git init super && + ( + cd super && + git -c protocol.file.allow=always submodule add "../sub" sub && + git commit -m "add submodule" && + + cd sub && + ROOT="$(test-tool path-utils real_path ..)" && + + echo "path.superproject-root.absolute=$ROOT" >expect.abs && + git repo info path.superproject-root.absolute >actual.abs && + test_cmp expect.abs actual.abs && + + echo "path.superproject-root.relative=../" >expect.rel && + git repo info path.superproject-root.relative >actual.rel && + test_cmp expect.rel actual.rel + ) +' + +test_expect_success 'path.superproject-root returns empty when not in a submodule' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + + echo "path.superproject-root.absolute=" >expect.abs && + git repo info path.superproject-root.absolute >actual.abs && + test_cmp expect.abs actual.abs && + + echo "path.superproject-root.relative=" >expect.rel && + git repo info path.superproject-root.relative >actual.rel && + test_cmp expect.rel actual.rel + ) +' + test_expect_success 'path.toplevel absolute and relative' ' test_when_finished "rm -rf repo" && git init repo && From 18d1b24d07c741c3453b24842de0bc267fc489d5 Mon Sep 17 00:00:00 2001 From: K Jayatheerth Date: Thu, 6 Aug 2026 15:45:52 +0530 Subject: [PATCH 073/259] repo: add path.hooks with absolute and relative suffixes Hooks are an integral part of a repository's configuration and are commonly used by tooling to automate repository-specific workflows. Currently, scripts typically retrieve the hooks directory by invoking `git rev-parse --git-path hooks`. Introduce `path.hooks.absolute` and `path.hooks.relative` keys to `git repo info`. This exposes the hooks directory as a scriptable config-like key using standard format rules, allowing scripts to retrieve it through the same interface as other repository path information. Mentored-by: Justin Tobler Mentored-by: Lucas Seiki Oshiro Signed-off-by: K Jayatheerth Signed-off-by: Junio C Hamano --- Documentation/git-repo.adoc | 9 +++++++++ builtin/repo.c | 22 ++++++++++++++++++++++ t/t1900-repo-info.sh | 28 ++++++++++++++++++++++++++-- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index e524a07f53668b..20836cf8f6a40b 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -119,6 +119,15 @@ values that they return: `path.gitdir.relative`:: The path to the Git repository directory relative to the current working directory. +`path.hooks.absolute`:: + The canonical absolute path to the repository's hooks directory. + Respects the `core.hooksPath` configuration. If `core.hooksPath` is + set to `/dev/null`, that value is returned unchanged. + +`path.hooks.relative`:: + The path to the repository's hooks directory relative to the current + working directory. Respects the `core.hooksPath` configuration. + `path.superproject-root.absolute`:: The canonical absolute path to the working tree root of the superproject if the current repository is an initialized submodule. Outputs an empty diff --git a/builtin/repo.c b/builtin/repo.c index 144526512e5fe5..acc8d9335dade0 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -122,6 +122,26 @@ static int get_path_gitdir_relative(struct repository *repo, struct strbuf *buf) return 0; } +static int get_path_hooks_absolute(struct repository *repo, struct strbuf *buf) +{ + struct strbuf hooks_path = STRBUF_INIT; + + repo_git_path_replace(repo, &hooks_path, "hooks"); + format_path(buf, hooks_path.buf, "", PATH_FORMAT_CANONICAL); + strbuf_release(&hooks_path); + return 0; +} + +static int get_path_hooks_relative(struct repository *repo, struct strbuf *buf) +{ + struct strbuf hooks_path = STRBUF_INIT; + + repo_git_path_replace(repo, &hooks_path, "hooks"); + format_path(buf, hooks_path.buf, repo->prefix, PATH_FORMAT_RELATIVE); + strbuf_release(&hooks_path); + return 0; +} + static int get_path_superproject_absolute(struct repository *repo UNUSED, struct strbuf *buf) { struct strbuf superproject = STRBUF_INIT; @@ -188,6 +208,8 @@ static const struct repo_info_field repo_info_field[] = { { "path.commondir.relative", get_path_commondir_relative }, { "path.gitdir.absolute", get_path_gitdir_absolute }, { "path.gitdir.relative", get_path_gitdir_relative }, + { "path.hooks.absolute", get_path_hooks_absolute }, + { "path.hooks.relative", get_path_hooks_relative }, { "path.superproject-root.absolute", get_path_superproject_absolute }, { "path.superproject-root.relative", get_path_superproject_relative }, { "path.toplevel.absolute", get_path_toplevel_absolute }, diff --git a/t/t1900-repo-info.sh b/t/t1900-repo-info.sh index 30037a49044e2a..fcd771b711c3ce 100755 --- a/t/t1900-repo-info.sh +++ b/t/t1900-repo-info.sh @@ -174,7 +174,11 @@ test_repo_info_path () { cd repo/sub && ROOT="$(test-tool path-utils real_path ..)" && export ROOT && eval "$init_command" && - echo "path.$field_name.absolute=$ROOT/$expected_dir" >expect && + case "$expected_dir" in + /*) EXPECT_ABS="$expected_dir" ;; + *) EXPECT_ABS="$ROOT/$expected_dir" ;; + esac && + echo "path.$field_name.absolute=$EXPECT_ABS" >expect && git repo info "path.$field_name.absolute" >actual && test_cmp expect actual ) @@ -188,7 +192,11 @@ test_repo_info_path () { cd repo/sub && ROOT="$(test-tool path-utils real_path ..)" && export ROOT && eval "$init_command" && - echo "path.$field_name.relative=../$expected_dir" >expect && + case "$expected_dir" in + /*) EXPECT_REL="$(test-tool path-utils relative_path "$expected_dir" "$PWD")" ;; + *) EXPECT_REL="../$expected_dir" ;; + esac && + echo "path.$field_name.relative=$EXPECT_REL" >expect && git repo info "path.$field_name.relative" >actual && test_cmp expect actual ) @@ -213,6 +221,22 @@ test_repo_info_path 'gitdir with explicit GIT_DIR' 'gitdir' \ '.git' \ 'GIT_DIR="../.git" && export GIT_DIR' +test_repo_info_path 'hooks standard' 'hooks' '.git/hooks' + +test_repo_info_path 'hooks with core.hooksPath override' 'hooks' \ + 'custom-hooks' \ + 'git config core.hooksPath "$ROOT/custom-hooks" && mkdir -p "$ROOT/custom-hooks"' + +# /dev/null is not a real, canonicalizable filesystem path on Windows, +# so path resolution for core.hooksPath=/dev/null cannot be expected to +# produce a literal "/dev/null" the way it does on POSIX systems. +if ! test_have_prereq MINGW +then + test_repo_info_path 'hooks with core.hooksPath=/dev/null' 'hooks' \ + '/dev/null' \ + 'git config core.hooksPath /dev/null' +fi + test_expect_success 'path.superproject-root absolute and relative' ' test_when_finished "rm -rf sub super" && git init sub && From e3f48646e1c7ae756463368e40649c4da88b7730 Mon Sep 17 00:00:00 2001 From: K Jayatheerth Date: Thu, 6 Aug 2026 15:45:53 +0530 Subject: [PATCH 074/259] repo: add path.index with absolute and relative suffixes The repository index is a fundamental component used by Git and related tooling to track the working tree state. Scripts that interact with the index currently retrieve its location by invoking `git rev-parse --git-path index`. Introduce `path.index.absolute` and `path.index.relative` keys to `git repo info`. This exposes the index file location as a scriptable config-like key using standard format rules, allowing scripts to retrieve it through the same interface as other repository path information. Mentored-by: Justin Tobler Mentored-by: Lucas Seiki Oshiro Signed-off-by: K Jayatheerth Signed-off-by: Junio C Hamano --- Documentation/git-repo.adoc | 12 ++++++++++++ builtin/repo.c | 24 ++++++++++++++++++++++++ t/t1900-repo-info.sh | 23 +++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index 20836cf8f6a40b..08ef47750c5fb1 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -128,6 +128,18 @@ values that they return: The path to the repository's hooks directory relative to the current working directory. Respects the `core.hooksPath` configuration. +`path.index.absolute`:: + The canonical absolute path to the repository's current index file. + Respects the `GIT_INDEX_FILE` environment override. Returns the + configured index path even if the repository is bare or the file does + not exist. + +`path.index.relative`:: + The path to the repository's current index file relative to the current + working directory. Respects the `GIT_INDEX_FILE` environment override. + Returns the configured index path even if the repository is bare or the + file does not exist. + `path.superproject-root.absolute`:: The canonical absolute path to the working tree root of the superproject if the current repository is an initialized submodule. Outputs an empty diff --git a/builtin/repo.c b/builtin/repo.c index acc8d9335dade0..7e1cad7a324ee2 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -142,6 +142,28 @@ static int get_path_hooks_relative(struct repository *repo, struct strbuf *buf) return 0; } +static int get_path_index_absolute(struct repository *repo, struct strbuf *buf) +{ + const char *index_file = repo_get_index_file(repo); + + if (!index_file) + return error(_("unable to get index file")); + + format_path(buf, index_file, "", PATH_FORMAT_CANONICAL); + return 0; +} + +static int get_path_index_relative(struct repository *repo, struct strbuf *buf) +{ + const char *index_file = repo_get_index_file(repo); + + if (!index_file) + return error(_("unable to get index file")); + + format_path(buf, index_file, repo->prefix, PATH_FORMAT_RELATIVE); + return 0; +} + static int get_path_superproject_absolute(struct repository *repo UNUSED, struct strbuf *buf) { struct strbuf superproject = STRBUF_INIT; @@ -210,6 +232,8 @@ static const struct repo_info_field repo_info_field[] = { { "path.gitdir.relative", get_path_gitdir_relative }, { "path.hooks.absolute", get_path_hooks_absolute }, { "path.hooks.relative", get_path_hooks_relative }, + { "path.index.absolute", get_path_index_absolute }, + { "path.index.relative", get_path_index_relative }, { "path.superproject-root.absolute", get_path_superproject_absolute }, { "path.superproject-root.relative", get_path_superproject_relative }, { "path.toplevel.absolute", get_path_toplevel_absolute }, diff --git a/t/t1900-repo-info.sh b/t/t1900-repo-info.sh index fcd771b711c3ce..999a95d33244a0 100755 --- a/t/t1900-repo-info.sh +++ b/t/t1900-repo-info.sh @@ -237,6 +237,29 @@ then 'git config core.hooksPath /dev/null' fi +test_repo_info_path 'index standard' 'index' '.git/index' + +test_repo_info_path 'index with GIT_INDEX_FILE override' 'index' \ + 'custom-index-file' \ + 'GIT_INDEX_FILE="$ROOT/custom-index-file" && export GIT_INDEX_FILE' + +test_expect_success 'path.index in a bare repository returns default index location' ' + test_when_finished "rm -rf bare.git" && + git init --bare bare.git && + ( + cd bare.git && + ROOT="$(test-tool path-utils real_path .)" && + + echo "path.index.absolute=$ROOT/index" >expect.abs && + git repo info path.index.absolute >actual.abs && + test_cmp expect.abs actual.abs && + + echo "path.index.relative=index" >expect.rel && + git repo info path.index.relative >actual.rel && + test_cmp expect.rel actual.rel + ) +' + test_expect_success 'path.superproject-root absolute and relative' ' test_when_finished "rm -rf sub super" && git init sub && From 21b464bcda406769c2ab4421b71c3105121ed6e4 Mon Sep 17 00:00:00 2001 From: K Jayatheerth Date: Thu, 6 Aug 2026 15:45:54 +0530 Subject: [PATCH 075/259] repo: add path.grafts with absolute and relative suffixes The repository grafts file specifies alternate parent relationships for commits and may be used by repository tooling that needs to inspect or manage grafts. Scripts currently retrieve its location by invoking `git rev-parse --git-path info/grafts`. Introduce `path.grafts.absolute` and `path.grafts.relative` keys to `git repo info`. This exposes the grafts file location as a scriptable config-like key using standard format rules, allowing scripts to retrieve it through the same interface as other repository path information. Mentored-by: Justin Tobler Mentored-by: Lucas Seiki Oshiro Signed-off-by: K Jayatheerth Signed-off-by: Junio C Hamano --- Documentation/git-repo.adoc | 11 +++++++++++ builtin/repo.c | 24 ++++++++++++++++++++++++ t/t1900-repo-info.sh | 6 ++++++ 3 files changed, 41 insertions(+) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index 08ef47750c5fb1..868ab0ed9fed40 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -119,6 +119,17 @@ values that they return: `path.gitdir.relative`:: The path to the Git repository directory relative to the current working directory. +`path.grafts.absolute`:: + The canonical absolute path to the repository's graft file. + Respects the `GIT_GRAFT_FILE` environment override. The path is + returned regardless of whether the file currently exists on disk. + +`path.grafts.relative`:: + The path to the repository's graft file relative to the current + working directory. Respects the `GIT_GRAFT_FILE` environment + override. The path is returned regardless of whether the file + currently exists on disk. + `path.hooks.absolute`:: The canonical absolute path to the repository's hooks directory. Respects the `core.hooksPath` configuration. If `core.hooksPath` is diff --git a/builtin/repo.c b/builtin/repo.c index 7e1cad7a324ee2..b20a96f251c997 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -122,6 +122,28 @@ static int get_path_gitdir_relative(struct repository *repo, struct strbuf *buf) return 0; } +static int get_path_grafts_absolute(struct repository *repo, struct strbuf *buf) +{ + const char *graft_file = repo_get_graft_file(repo); + + if (!graft_file) + return error(_("unable to get graft file")); + + format_path(buf, graft_file, "", PATH_FORMAT_CANONICAL); + return 0; +} + +static int get_path_grafts_relative(struct repository *repo, struct strbuf *buf) +{ + const char *graft_file = repo_get_graft_file(repo); + + if (!graft_file) + return error(_("unable to get graft file")); + + format_path(buf, graft_file, repo->prefix, PATH_FORMAT_RELATIVE); + return 0; +} + static int get_path_hooks_absolute(struct repository *repo, struct strbuf *buf) { struct strbuf hooks_path = STRBUF_INIT; @@ -230,6 +252,8 @@ static const struct repo_info_field repo_info_field[] = { { "path.commondir.relative", get_path_commondir_relative }, { "path.gitdir.absolute", get_path_gitdir_absolute }, { "path.gitdir.relative", get_path_gitdir_relative }, + { "path.grafts.absolute", get_path_grafts_absolute }, + { "path.grafts.relative", get_path_grafts_relative }, { "path.hooks.absolute", get_path_hooks_absolute }, { "path.hooks.relative", get_path_hooks_relative }, { "path.index.absolute", get_path_index_absolute }, diff --git a/t/t1900-repo-info.sh b/t/t1900-repo-info.sh index 999a95d33244a0..74ace464adee24 100755 --- a/t/t1900-repo-info.sh +++ b/t/t1900-repo-info.sh @@ -221,6 +221,12 @@ test_repo_info_path 'gitdir with explicit GIT_DIR' 'gitdir' \ '.git' \ 'GIT_DIR="../.git" && export GIT_DIR' +test_repo_info_path 'grafts standard' 'grafts' '.git/info/grafts' + +test_repo_info_path 'grafts with GIT_GRAFT_FILE override' 'grafts' \ + 'custom-graft-file' \ + 'GIT_GRAFT_FILE="$ROOT/custom-graft-file" && export GIT_GRAFT_FILE' + test_repo_info_path 'hooks standard' 'hooks' '.git/hooks' test_repo_info_path 'hooks with core.hooksPath override' 'hooks' \ From 30639982cb017cfb68e694d44737bc4eaf6f4e07 Mon Sep 17 00:00:00 2001 From: K Jayatheerth Date: Thu, 6 Aug 2026 15:45:55 +0530 Subject: [PATCH 076/259] repo: add path.git-prefix Scripts sometimes need the path from the repository's working tree root to the current working directory. While this information can be derived through existing Git commands, `git repo info` does not currently expose it as a scriptable key. Introduce the `path.git-prefix` key to `git repo info`. The key returns the path from the working tree root to the current working directory, returning the empty string when invoked from the working tree root. Mentored-by: Justin Tobler Mentored-by: Lucas Seiki Oshiro Signed-off-by: K Jayatheerth Signed-off-by: Junio C Hamano --- Documentation/git-repo.adoc | 5 +++++ builtin/repo.c | 11 +++++++++++ t/t1900-repo-info.sh | 23 +++++++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index 868ab0ed9fed40..fb5aceae8f2ce7 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -113,6 +113,11 @@ values that they return: The path to the Git repository's common directory relative to the current working directory. +`path.git-prefix`:: + The path from the root of the working tree to the current working + directory. Returns the empty string when the current working directory + is the root of the working tree. + `path.gitdir.absolute`:: The canonical absolute path to the Git repository directory (the `.git` directory). diff --git a/builtin/repo.c b/builtin/repo.c index b20a96f251c997..2a4e012e0605c4 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -100,6 +100,16 @@ static int get_path_commondir_relative(struct repository *repo, struct strbuf *b return 0; } +static int get_path_git_prefix(struct repository *repo, struct strbuf *buf) +{ + /* + * repo->prefix is NULL when the current working directory is + * the worktree root. + */ + strbuf_addstr(buf, repo->prefix ? repo->prefix : ""); + return 0; +} + static int get_path_gitdir_absolute(struct repository *repo, struct strbuf *buf) { const char *git_dir = repo_get_git_dir(repo); @@ -250,6 +260,7 @@ static const struct repo_info_field repo_info_field[] = { { "object.format", get_object_format }, { "path.commondir.absolute", get_path_commondir_absolute }, { "path.commondir.relative", get_path_commondir_relative }, + { "path.git-prefix", get_path_git_prefix }, { "path.gitdir.absolute", get_path_gitdir_absolute }, { "path.gitdir.relative", get_path_gitdir_relative }, { "path.grafts.absolute", get_path_grafts_absolute }, diff --git a/t/t1900-repo-info.sh b/t/t1900-repo-info.sh index 74ace464adee24..da3014a1d6c5a9 100755 --- a/t/t1900-repo-info.sh +++ b/t/t1900-repo-info.sh @@ -215,6 +215,29 @@ test_repo_info_path 'commondir with only GIT_DIR' 'commondir' \ '.git' \ 'GIT_DIR="../.git" && export GIT_DIR' +test_expect_success 'path.git-prefix at repository root' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + echo "path.git-prefix=" >expect && + git repo info path.git-prefix >actual && + test_cmp expect actual + ) +' + +test_expect_success 'path.git-prefix in subdirectory' ' + test_when_finished "rm -rf repo" && + git init repo && + mkdir -p repo/sub/dir && + ( + cd repo/sub/dir && + echo "path.git-prefix=sub/dir/" >expect && + git repo info path.git-prefix >actual && + test_cmp expect actual + ) +' + test_repo_info_path 'gitdir standard' 'gitdir' '.git' test_repo_info_path 'gitdir with explicit GIT_DIR' 'gitdir' \ From 84d9b4dc979ee107d4bcc525a8c576db44570a30 Mon Sep 17 00:00:00 2001 From: K Jayatheerth Date: Thu, 6 Aug 2026 15:45:56 +0530 Subject: [PATCH 077/259] repo: remove unused setup.h include The repository prefix is now stored in `struct repository`, so builtin/repo.c no longer uses any declarations from setup.h. Remove the now-unused include. Mentored-by: Justin Tobler Mentored-by: Lucas Seiki Oshiro Signed-off-by: K Jayatheerth Signed-off-by: Junio C Hamano --- builtin/repo.c | 1 - 1 file changed, 1 deletion(-) diff --git a/builtin/repo.c b/builtin/repo.c index 2a4e012e0605c4..3df938e852e5a3 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -14,7 +14,6 @@ #include "ref-filter.h" #include "refs.h" #include "revision.h" -#include "setup.h" #include "strbuf.h" #include "string-list.h" #include "shallow.h" From ca571025d86b55933d493e38d6e72824bcf5a80a Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Aug 2026 05:34:25 +0200 Subject: [PATCH 078/259] loose: load loose object map for the correct source When loading the loose object map via `load_one_loose_object_map()` we pass in both a repository and the corresponding source. We ultimately don't really respect the passed-in source though as we instead always load the map via the common directory. This doesn't make any sense though, as the function is called in a loop through all sources, and as such the expectation is that we'll load the map that belongs to the given source. The consequence is that we'll ignore loose object maps of any configured alternates. Fix this bug by instead loading the map via the loose source's path. Helped-by: Toon Claes Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- loose.c | 18 ++++++++++-------- t/t1016-compatObjectFormat.sh | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/loose.c b/loose.c index bf01d3e42def34..9dad75373b8080 100644 --- a/loose.c +++ b/loose.c @@ -61,9 +61,11 @@ static int insert_loose_map(struct odb_source_loose *loose, return inserted; } -static int load_one_loose_object_map(struct repository *repo, struct odb_source_loose *loose) +static int load_one_loose_object_map(struct odb_source_loose *loose) { - struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT; + struct repository *repo = loose->base.odb->repo; + struct strbuf buf = STRBUF_INIT; + char *path; FILE *fp; int ret = -1; @@ -78,10 +80,10 @@ static int load_one_loose_object_map(struct repository *repo, struct odb_source_ insert_loose_map(loose, repo->hash_algo->empty_blob, repo->compat_hash_algo->empty_blob); insert_loose_map(loose, repo->hash_algo->null_oid, repo->compat_hash_algo->null_oid); - repo_common_path_replace(repo, &path, "objects/loose-object-idx"); - fp = fopen(path.buf, "rb"); + path = xstrfmt("%s/loose-object-idx", loose->base.path); + fp = fopen(path, "rb"); if (!fp) { - strbuf_release(&path); + free(path); return 0; } @@ -102,7 +104,7 @@ static int load_one_loose_object_map(struct repository *repo, struct odb_source_ err: fclose(fp); strbuf_release(&buf); - strbuf_release(&path); + free(path); return ret; } @@ -117,10 +119,10 @@ int repo_read_loose_object_map(struct repository *repo) for (source = repo->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - if (load_one_loose_object_map(repo, files->loose) < 0) { + if (load_one_loose_object_map(files->loose) < 0) return -1; - } } + return 0; } diff --git a/t/t1016-compatObjectFormat.sh b/t/t1016-compatObjectFormat.sh index 92d48b96a10932..9cafcee5098692 100755 --- a/t/t1016-compatObjectFormat.sh +++ b/t/t1016-compatObjectFormat.sh @@ -187,6 +187,24 @@ do eval signedtag3_${hash}_oid=$(git hash-object -t tag -w ../${hash}_signedtag3) && eval signedtag4_${hash}_oid=$(git hash-object -t tag -w ../${hash}_signedtag4) ' + + test_expect_success 'rev-parse maps oid of object borrowed from alternate' ' + for repo in alt borrow + do + test_when_finished "rm -rf $repo" && + git init --object-format=$hash $repo && + git -C $repo config set core.repositoryformatversion 1 && + git -C $repo config set extensions.compatObjectFormat $(compat_hash $hash) || exit 1 + done && + + git -C alt commit --allow-empty --message A && + echo "$(pwd)/alt/.git/objects" >borrow/.git/objects/info/alternates && + + oid=$(git -C alt rev-parse HEAD) && + git -C alt rev-parse --output-object-format=$(compat_hash $hash) "$oid" >expect && + git -C borrow rev-parse --output-object-format=$(compat_hash $hash) "$oid" >actual && + test_cmp expect actual + ' done cd "$base" From 8a1ba94eb5863cd7491899bb23a290081e760453 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Aug 2026 05:34:26 +0200 Subject: [PATCH 079/259] setup: detangle loading of loose object maps When a repository is configured to use a compatibility hash function then we load the loose object map when we initialize the repository. This object map provides the mappings between the canonical object hash and the compatibility object hash. Loading the object map happens in `repo_set_compat_hash_algo()`, which calls `repo_read_loose_object_map()` in case the compatibility object hash is non-zero. This setup sequence has two major downsides: - We assume that the primary object database is the "files" object database and unconditionally downcast it. This will cause us to BUG in case a different object database type was used together with a compat hash algorithm. - We require the object database to already have been initialized when configuring the object database. This means that we must intermix configuration of the repository and initialization of its sub-structures in a weird way. Refactor the logic so that we instead load the loose object map via the "loose" backend, which fixes both of the above issues. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- loose.c | 11 +++++------ loose.h | 1 + odb/source-loose.c | 2 ++ repository.c | 2 -- setup.c | 5 +++-- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/loose.c b/loose.c index 9dad75373b8080..a3b2dcedc23607 100644 --- a/loose.c +++ b/loose.c @@ -61,7 +61,7 @@ static int insert_loose_map(struct odb_source_loose *loose, return inserted; } -static int load_one_loose_object_map(struct odb_source_loose *loose) +int loose_object_map_load(struct odb_source_loose *loose) { struct repository *repo = loose->base.odb->repo; struct strbuf buf = STRBUF_INIT; @@ -69,6 +69,9 @@ static int load_one_loose_object_map(struct odb_source_loose *loose) FILE *fp; int ret = -1; + if (!should_use_loose_object_map(repo)) + return 0; + if (!loose->map) loose_object_map_init(&loose->map); if (!loose->cache) { @@ -112,14 +115,10 @@ int repo_read_loose_object_map(struct repository *repo) { struct odb_source *source; - if (!should_use_loose_object_map(repo)) - return 0; - odb_prepare_alternates(repo->objects); - for (source = repo->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - if (load_one_loose_object_map(files->loose) < 0) + if (loose_object_map_load(files->loose) < 0) return -1; } diff --git a/loose.h b/loose.h index 6c9b3f4571602f..ed663ac550fbb7 100644 --- a/loose.h +++ b/loose.h @@ -13,6 +13,7 @@ struct loose_object_map { void loose_object_map_init(struct loose_object_map **map); void loose_object_map_clear(struct loose_object_map **map); +int loose_object_map_load(struct odb_source_loose *loose); int repo_loose_object_map_oid(struct repository *repo, const struct object_id *src, const struct git_hash_algo *dest_algo, diff --git a/odb/source-loose.c b/odb/source-loose.c index 3f7d04a56e36ce..812ca1c1381bab 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -727,5 +727,7 @@ struct odb_source_loose *odb_source_loose_new(struct object_database *odb, if (!is_absolute_path(loose->base.path)) chdir_notify_register(NULL, odb_source_loose_reparent, loose); + loose_object_map_load(loose); + return loose; } diff --git a/repository.c b/repository.c index 2ef0778846bcf1..6d633002b4e3c4 100644 --- a/repository.c +++ b/repository.c @@ -201,8 +201,6 @@ void repo_set_compat_hash_algo(struct repository *repo MAYBE_UNUSED, uint32_t al if (hash_algo_by_ptr(repo->hash_algo) == algo) BUG("hash_algo and compat_hash_algo match"); repo->compat_hash_algo = algo ? &hash_algos[algo] : NULL; - if (repo->compat_hash_algo) - repo_read_loose_object_map(repo); #else if (algo) die(_("compatibility hash algorithm support requires Rust")); diff --git a/setup.c b/setup.c index d31808130b47fc..825572f5f1ad06 100644 --- a/setup.c +++ b/setup.c @@ -1788,8 +1788,6 @@ int apply_repository_format(struct repository *repo, repo->bare_cfg = format->is_bare; repo_set_hash_algo(repo, format->hash_algo); - repo->objects = odb_new(repo, object_directory, - alternate_object_directories); repo_set_compat_hash_algo(repo, format->compat_hash_algo); repo_set_ref_storage_format(repo, format->ref_storage_format, @@ -1805,6 +1803,9 @@ int apply_repository_format(struct repository *repo, repo->repository_format_precious_objects = format->precious_objects; + repo->objects = odb_new(repo, object_directory, + alternate_object_directories); + free(alternate_object_directories); free(object_directory); return 0; From 30bc6f0e8c2aef5f9280468fa2ca7c170209603f Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Aug 2026 05:34:27 +0200 Subject: [PATCH 080/259] setup: handle ODB-related environment variables in `odb_new()` When initializing a repository's object database we have to respect the GIT_OBJECT_DIRECTORY and GIT_ALTERNATE_OBJECT_DIRECTORIES environment variables, which can be set by the user to override the default location of where we write objects to and read objects from. This is handled in `apply_repository_format()`, which is fine. But in a subsequent commit we'll have to defer constructing the object database to a later point in some cases, and that will require a second site where we call `odb_new()`. And of course, that second site would have to handle those environment variables, as well. It would be somewhat awkward to duplicate the logic though. But there's a better alternative: instead of handling this logic in "setup.c", we can easily handle environment variables in `odb_new()` itself. This ensures that object database creation is neatly self-contained, and we don't have to duplicate any of the logic. Another benefit is that in a future patch series we plan to move handling of alternates into the backends themselves [1], and that will require us to also handle those environment variables in the "files" backend itself. So moving the logic into the ODB level already gets us one step closer to that goal. Refactor the logic accordingly. [1]: https://lore.kernel.org/git/amLgMqkqxR8mKIbT@pks.im/ Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb.c | 21 ++++++++++++--------- odb.h | 17 +++++++++++++++-- setup.c | 11 ++++------- t/unit-tests/u-odb-inmemory.c | 2 +- 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/odb.c b/odb.c index cf6e7938c01e56..ed1d63f4bd3e9e 100644 --- a/odb.c +++ b/odb.c @@ -1004,26 +1004,29 @@ int odb_write_object_stream(struct object_database *odb, } struct object_database *odb_new(struct repository *repo, - const char *primary_source, - const char *secondary_sources) + enum odb_new_flags flags) { - struct object_database *o = xmalloc(sizeof(*o)); - char *to_free = NULL; + char *primary_source = NULL, *secondary_sources = NULL; + struct object_database *o; - memset(o, 0, sizeof(*o)); + CALLOC_ARRAY(o, 1); o->repo = repo; pthread_mutex_init(&o->replace_mutex, NULL); string_list_init_dup(&o->submodule_source_paths); + if (flags & ODB_NEW_HONOR_ENV) { + primary_source = xstrdup_or_null(getenv(DB_ENVIRONMENT)); + secondary_sources = xstrdup_or_null(getenv(ALTERNATE_DB_ENVIRONMENT)); + } if (!primary_source) - primary_source = to_free = xstrfmt("%s/objects", repo->commondir); + primary_source = xstrfmt("%s/objects", repo->commondir); + o->sources = odb_source_new(o, primary_source, true); o->sources_tail = &o->sources->next; - o->alternate_db = xstrdup_or_null(secondary_sources); + o->alternate_db = secondary_sources; o->inmemory_objects = &odb_source_inmemory_new(o)->base; - free(to_free); - + free(primary_source); return o; } diff --git a/odb.h b/odb.h index 7995bed97bc36e..8ec335c7f75a72 100644 --- a/odb.h +++ b/odb.h @@ -100,6 +100,20 @@ struct object_database { struct string_list submodule_source_paths; }; +enum odb_new_flags { + /* + * Honor environment variables when constructing the object database + * sources. This makes us respect the following environment variables: + * + * - GIT_OBJECT_DIRECTORY to override the primary object directory. + * + * - GIT_ALTERNATE_OBJECT_DIRECTORIES to override alternates. + * + * Environment variables may be backend-specific. + */ + ODB_NEW_HONOR_ENV = (1 << 0), +}; + /* * Create a new object database for the given repository. * @@ -112,8 +126,7 @@ struct object_database { * Returns the newly created object database. */ struct object_database *odb_new(struct repository *repo, - const char *primary_source, - const char *alternate_sources); + enum odb_new_flags flags); /* Free the object database and release all resources. */ void odb_free(struct object_database *o); diff --git a/setup.c b/setup.c index 825572f5f1ad06..5dfab3e79e54ba 100644 --- a/setup.c +++ b/setup.c @@ -1765,7 +1765,7 @@ int apply_repository_format(struct repository *repo, enum apply_repository_format_flags flags, struct strbuf *err) { - char *object_directory = NULL, *alternate_object_directories = NULL; + enum odb_new_flags odb_new_flags = 0; if (verify_repository_format(format, err) < 0) return -1; @@ -1779,8 +1779,6 @@ int apply_repository_format(struct repository *repo, if (flags & APPLY_REPOSITORY_FORMAT_HONOR_ENV) { const char *shallow_file; - object_directory = xstrdup_or_null(getenv(DB_ENVIRONMENT)); - alternate_object_directories = xstrdup_or_null(getenv(ALTERNATE_DB_ENVIRONMENT)); shallow_file = getenv(GIT_SHALLOW_FILE_ENVIRONMENT); if (shallow_file) set_alternate_shallow_file(repo, shallow_file); @@ -1803,11 +1801,10 @@ int apply_repository_format(struct repository *repo, repo->repository_format_precious_objects = format->precious_objects; - repo->objects = odb_new(repo, object_directory, - alternate_object_directories); + if (flags & APPLY_REPOSITORY_FORMAT_HONOR_ENV) + odb_new_flags |= ODB_NEW_HONOR_ENV; + repo->objects = odb_new(repo, odb_new_flags); - free(alternate_object_directories); - free(object_directory); return 0; } diff --git a/t/unit-tests/u-odb-inmemory.c b/t/unit-tests/u-odb-inmemory.c index 6844bfc37ccfdc..db323e10fd4b2c 100644 --- a/t/unit-tests/u-odb-inmemory.c +++ b/t/unit-tests/u-odb-inmemory.c @@ -38,7 +38,7 @@ static void cl_assert_object_info(struct odb_source_inmemory *source, void test_odb_inmemory__initialize(void) { - odb = odb_new(&repo, "", ""); + odb = odb_new(&repo, 0); } void test_odb_inmemory__cleanup(void) From c1d233bd3001530042ff097f6aef0a658b7f79cb Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Aug 2026 05:34:28 +0200 Subject: [PATCH 081/259] setup: defer object database creation In a subsequent commit we'll make the creation of the on-disk data structures of an object database pluggable. This will lead to an in-between state where we have already configured the repository's object database, but it's not usable yet until we eventually call `create_object_directory()`. Lift the call to `odb_new()` out of `apply_repository_format()` so that callers have more wiggle room with when exactly they call it, and adapt them accordingly. The only exception is `init_db()`, where we now defer creating the object database until we call `create_object_database()`. With this change, initializing and creating the object database on disk is now neatly encapsulated in a single function, which will make it easier for a subsequent commit to move creation of the on-disk data structures into the `struct odb_source` backends. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- repository.c | 1 + setup.c | 17 ++++++++--------- setup.h | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/repository.c b/repository.c index 6d633002b4e3c4..5ec264e6072865 100644 --- a/repository.c +++ b/repository.c @@ -294,6 +294,7 @@ int repo_init(struct repository *repo, warning("%s", err.buf); goto error; } + repo->objects = odb_new(repo, 0); if (worktree) repo_set_worktree(repo, worktree); diff --git a/setup.c b/setup.c index 5dfab3e79e54ba..97338cbc51fba9 100644 --- a/setup.c +++ b/setup.c @@ -1765,8 +1765,6 @@ int apply_repository_format(struct repository *repo, enum apply_repository_format_flags flags, struct strbuf *err) { - enum odb_new_flags odb_new_flags = 0; - if (verify_repository_format(format, err) < 0) return -1; @@ -1801,10 +1799,6 @@ int apply_repository_format(struct repository *repo, repo->repository_format_precious_objects = format->precious_objects; - if (flags & APPLY_REPOSITORY_FORMAT_HONOR_ENV) - odb_new_flags |= ODB_NEW_HONOR_ENV; - repo->objects = odb_new(repo, odb_new_flags); - return 0; } @@ -1888,6 +1882,7 @@ const char *enter_repo(struct repository *repo, const char *path, unsigned flags read_and_verify_repository_format(&fmt, ".", NULL); if (apply_repository_format(repo, &fmt, APPLY_REPOSITORY_FORMAT_HONOR_ENV, &err) < 0) die("%s", err.buf); + repo->objects = odb_new(repo, ODB_NEW_HONOR_ENV); startup_info->have_repository = 1; clear_repository_format(&fmt); @@ -2090,6 +2085,7 @@ const char *setup_git_directory_gently(struct repository *repo, int *nongit_ok) if (apply_repository_format(repo, &discovery.format, APPLY_REPOSITORY_FORMAT_HONOR_ENV, &err) < 0) die("%s", err.buf); + repo->objects = odb_new(repo, ODB_NEW_HONOR_ENV); clear_repository_format(&discovery.format); strbuf_release(&err); @@ -2651,11 +2647,13 @@ static int create_default_files(struct repository *repo, return reinit; } -static void create_object_directory(struct repository *repo) +static void create_object_database(struct repository *repo) { struct strbuf path = STRBUF_INIT; size_t baselen; + repo->objects = odb_new(repo, ODB_NEW_HONOR_ENV); + strbuf_addstr(&path, repo_get_object_directory(repo)); baselen = path.len; @@ -2866,7 +2864,6 @@ int init_db(struct repository *repo, repository_format_configure(&repo_fmt, hash, ref_storage_format); if (apply_repository_format(repo, &repo_fmt, APPLY_REPOSITORY_FORMAT_HONOR_ENV, &err) < 0) die("%s", err.buf); - startup_info->have_repository = 1; /* * Ensure `core.hidedotfiles` is processed. This must happen after we @@ -2882,7 +2879,9 @@ int init_db(struct repository *repo, if (!(flags & INIT_DB_SKIP_REFDB)) create_reference_database(repo, initial_branch, flags & INIT_DB_QUIET); - create_object_directory(repo); + create_object_database(repo); + + startup_info->have_repository = 1; if (repo_settings_get_shared_repository(repo)) { char buf[10]; diff --git a/setup.h b/setup.h index 654f10e059b995..763fd384e86c28 100644 --- a/setup.h +++ b/setup.h @@ -245,8 +245,8 @@ enum apply_repository_format_flags { /* * Apply the given repository format to the repo. This initializes extensions - * and basic data structures required for normal operation. Returns 0 on - * success, a negative error code when the format is not valid as determined by + * required for normal operation. Returns 0 on success, a negative error code + * when the format is not valid as determined by * `verify_repository_format()`. */ int apply_repository_format(struct repository *repo, From 335fe2545e4d64b79fc28acc945bc3278739d078 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Aug 2026 05:34:29 +0200 Subject: [PATCH 082/259] odb/source: introduce function to map source type to name Introduce a new function that maps an object source's type to a human-readable name. Use the function to provide better human-readable error messages for the downcasting functions. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb/source-files.h | 4 +++- odb/source-inmemory.h | 4 +++- odb/source-loose.h | 4 +++- odb/source-packed.h | 4 +++- odb/source.c | 19 +++++++++++++++++++ odb/source.h | 6 ++++++ 6 files changed, 37 insertions(+), 4 deletions(-) diff --git a/odb/source-files.h b/odb/source-files.h index d7ac3c1c81d892..6a803afdda3e86 100644 --- a/odb/source-files.h +++ b/odb/source-files.h @@ -28,7 +28,9 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, static inline struct odb_source_files *odb_source_files_downcast(struct odb_source *source) { if (source->type != ODB_SOURCE_FILES) - BUG("trying to downcast source of type '%d' to files", source->type); + BUG("trying to downcast source of type '%s' to '%s'", + odb_source_type_to_name(source->type), + odb_source_type_to_name(ODB_SOURCE_FILES)); return container_of(source, struct odb_source_files, base); } diff --git a/odb/source-inmemory.h b/odb/source-inmemory.h index a88fc2e320ed5c..adbad23e8b26af 100644 --- a/odb/source-inmemory.h +++ b/odb/source-inmemory.h @@ -26,7 +26,9 @@ struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb) static inline struct odb_source_inmemory *odb_source_inmemory_downcast(struct odb_source *source) { if (source->type != ODB_SOURCE_INMEMORY) - BUG("trying to downcast source of type '%d' to in-memory", source->type); + BUG("trying to downcast source of type '%s' to '%s'", + odb_source_type_to_name(source->type), + odb_source_type_to_name(ODB_SOURCE_INMEMORY)); return container_of(source, struct odb_source_inmemory, base); } diff --git a/odb/source-loose.h b/odb/source-loose.h index 6070aaf3ce6ab2..3cf2e1f8f1d5e2 100644 --- a/odb/source-loose.h +++ b/odb/source-loose.h @@ -41,7 +41,9 @@ struct odb_source_loose *odb_source_loose_new(struct object_database *odb, static inline struct odb_source_loose *odb_source_loose_downcast(struct odb_source *source) { if (source->type != ODB_SOURCE_LOOSE) - BUG("trying to downcast source of type '%d' to loose", source->type); + BUG("trying to downcast source of type '%s' to '%s'", + odb_source_type_to_name(source->type), + odb_source_type_to_name(ODB_SOURCE_LOOSE)); return container_of(source, struct odb_source_loose, base); } diff --git a/odb/source-packed.h b/odb/source-packed.h index 77309ddd0932b6..a0f6b5096dcd0f 100644 --- a/odb/source-packed.h +++ b/odb/source-packed.h @@ -78,7 +78,9 @@ struct odb_source_packed *odb_source_packed_new(struct object_database *odb, static inline struct odb_source_packed *odb_source_packed_downcast(struct odb_source *source) { if (source->type != ODB_SOURCE_PACKED) - BUG("trying to downcast source of type '%d' to packed", source->type); + BUG("trying to downcast source of type '%s' to '%s'", + odb_source_type_to_name(source->type), + odb_source_type_to_name(ODB_SOURCE_PACKED)); return container_of(source, struct odb_source_packed, base); } diff --git a/odb/source.c b/odb/source.c index 7993dcbd659399..30188b806d524d 100644 --- a/odb/source.c +++ b/odb/source.c @@ -4,6 +4,25 @@ #include "odb/source.h" #include "packfile.h" +static const char * const odb_source_names_by_type[] = { + [ODB_SOURCE_UNKNOWN] = "unknown", + [ODB_SOURCE_FILES] = "files", + [ODB_SOURCE_LOOSE] = "loose", + [ODB_SOURCE_PACKED] = "packed", + [ODB_SOURCE_INMEMORY] = "in-memory", +}; + +const char *odb_source_type_to_name(enum odb_source_type type) +{ + const char *name; + if (type < 0 || type >= ARRAY_SIZE(odb_source_names_by_type)) + type = ODB_SOURCE_UNKNOWN; + name = odb_source_names_by_type[type]; + if (!name) + BUG("name missing in `odb_source_names_by_type` for '%d'", type); + return name; +} + struct odb_source *odb_source_new(struct object_database *odb, const char *path, bool local) diff --git a/odb/source.h b/odb/source.h index cd63dba91f4e2f..ab16d152f43082 100644 --- a/odb/source.h +++ b/odb/source.h @@ -25,6 +25,12 @@ enum odb_source_type { ODB_SOURCE_INMEMORY, }; +/* + * Convert between the enum and its name. Returns the equivalent of "unknown" + * for unknown types. + */ +const char *odb_source_type_to_name(enum odb_source_type type); + struct object_id; struct odb_read_stream; struct strvec; From e927cfeb21d6a217b708216862deb36f144f064b Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Aug 2026 05:34:30 +0200 Subject: [PATCH 083/259] odb: make creation of on-disk structures pluggable When creating a new "files" object database source we have to create a couple of directories. These directories are of course specific to this particular backend, and a different backend may require a setup that is completely different. Make the creation of on-disk structures pluggable to accommodate for this. Note that there is one exception though: the "objects" directory must exist in a repository regardless of which backend is in use. If it doesn't exist then the repository is not treated as a Git repository at all. Consequently, we create this directory regardless of the backend. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb/source-files.c | 19 +++++++++++++++++++ odb/source.h | 23 +++++++++++++++++++++++ setup.c | 34 ++++++++++++++++++---------------- 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/odb/source-files.c b/odb/source-files.c index 413875851135a5..0db6e681fee9e6 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -9,6 +9,7 @@ #include "odb/source-files.h" #include "odb/source-loose.h" #include "packfile.h" +#include "path.h" #include "strbuf.h" #include "write-or-die.h" @@ -41,6 +42,23 @@ static void odb_source_files_close(struct odb_source *source) odb_source_close(&files->packed->base); } +static int odb_source_files_create_on_disk(struct odb_source *source) +{ + struct strbuf path = STRBUF_INIT; + + safe_create_dir(source->odb->repo, source->path, 1); + + strbuf_addf(&path, "%s/pack", source->path); + safe_create_dir(source->odb->repo, path.buf, 1); + + strbuf_reset(&path); + strbuf_addf(&path, "%s/info", source->path); + safe_create_dir(source->odb->repo, path.buf, 1); + + strbuf_release(&path); + return 0; +} + static void odb_source_files_prepare(struct odb_source *source, enum odb_prepare_flags flags) { @@ -271,6 +289,7 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, files->base.free = odb_source_files_free; files->base.close = odb_source_files_close; + files->base.create_on_disk = odb_source_files_create_on_disk; files->base.prepare = odb_source_files_prepare; files->base.read_object_info = odb_source_files_read_object_info; files->base.read_object_stream = odb_source_files_read_object_stream; diff --git a/odb/source.h b/odb/source.h index ab16d152f43082..4abc418bdd70fc 100644 --- a/odb/source.h +++ b/odb/source.h @@ -89,6 +89,18 @@ struct odb_source { */ void (*close)(struct odb_source *source); + /* + * This callback is expected to create on-disk data structures that are + * required for this source to operate. + * + * The callback is expected to return 0 on success, a negative error + * code otherwise. + * + * This callback may be NULL in case the source does not need any + * on-disk setup. + */ + int (*create_on_disk)(struct odb_source *source); + /* * This callback is expected to prepare the source so that it becomes * ready for use. It optionally clears underlying caches of the object @@ -316,6 +328,17 @@ static inline void odb_source_close(struct odb_source *source) source->close(source); } +/* + * Create on-disk data structures that are required for this source to operate + * correctly. Returns 0 on success, a negative error code otherwise. + */ +static inline int odb_source_create_on_disk(struct odb_source *source) +{ + if (!source->create_on_disk) + return 0; + return source->create_on_disk(source); +} + /* * Prepare the object database source and clear any caches. Depending on the * backend used this may have the effect that concurrently-written objects diff --git a/setup.c b/setup.c index 97338cbc51fba9..ace3c59d184626 100644 --- a/setup.c +++ b/setup.c @@ -2649,25 +2649,27 @@ static int create_default_files(struct repository *repo, static void create_object_database(struct repository *repo) { - struct strbuf path = STRBUF_INIT; - size_t baselen; + /* + * Create the "objects" directory in the common directory. This is done + * so that the repository can be discovered regardless of the backend + * used. + * + * Note that we only do this in case the object directory wasn't + * overwritten via an environment variable. If it _is_ being overridden + * then we skip this step, as the repository won't be discoverable + * anyway without the environment variable. + */ + if (!getenv(DB_ENVIRONMENT)) { + struct strbuf objects_dir = STRBUF_INIT; + repo_common_path_append(repo, &objects_dir, "objects"); + safe_create_dir(repo, objects_dir.buf, 1); + strbuf_release(&objects_dir); + } repo->objects = odb_new(repo, ODB_NEW_HONOR_ENV); - strbuf_addstr(&path, repo_get_object_directory(repo)); - baselen = path.len; - - safe_create_dir(repo, path.buf, 1); - - strbuf_setlen(&path, baselen); - strbuf_addstr(&path, "/pack"); - safe_create_dir(repo, path.buf, 1); - - strbuf_setlen(&path, baselen); - strbuf_addstr(&path, "/info"); - safe_create_dir(repo, path.buf, 1); - - strbuf_release(&path); + if (odb_source_create_on_disk(repo->objects->sources) < 0) + die(_("failed creating object database")); } static void separate_git_dir(const char *git_dir, const char *git_link) From 8b0ab33247e7ac86f2cecd144991301b6fe6a55b Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Aug 2026 08:18:03 +0200 Subject: [PATCH 084/259] compat/posix: introduce writev(3p) wrapper In a subsequent commit we're going to add the first caller to writev(3p). Introduce a compatibility wrapper for this syscall that we can use on systems that don't have this syscall. The syscall exists on modern Unixes like Linux and macOS, and seemingly even for NonStop according to [1]. It doesn't seem to exist on Windows though. [1]: http://nonstoptools.com/manuals/OSS-SystemCalls.pdf [2]: https://www.gnu.org/software/gnulib/manual/html_node/writev.html Helped-by: Johannes Schindelin Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- Makefile | 4 +++ compat/posix.h | 14 ++++++++++ compat/writev.c | 41 +++++++++++++++++++++++++++++ config.mak.uname | 2 ++ contrib/buildsystems/CMakeLists.txt | 6 ++++- meson.build | 1 + 6 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 compat/writev.c diff --git a/Makefile b/Makefile index 1f3f099f5c5705..eda5ecc5b4ad32 100644 --- a/Makefile +++ b/Makefile @@ -2033,6 +2033,10 @@ ifdef NO_PREAD COMPAT_CFLAGS += -DNO_PREAD COMPAT_OBJS += compat/pread.o endif +ifdef NO_WRITEV + COMPAT_CFLAGS += -DNO_WRITEV + COMPAT_OBJS += compat/writev.o +endif ifdef NO_FAST_WORKING_DIRECTORY BASIC_CFLAGS += -DNO_FAST_WORKING_DIRECTORY endif diff --git a/compat/posix.h b/compat/posix.h index e2e794cad7d419..71cc7316204187 100644 --- a/compat/posix.h +++ b/compat/posix.h @@ -148,6 +148,9 @@ #include #include #include +#ifndef NO_WRITEV +#include +#endif #include #ifndef NO_SYS_SELECT_H #include @@ -334,6 +337,17 @@ int git_lstat(const char *, struct stat *); ssize_t git_pread(int fd, void *buf, size_t count, off_t offset); #endif +#ifdef NO_WRITEV +#define writev git_writev +#define iovec git_iovec +struct git_iovec { + void *iov_base; + size_t iov_len; +}; + +ssize_t git_writev(int fd, const struct iovec *iov, int iovcnt); +#endif + #ifdef NO_SETENV #define setenv gitsetenv int gitsetenv(const char *, const char *, int); diff --git a/compat/writev.c b/compat/writev.c new file mode 100644 index 00000000000000..540f66de61fb09 --- /dev/null +++ b/compat/writev.c @@ -0,0 +1,41 @@ +#include "../git-compat-util.h" +#include "../wrapper.h" + +ssize_t git_writev(int fd, const struct iovec *iov, int iovcnt) +{ + size_t sum = 0; + + if (iovcnt <= 0) { + errno = EINVAL; + return -1; + } + + /* + * According to writev(3p), the syscall shall error with EINVAL in case + * the sum of `iov_len` overflows `ssize_t`. + */ + for (int i = 0; i < iovcnt; i++) { + if (iov[i].iov_len > maximum_signed_value_of_type(ssize_t) || + unsigned_add_overflows(iov[i].iov_len, sum) || + iov[i].iov_len + sum > maximum_signed_value_of_type(ssize_t)) { + errno = EINVAL; + return -1; + } + + sum += iov[i].iov_len; + } + + /* + * We only ever write the first non-empty vector so that we can + * guarantee the call to be non-interleaving as guaranteed by POSIX. + * This works just fine as callers have to loop around writev anyway. + */ + for (int i = 0; i < iovcnt; i++) { + if (!iov[i].iov_len) + continue; + return xwrite(fd, iov[i].iov_base, iov[i].iov_len); + } + + /* When all iovec members were zero we ought to return 0 according to POSIX. */ + return 0; +} diff --git a/config.mak.uname b/config.mak.uname index 9ebd240378ca59..95ef6e64dcabff 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -483,6 +483,7 @@ ifeq ($(uname_S),Windows) SANE_TOOL_PATH ?= $(msvc_bin_dir_msys) HAVE_ALLOCA_H = YesPlease NO_PREAD = YesPlease + NO_WRITEV = YesPlease NEEDS_CRYPTO_WITH_SSL = YesPlease NO_LIBGEN_H = YesPlease NO_POLL = YesPlease @@ -697,6 +698,7 @@ ifeq ($(uname_S),MINGW) pathsep = ; HAVE_ALLOCA_H = YesPlease NO_PREAD = YesPlease + NO_WRITEV = YesPlease NEEDS_CRYPTO_WITH_SSL = YesPlease NO_LIBGEN_H = YesPlease NO_POLL = YesPlease diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index a57c4b464fa456..8f56203f34d9bc 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -378,7 +378,7 @@ endif() #function checks set(function_checks strcasestr memmem strlcpy strtoimax strtoumax strtoull - setenv mkdtemp poll pread memmem) + setenv mkdtemp poll pread memmem writev) #unsetenv,hstrerror are incompatible with windows build if(NOT WIN32) @@ -423,6 +423,10 @@ if(NOT HAVE_MEMMEM) list(APPEND compat_SOURCES compat/memmem.c) endif() +if(NOT HAVE_WRITEV) + list(APPEND compat_SOURCES compat/writev.c) +endif() + if(NOT WIN32) if(NOT HAVE_UNSETENV) list(APPEND compat_SOURCES compat/unsetenv.c) diff --git a/meson.build b/meson.build index 9434b56960ba80..43373924aa79c3 100644 --- a/meson.build +++ b/meson.build @@ -1448,6 +1448,7 @@ checkfuncs = { 'initgroups' : [], 'strtoumax' : ['strtoumax.c', 'strtoimax.c'], 'pread' : ['pread.c'], + 'writev' : ['writev.c'], } if host_machine.system() == 'windows' From d70eb7f3600db5fabb538ce35b186db68854b2a8 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Aug 2026 08:18:04 +0200 Subject: [PATCH 085/259] wrapper: introduce writev(3p) wrappers In the preceding commit we have added a compatibility wrapper for the writev(3p) syscall. Introduce some generic wrappers for this function that we nowadays take for granted in the Git codebase. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- wrapper.c | 41 +++++++++++++++++++++++++++++++++++++++++ wrapper.h | 9 +++++++++ write-or-die.c | 8 ++++++++ write-or-die.h | 1 + 4 files changed, 59 insertions(+) diff --git a/wrapper.c b/wrapper.c index 16f5a63fbb614a..be8fa575e6f425 100644 --- a/wrapper.c +++ b/wrapper.c @@ -323,6 +323,47 @@ ssize_t write_in_full(int fd, const void *buf, size_t count) return total; } +ssize_t writev_in_full(int fd, struct iovec *iov, int iovcnt) +{ + ssize_t total_written = 0; + + while (iovcnt) { + ssize_t bytes_written = writev(fd, iov, iovcnt); + if (bytes_written < 0) { + if (errno == EINTR || errno == EAGAIN) + continue; + return -1; + } + if (!bytes_written) { + errno = ENOSPC; + return -1; + } + + total_written += bytes_written; + + /* + * We first need to discard any iovec entities that have been + * fully written. + */ + while (iovcnt && (size_t)bytes_written >= iov->iov_len) { + bytes_written -= iov->iov_len; + iov++; + iovcnt--; + } + + /* + * Finally, we need to adjust the last iovec in case we have + * performed a partial write. + */ + if (iovcnt && bytes_written) { + iov->iov_base = (char *) iov->iov_base + bytes_written; + iov->iov_len -= bytes_written; + } + } + + return total_written; +} + ssize_t pread_in_full(int fd, void *buf, size_t count, off_t offset) { char *p = buf; diff --git a/wrapper.h b/wrapper.h index 15ac3bab6e9748..27519b32d1782d 100644 --- a/wrapper.h +++ b/wrapper.h @@ -47,6 +47,15 @@ ssize_t read_in_full(int fd, void *buf, size_t count); ssize_t write_in_full(int fd, const void *buf, size_t count); ssize_t pread_in_full(int fd, void *buf, size_t count, off_t offset); +/* + * Try to write all iovecs. Returns -1 in case an error occurred with a proper + * errno set, the number of bytes written otherwise. + * + * Note that the iovec will be modified as a result of this call to adjust for + * partial writes! + */ +ssize_t writev_in_full(int fd, struct iovec *iov, int iovcnt); + static inline ssize_t write_str_in_full(int fd, const char *str) { return write_in_full(fd, str, strlen(str)); diff --git a/write-or-die.c b/write-or-die.c index 01a9a51fa2fcd7..5f522fb7287382 100644 --- a/write-or-die.c +++ b/write-or-die.c @@ -96,6 +96,14 @@ void write_or_die(int fd, const void *buf, size_t count) } } +void writev_or_die(int fd, struct iovec *iov, int iovlen) +{ + if (writev_in_full(fd, iov, iovlen) < 0) { + check_pipe(errno); + die_errno("writev error"); + } +} + void fwrite_or_die(FILE *f, const void *buf, size_t count) { if (fwrite(buf, 1, count, f) != count) diff --git a/write-or-die.h b/write-or-die.h index ff0408bd849fd8..a045bdfaef1b2e 100644 --- a/write-or-die.h +++ b/write-or-die.h @@ -7,6 +7,7 @@ void fprintf_or_die(FILE *, const char *fmt, ...); void fwrite_or_die(FILE *f, const void *buf, size_t count); void fflush_or_die(FILE *f); void write_or_die(int fd, const void *buf, size_t count); +void writev_or_die(int fd, struct iovec *iov, int iovlen); /* * These values are used to help identify parts of a repository to fsync. From a4e2c0fc81198fa84c1107daa0a33de6cc6d9c3a Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Aug 2026 08:18:05 +0200 Subject: [PATCH 086/259] wrapper: properly handle MAX_IO_SIZE in writev(3p) Some systems like NonStop set a comparatively small `MAX_IO_SIZE`, which limits the maximum number of bytes we're allowed to write in a single call. We already handle this limit properly in `xwrite()`, but we have recently introduced wrappers for writev(3p) where we don't. This will cause the syscall to return EINVAL in case somebody passes an iovec entry to writev(3p) that is larger than `MAX_IO_SIZE`. Introduce a new function `xwritev()` that is similar to `xwrite()` in that it handles such platform-specific nuances: - We only pass the leading iovec entries to writev(3p) that fit into `MAX_IO_SIZE`, pretending that the underlying syscall performed a short write. This mirrors how `xwrite()` chomps overly large requests before handing them to write(3p). As a consequence, callers will never see writev(3p)'s EINVAL error for requests whose summed length would overflow an ssize_t, but observe a short write instead. - If already the first iovec entry exceeds the limit we instead punt to `xwrite()`, which knows to handle this case for us. - We restart the underlying syscall on EINTR and EAGAIN, just like `xwrite()` does for write(3p). Adapt `writev_in_full()` to use this new wrapper. With the retry logic now living in `xwritev()`, the calling loop becomes the exact mirror image of `write_in_full()`, which also retains the responsibility of translating a zero-length write into ENOSPC. Reported-by: Randall Becker Helped-by: Jeff King Helped-by: Junio C Hamano Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- wrapper.c | 47 ++++++++++++++++++++++++++++++++++++++++++----- wrapper.h | 1 + 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/wrapper.c b/wrapper.c index be8fa575e6f425..561f9ee9c99fc1 100644 --- a/wrapper.c +++ b/wrapper.c @@ -323,17 +323,54 @@ ssize_t write_in_full(int fd, const void *buf, size_t count) return total; } +ssize_t xwritev(int fd, struct iovec *iov, int iovcnt) +{ + size_t allowed = MAX_IO_SIZE; + int i; + + /* + * Some platforms define a comparatively small `MAX_IO_SIZE` that + * limits how many bytes can be written with a single call to + * write(3p) or writev(3p); exceeding that limit causes the syscall to + * fail with EINVAL. Just like xwrite() chomps overly large requests + * for write(3p), pretend that the underlying writev(3p) performed a + * short write by only passing along the leading iovec entries that + * fit into that limit. + */ + for (i = 0; i < iovcnt; i++) { + if (iov[i].iov_len > allowed) { + /* + * If the first buffer is larger than MAX_IO_SIZE, + * let xwrite() deal with it. + */ + if (!i) + return xwrite(fd, iov->iov_base, iov->iov_len); + break; + } + allowed -= iov[i].iov_len; + } + + while (1) { + ssize_t bytes_written = writev(fd, iov, i); + if (bytes_written < 0) { + if (errno == EINTR) + continue; + if (handle_nonblock(fd, POLLOUT, errno)) + continue; + } + + return bytes_written; + } +} + ssize_t writev_in_full(int fd, struct iovec *iov, int iovcnt) { ssize_t total_written = 0; while (iovcnt) { - ssize_t bytes_written = writev(fd, iov, iovcnt); - if (bytes_written < 0) { - if (errno == EINTR || errno == EAGAIN) - continue; + ssize_t bytes_written = xwritev(fd, iov, iovcnt); + if (bytes_written < 0) return -1; - } if (!bytes_written) { errno = ENOSPC; return -1; diff --git a/wrapper.h b/wrapper.h index 27519b32d1782d..a6287d7f4d11be 100644 --- a/wrapper.h +++ b/wrapper.h @@ -16,6 +16,7 @@ void *xmmap_gently(void *start, size_t length, int prot, int flags, int fd, off_ int xopen(const char *path, int flags, ...); ssize_t xread(int fd, void *buf, size_t len); ssize_t xwrite(int fd, const void *buf, size_t len); +ssize_t xwritev(int fd, struct iovec *iov, int iovcnt); ssize_t xpread(int fd, void *buf, size_t len, off_t offset); int xdup(int fd); FILE *xfopen(const char *path, const char *mode); From 21db416cd2bf658ce79fc928c65b86e981062e8e Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Aug 2026 08:18:06 +0200 Subject: [PATCH 087/259] sideband: use writev(3p) to send pktlines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every pktline that we send out via `send_sideband()` currently requires two syscalls: one to write the pktline's length, and one to send its data. This typically isn't all that much of a problem, but under extreme load the syscalls may cause contention in the kernel. Refactor the code to instead use the newly introduced writev(3p) infra so that we can send out the data with a single syscall. This reduces the number of syscalls from around 133,000 calls to write(3p) to around 67,000 calls to writev(3p). This change leads to a performance improvement for git-upload-pack(1), but we have to cheat a bit to really make it measurable. Usually, the time is strongly dominated by generating the packfile itself. But if we precompute the pack and serve it via the pack-objects hook then we can essentially eliminate that overhead. The following setup is executed in the Git repository: $ cat >request <<-EOF 0048want 5ce91c059e41090e7d2cffad39c04af8acf98dc1 side-band no-progress 00000009done EOF $ echo 5ce91c059e41090e7d2cffad39c04af8acf98dc1 | git pack-objects --revs --stdout >pack $ cat >hook <<-EOF #!/bin/sh cat >/dev/null cat "$(pwd)"/pack EOF $ chmod u+x hook $ git -c uploadpack.packObjectsHook="$(pwd)"/hook upload-pack . Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- sideband.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sideband.c b/sideband.c index 1523a53e1d781d..94e5b561728567 100644 --- a/sideband.c +++ b/sideband.c @@ -441,6 +441,7 @@ void send_sideband(int fd, int band, const char *data, ssize_t sz, int packet_ma const char *p = data; while (sz) { + struct iovec iov[2]; unsigned n; char hdr[5]; @@ -450,12 +451,19 @@ void send_sideband(int fd, int band, const char *data, ssize_t sz, int packet_ma if (0 <= band) { xsnprintf(hdr, sizeof(hdr), "%04x", n + 5); hdr[4] = band; - write_or_die(fd, hdr, 5); + iov[0].iov_base = hdr; + iov[0].iov_len = 5; } else { xsnprintf(hdr, sizeof(hdr), "%04x", n + 4); - write_or_die(fd, hdr, 4); + iov[0].iov_base = hdr; + iov[0].iov_len = 4; } - write_or_die(fd, p, n); + + iov[1].iov_base = (void *) p; + iov[1].iov_len = n; + + writev_or_die(fd, iov, ARRAY_SIZE(iov)); + p += n; sz -= n; } From 5bd4f43456aae6fa942eb6c6ace6244d09e01d08 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 7 Aug 2026 08:18:07 +0200 Subject: [PATCH 088/259] fast-import: use writev(3p) to send cat-blob responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When answering a `cat-blob` command, `cat_blob()` issues three separate calls to write(3p) on the cat-blob fd: one for the header line, one for the full blob payload, and one for the trailing newline. Frontends like git-filter-repo issue these commands in bulk, once per rewritten blob, so the syscall overhead adds up. Use `writev_in_full()` to send all three parts with a single syscall. This can be benchmarked with the following setup: $ git cat-file --unordered --filter=object:type=blob --batch-check='cat-blob %(objectname)' --batch-all-objects >request $ git fast-import --cat-blob-fd=3 Signed-off-by: Junio C Hamano --- builtin/fast-import.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/builtin/fast-import.c b/builtin/fast-import.c index aa656c5195d366..48fda01c94359c 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -3332,6 +3332,7 @@ static void cat_blob_write(const char *buf, unsigned long size) static void cat_blob(struct object_entry *oe, struct object_id *oid) { struct strbuf line = STRBUF_INIT; + struct iovec iov[3]; unsigned long size; enum object_type type = 0; char *buf; @@ -3365,10 +3366,21 @@ static void cat_blob(struct object_entry *oe, struct object_id *oid) strbuf_reset(&line); strbuf_addf(&line, "%s %s %"PRIuMAX"\n", oid_to_hex(oid), type_name(type), (uintmax_t)size); - cat_blob_write(line.buf, line.len); + + /* + * Write the header, the payload and the trailing newline with a + * single writev(3p) call instead of three separate write(3p) calls. + */ + iov[0].iov_base = line.buf; + iov[0].iov_len = line.len; + iov[1].iov_base = buf; + iov[1].iov_len = size; + iov[2].iov_base = (void *) "\n"; + iov[2].iov_len = 1; + + if (writev_in_full(cat_blob_fd, iov, ARRAY_SIZE(iov)) < 0) + die_errno(_("write to frontend failed")); strbuf_release(&line); - cat_blob_write(buf, size); - cat_blob_write("\n", 1); if (oe && oe->pack_id == pack_id) { last_blob.offset = oe->idx.offset; strbuf_attach(&last_blob.data, buf, size, size + 1); From 52c95408aa6cf1d6f23b145959044bbc36e57c2f Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Fri, 7 Aug 2026 07:39:24 +0000 Subject: [PATCH 089/259] history: extract helper for a commit's parent tree Three places resolve the tree of a commit's first parent, falling back to the empty tree for a root commit, each repeating the same parse and oidcpy dance. Extract a first_parent_tree_oid() helper and route the existing callers through it. No change in behavior. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- builtin/history.c | 58 +++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/builtin/history.c b/builtin/history.c index d28c1f08bb66ea..673744a55a33f1 100644 --- a/builtin/history.c +++ b/builtin/history.c @@ -164,6 +164,25 @@ static int commit_tree_ext(struct repository *repo, return ret; } +static int first_parent_tree_oid(struct repository *repo, + struct commit *commit, + struct object_id *out) +{ + struct commit *parent = commit->parents ? commit->parents->item : NULL; + + if (!parent) { + oidcpy(out, repo->hash_algo->empty_tree); + return 0; + } + + if (repo_parse_commit(repo, parent)) + return error(_("unable to parse parent commit %s"), + oid_to_hex(&parent->object.oid)); + + oidcpy(out, &repo_get_commit_tree(repo, parent)->object.oid); + return 0; +} + static int commit_tree_with_edited_message(struct repository *repo, const char *action, struct commit *original, @@ -171,21 +190,11 @@ static int commit_tree_with_edited_message(struct repository *repo, { struct object_id parent_tree_oid; const struct object_id *tree_oid; - struct commit *parent; tree_oid = &repo_get_commit_tree(repo, original)->object.oid; - parent = original->parents ? original->parents->item : NULL; - if (parent) { - if (repo_parse_commit(repo, parent)) { - return error(_("unable to parse parent commit %s"), - oid_to_hex(&parent->object.oid)); - } - - parent_tree_oid = repo_get_commit_tree(repo, parent)->object.oid; - } else { - oidcpy(&parent_tree_oid, repo->hash_algo->empty_tree); - } + if (first_parent_tree_oid(repo, original, &parent_tree_oid) < 0) + return -1; return commit_tree_ext(repo, action, original, original->parents, &parent_tree_oid, tree_oid, out, COMMIT_TREE_EDIT_MESSAGE); @@ -475,18 +484,10 @@ static int commit_became_empty(struct repository *repo, struct commit *original, struct tree *result) { - struct commit *parent = original->parents ? original->parents->item : NULL; struct object_id parent_tree_oid; - if (parent) { - if (repo_parse_commit(repo, parent)) - return error(_("unable to parse parent of %s"), - oid_to_hex(&original->object.oid)); - - parent_tree_oid = repo_get_commit_tree(repo, parent)->object.oid; - } else { - oidcpy(&parent_tree_oid, repo->hash_algo->empty_tree); - } + if (first_parent_tree_oid(repo, original, &parent_tree_oid) < 0) + return -1; return oideq(&result->object.oid, &parent_tree_oid); } @@ -830,16 +831,9 @@ static int split_commit(struct repository *repo, struct tree *split_tree; int ret; - if (original->parents) { - if (repo_parse_commit(repo, original->parents->item)) { - ret = error(_("unable to parse parent commit %s"), - oid_to_hex(&original->parents->item->object.oid)); - goto out; - } - - parent_tree_oid = *get_commit_tree_oid(original->parents->item); - } else { - oidcpy(&parent_tree_oid, repo->hash_algo->empty_tree); + if (first_parent_tree_oid(repo, original, &parent_tree_oid) < 0) { + ret = -1; + goto out; } original_commit_tree_oid = get_commit_tree_oid(original); From 77a1eb51740fecafb8c6ea10ab03113492c12746 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Fri, 7 Aug 2026 07:39:25 +0000 Subject: [PATCH 090/259] history: give commit_tree_ext a message template commit_tree_ext() reuses the message of the commit it is handed. A caller that folds several commits together wants to seed the message from more than that single commit, so add an optional message_template parameter. When NULL, the behavior is unchanged. Pass NULL from the existing fixup and split callers. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- builtin/history.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/builtin/history.c b/builtin/history.c index 673744a55a33f1..b592b98393c640 100644 --- a/builtin/history.c +++ b/builtin/history.c @@ -108,6 +108,7 @@ enum commit_tree_flags { static int commit_tree_ext(struct repository *repo, const char *action, struct commit *commit_with_message, + const char *message_template, const struct commit_list *parents, const struct object_id *old_tree, const struct object_id *new_tree, @@ -137,13 +138,16 @@ static int commit_tree_ext(struct repository *repo, original_author = xmemdupz(ptr, len); find_commit_subject(original_message, &original_body); + if (!message_template) + message_template = original_body; + if (flags & COMMIT_TREE_EDIT_MESSAGE) { ret = fill_commit_message(repo, old_tree, new_tree, - original_body, action, &commit_message); + message_template, action, &commit_message); if (ret < 0) goto out; } else { - strbuf_addstr(&commit_message, original_body); + strbuf_addstr(&commit_message, message_template); } original_extra_headers = read_commit_extra_headers(commit_with_message, @@ -196,7 +200,7 @@ static int commit_tree_with_edited_message(struct repository *repo, if (first_parent_tree_oid(repo, original, &parent_tree_oid) < 0) return -1; - return commit_tree_ext(repo, action, original, original->parents, + return commit_tree_ext(repo, action, original, NULL, original->parents, &parent_tree_oid, tree_oid, out, COMMIT_TREE_EDIT_MESSAGE); } @@ -675,7 +679,7 @@ static int cmd_history_fixup(int argc, goto out; if (!skip_commit) { - ret = commit_tree_ext(repo, "fixup", original, original->parents, + ret = commit_tree_ext(repo, "fixup", original, NULL, original->parents, &original_tree->object.oid, &merge_result.tree->object.oid, &rewritten, flags); if (ret < 0) { @@ -886,7 +890,7 @@ static int split_commit(struct repository *repo, * The first commit is constructed from the split-out tree. The base * that shall be diffed against is the parent of the original commit. */ - ret = commit_tree_ext(repo, "split-out", original, original->parents, &parent_tree_oid, + ret = commit_tree_ext(repo, "split-out", original, NULL, original->parents, &parent_tree_oid, &split_tree->object.oid, &first_commit, COMMIT_TREE_EDIT_MESSAGE); if (ret < 0) { ret = error(_("failed writing first commit")); @@ -903,7 +907,7 @@ static int split_commit(struct repository *repo, old_tree_oid = &repo_get_commit_tree(repo, first_commit)->object.oid; new_tree_oid = &repo_get_commit_tree(repo, original)->object.oid; - ret = commit_tree_ext(repo, "split-out", original, parents, old_tree_oid, + ret = commit_tree_ext(repo, "split-out", original, NULL, parents, old_tree_oid, new_tree_oid, &second_commit, COMMIT_TREE_EDIT_MESSAGE); if (ret < 0) { ret = error(_("failed writing second commit")); From 4b594b150e5d99a2a068e118e596c8637d7d1ccb Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Fri, 7 Aug 2026 07:39:26 +0000 Subject: [PATCH 091/259] sequencer: share the squash message marker helpers and flags When "git rebase -i" squashes commits it builds an editor template with a "This is a combination of N commits." banner, a "This is the 1st/Nth commit message:" header above each kept message (or a "will be skipped" header for a dropped one), and a commented-out subject for any fixup!, squash! or amend! commit. The banner, the headers and the subject-commenting all live in static helpers in sequencer.c wired to the rebase state, so no other command can present a squash the same way. Pull the three pieces out into add_squash_combination_header(), add_squash_message_header() (which takes a flag for the "will be skipped" variant) and squash_subject_comment_len(), and use them from update_squash_messages() and append_squash_message(). Also move the todo_item_flags enum to the header, so a caller reading the output of todo_list_rearrange_squash() can tell an amend! (TODO_REPLACE_FIXUP_MSG) from a plain fixup!. A later change reuses all of this to give "git history squash" the same template. No change in behavior. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- sequencer.c | 70 +++++++++++++++++++++++++++++------------------------ sequencer.h | 30 +++++++++++++++++++++++ 2 files changed, 69 insertions(+), 31 deletions(-) diff --git a/sequencer.c b/sequencer.c index 1355a99a092268..3c704fd5ab1346 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1880,18 +1880,38 @@ static int is_pick_or_similar(enum todo_command command) } } -enum todo_item_flags { - TODO_EDIT_MERGE_MSG = (1 << 0), - TODO_REPLACE_FIXUP_MSG = (1 << 1), - TODO_EDIT_FIXUP_MSG = (1 << 2), -}; - static const char first_commit_msg_str[] = N_("This is the 1st commit message:"); static const char nth_commit_msg_fmt[] = N_("This is the commit message #%d:"); static const char skip_first_commit_msg_str[] = N_("The 1st commit message will be skipped:"); static const char skip_nth_commit_msg_fmt[] = N_("The commit message #%d will be skipped:"); static const char combined_commit_msg_fmt[] = N_("This is a combination of %d commits."); +void add_squash_combination_header(struct strbuf *buf, int n) +{ + strbuf_addf(buf, "%s ", comment_line_str); + strbuf_addf(buf, _(combined_commit_msg_fmt), n); +} + +void add_squash_message_header(struct strbuf *buf, int n, int skip) +{ + strbuf_addf(buf, "%s ", comment_line_str); + if (n == 1) + strbuf_addstr(buf, skip ? _(skip_first_commit_msg_str) : + _(first_commit_msg_str)); + else + strbuf_addf(buf, skip ? _(skip_nth_commit_msg_fmt) : + _(nth_commit_msg_fmt), n); +} + +size_t squash_subject_comment_len(const char *body, int squashing) +{ + if (starts_with(body, "amend!") || + (squashing && (starts_with(body, "squash!") || + starts_with(body, "fixup!")))) + return commit_subject_length(body); + return 0; +} + static int is_fixup_flag(enum todo_command command, unsigned flag) { return command == TODO_FIXUP && ((flag & TODO_REPLACE_FIXUP_MSG) || @@ -2005,20 +2025,13 @@ static int append_squash_message(struct strbuf *buf, const char *body, { struct replay_ctx *ctx = opts->ctx; const char *fixup_msg; - size_t commented_len = 0, fixup_off; - /* - * amend is non-interactive and not normally used with fixup! - * or squash! commits, so only comment out those subjects when - * squashing commit messages. - */ - if (starts_with(body, "amend!") || - ((command == TODO_SQUASH || seen_squash(ctx)) && - (starts_with(body, "squash!") || starts_with(body, "fixup!")))) - commented_len = commit_subject_length(body); + size_t commented_len, fixup_off; + + commented_len = squash_subject_comment_len(body, + command == TODO_SQUASH || seen_squash(ctx)); - strbuf_addf(buf, "\n%s ", comment_line_str); - strbuf_addf(buf, _(nth_commit_msg_fmt), - ++ctx->current_fixup_count + 1); + strbuf_addch(buf, '\n'); + add_squash_message_header(buf, ++ctx->current_fixup_count + 1, 0); strbuf_addstr(buf, "\n\n"); strbuf_add_commented_lines(buf, body, commented_len, comment_line_str); /* buf->buf may be reallocated so store an offset into the buffer */ @@ -2083,9 +2096,8 @@ static int update_squash_messages(struct repository *r, eol = !starts_with(buf.buf, comment_line_str) ? buf.buf : strchrnul(buf.buf, '\n'); - strbuf_addf(&header, "%s ", comment_line_str); - strbuf_addf(&header, _(combined_commit_msg_fmt), - ctx->current_fixup_count + 2); + add_squash_combination_header(&header, + ctx->current_fixup_count + 2); strbuf_splice(&buf, 0, eol - buf.buf, header.buf, header.len); strbuf_release(&header); if (is_fixup_flag(command, flag) && !seen_squash(ctx)) @@ -2109,12 +2121,9 @@ static int update_squash_messages(struct repository *r, repo_unuse_commit_buffer(r, head_commit, head_message); return error(_("cannot write '%s'"), rebase_path_fixup_msg()); } - strbuf_addf(&buf, "%s ", comment_line_str); - strbuf_addf(&buf, _(combined_commit_msg_fmt), 2); - strbuf_addf(&buf, "\n%s ", comment_line_str); - strbuf_addstr(&buf, is_fixup_flag(command, flag) ? - _(skip_first_commit_msg_str) : - _(first_commit_msg_str)); + add_squash_combination_header(&buf, 2); + strbuf_addch(&buf, '\n'); + add_squash_message_header(&buf, 1, is_fixup_flag(command, flag)); strbuf_addstr(&buf, "\n\n"); if (is_fixup_flag(command, flag)) strbuf_add_commented_lines(&buf, body, strlen(body), @@ -2133,9 +2142,8 @@ static int update_squash_messages(struct repository *r, if (command == TODO_SQUASH || is_fixup_flag(command, flag)) { res = append_squash_message(&buf, body, command, opts, flag); } else if (command == TODO_FIXUP) { - strbuf_addf(&buf, "\n%s ", comment_line_str); - strbuf_addf(&buf, _(skip_nth_commit_msg_fmt), - ++ctx->current_fixup_count + 1); + strbuf_addch(&buf, '\n'); + add_squash_message_header(&buf, ++ctx->current_fixup_count + 1, 1); strbuf_addstr(&buf, "\n\n"); strbuf_add_commented_lines(&buf, body, strlen(body), comment_line_str); diff --git a/sequencer.h b/sequencer.h index 64a9c7fb1beccd..b01f8970201fc3 100644 --- a/sequencer.h +++ b/sequencer.h @@ -119,6 +119,13 @@ enum todo_command { TODO_COMMENT }; +/* Bits for the "flags" member of struct todo_item */ +enum todo_item_flags { + TODO_EDIT_MERGE_MSG = (1 << 0), + TODO_REPLACE_FIXUP_MSG = (1 << 1), + TODO_EDIT_FIXUP_MSG = (1 << 2), +}; + struct todo_item { enum todo_command command; struct commit *commit; @@ -208,6 +215,29 @@ int todo_list_rearrange_squash(struct todo_list *todo_list); */ void append_signoff(struct strbuf *msgbuf, size_t ignore_footer, unsigned flag); +/* + * Append the "This is a combination of N commits." banner that "git rebase + * -i" writes at the top of a squashed commit's message, commented out with + * the comment character. + */ +void add_squash_combination_header(struct strbuf *buf, int n); + +/* + * Append the header (1-based N) that "git rebase -i" writes above each message + * when squashing, commented out with the comment character. With SKIP it reads + * "The ... commit message will be skipped" for a message that is dropped (a + * fixup), otherwise "This is the ... commit message". + */ +void add_squash_message_header(struct strbuf *buf, int n, int skip); + +/* + * Return the length of the leading subject of BODY when it should be commented + * out in a squash message, or 0 otherwise. An "amend!" subject always + * qualifies; "squash!" and "fixup!" subjects only when SQUASHING, since a + * plain fixup chain keeps them. + */ +size_t squash_subject_comment_len(const char *body, int squashing); + void append_conflicts_hint(struct index_state *istate, struct strbuf *msgbuf, enum commit_msg_cleanup_mode cleanup_mode); enum commit_msg_cleanup_mode get_cleanup_mode(const char *cleanup_arg, From 765a0092a6faf9bcbc022fd9fbde747012ac82b9 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Fri, 7 Aug 2026 07:39:27 +0000 Subject: [PATCH 092/259] history: add skeleton for squash subcommand Add the entry point and option parsing for "git history squash". Pass the remaining arguments through setup_revisions() so the command accepts revision ranges and rev-list options, while restoring the ordering and simplification settings required by the fold if an option changes them. Require at least one BOTTOM revision. The squashed commit needs a commit outside the selected range to serve as its base, so a single positive revision is not a sufficient range. Keep this step limited to defining the revision input contract so graph validation and the rewrite can be added independently. Helped-by: Phillip Wood Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- Documentation/git-history.adoc | 1 + builtin/history.c | 94 ++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/Documentation/git-history.adoc b/Documentation/git-history.adoc index 28b477cd378150..b660baf94db033 100644 --- a/Documentation/git-history.adoc +++ b/Documentation/git-history.adoc @@ -12,6 +12,7 @@ git history drop [--dry-run] [--update-refs=(branches|head)] [--empty=( git history fixup [--dry-run] [--update-refs=(branches|head)] [--reedit-message] [--empty=(drop|keep|abort)] git history reword [--dry-run] [--update-refs=(branches|head)] git history split [--dry-run] [--update-refs=(branches|head)] [--] [...] +git history squash [--dry-run] [--update-refs=(branches|head)] [--[no-]edit] DESCRIPTION ----------- diff --git a/builtin/history.c b/builtin/history.c index b592b98393c640..b0502462817567 100644 --- a/builtin/history.c +++ b/builtin/history.c @@ -34,6 +34,8 @@ N_("git history reword [--dry-run] [--update-refs=(branches|head)]") #define GIT_HISTORY_SPLIT_USAGE \ N_("git history split [--dry-run] [--update-refs=(branches|head)] [--] [...]") +#define GIT_HISTORY_SQUASH_USAGE \ + N_("git history squash [--dry-run] [--update-refs=(branches|head)] [--[no-]edit] ") static void change_data_free(void *util, const char *str UNUSED) { @@ -1004,6 +1006,96 @@ static int cmd_history_split(int argc, return ret; } +static int setup_squash_revisions(struct repository *repo, + int argc, const char **argv, + struct rev_info *revs) +{ + repo_init_revisions(repo, revs, NULL); + revs->reverse = 1; + revs->topo_order = 1; + revs->sort_order = REV_SORT_IN_GRAPH_ORDER; + revs->simplify_history = 0; + revs->ancestry_path = 1; + revs->limited = 1; + revs->ancestry_path_implicit_bottoms = 1; + + argc = setup_revisions(argc, argv, revs, NULL); + if (argc > 1) + return error(_("unrecognized argument: %s"), argv[1]); + + if (revs->reverse != 1 || revs->topo_order != 1 || + revs->sort_order != REV_SORT_IN_GRAPH_ORDER || + revs->simplify_history != 0 || revs->boundary == 1 || + revs->ancestry_path != 1 || revs->limited != 1 || + revs->ancestry_path_implicit_bottoms != 1) { + warning(_("ignoring rev-list options that would change how the " + "range is walked")); + revs->reverse = 1; + revs->topo_order = 1; + revs->sort_order = REV_SORT_IN_GRAPH_ORDER; + revs->simplify_history = 0; + revs->boundary = 0; + revs->ancestry_path = 1; + revs->limited = 1; + revs->ancestry_path_implicit_bottoms = 1; + } + + /* + * A squash needs a base to reparent onto, so the range has to exclude + * something, as in "..". A revision range with no such + * bottom commit cannot be squashed. + */ + for (size_t i = 0; i < revs->cmdline.nr; i++) + if (revs->cmdline.rev[i].flags & BOTTOM) + return 0; + + return error(_("not a '..' revision range")); +} + +static int cmd_history_squash(int argc, + const char **argv, + const char *prefix, + struct repository *repo) +{ + const char * const usage[] = { + GIT_HISTORY_SQUASH_USAGE, + NULL, + }; + enum ref_action action = REF_ACTION_DEFAULT; + int dry_run = 0; + int edit = 1; + struct option options[] = { + OPT_CALLBACK_F(0, "update-refs", &action, "(branches|head)", + N_("control which refs should be updated"), + PARSE_OPT_NONEG, parse_ref_action), + OPT_BOOL('n', "dry-run", &dry_run, + N_("perform a dry-run without updating any refs")), + OPT_BOOL('e', "edit", &edit, + N_("edit the commit message")), + OPT_END(), + }; + struct rev_info revs = { 0 }; + int ret; + + argc = parse_options(argc, argv, prefix, options, usage, + PARSE_OPT_KEEP_UNKNOWN_OPT | PARSE_OPT_KEEP_ARGV0); + if (argc < 2) { + ret = error(_("command expects a revision range")); + goto out; + } + repo_config(repo, git_default_config, NULL); + + ret = setup_squash_revisions(repo, argc, argv, &revs); + if (ret < 0) + goto out; + + ret = error(_("squashing commits is not implemented yet")); + +out: + release_revisions(&revs); + return ret; +} + static int update_worktree(struct repository *repo, const struct commit *old_head, const struct commit *new_head, @@ -1192,6 +1284,7 @@ int cmd_history(int argc, GIT_HISTORY_FIXUP_USAGE, GIT_HISTORY_REWORD_USAGE, GIT_HISTORY_SPLIT_USAGE, + GIT_HISTORY_SQUASH_USAGE, NULL, }; parse_opt_subcommand_fn *fn = NULL; @@ -1200,6 +1293,7 @@ int cmd_history(int argc, OPT_SUBCOMMAND("fixup", &fn, cmd_history_fixup), OPT_SUBCOMMAND("reword", &fn, cmd_history_reword), OPT_SUBCOMMAND("split", &fn, cmd_history_split), + OPT_SUBCOMMAND("squash", &fn, cmd_history_squash), OPT_END(), }; From 044a87ed3f644228ba14014724f75665c8ddfdb0 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Fri, 7 Aug 2026 07:39:28 +0000 Subject: [PATCH 093/259] history: validate squash revision ranges Walk the selected commits in topological order from oldest to newest and mark each one as it is seen. Every parent after the oldest commit must already be selected or also be a parent of the oldest commit. This accepts merges contained by the range while rejecting a merge arm that entered it from elsewhere. Track the remaining graph tips during the same walk and require exactly one. Also reject empty and single-commit ranges and any selection that reaches a root commit. These checks identify the oldest commit whose parents will be preserved and the single tip whose tree will be used by the rewrite. Helped-by: Phillip Wood Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- builtin/history.c | 119 ++++++++++++++++++++++++++++++++++---- object.h | 1 + t/meson.build | 1 + t/t3455-history-squash.sh | 65 +++++++++++++++++++++ 4 files changed, 174 insertions(+), 12 deletions(-) create mode 100755 t/t3455-history-squash.sh diff --git a/builtin/history.c b/builtin/history.c index b0502462817567..a7bf583862c048 100644 --- a/builtin/history.c +++ b/builtin/history.c @@ -1006,6 +1006,9 @@ static int cmd_history_split(int argc, return ret; } +#define SQUASH_SEEN (1u << 11) +#define SQUASH_TIP (1u << 12) + static int setup_squash_revisions(struct repository *repo, int argc, const char **argv, struct rev_info *revs) @@ -1052,6 +1055,104 @@ static int setup_squash_revisions(struct repository *repo, return error(_("not a '..' revision range")); } +/* + * Resolve a revision range into its oldest commit and single tip. Every + * parent after the oldest commit must either be selected or also be a parent + * of the oldest commit. + */ +static int resolve_squash_range(struct repository *repo, + int argc, const char **argv, + struct commit **oldest_out, + struct commit **tip_out) +{ + struct rev_info revs; + struct commit *commit, *oldest = NULL, *tip = NULL; + int ret, tip_count = 0; + bool walk_started = false; + + ret = setup_squash_revisions(repo, argc, argv, &revs); + if (ret < 0) + goto out; + + if (prepare_revision_walk(&revs) < 0) { + ret = error(_("error preparing revisions")); + goto out; + } + walk_started = true; + while ((commit = get_revision(&revs))) { + struct commit_list *p; + + if (!commit->parents) { + ret = error(_("cannot squash down to root commit")); + goto out; + } + for (p = commit->parents; oldest && p; p = p->next) { + struct commit_list *q; + struct object *o; + bool seen; + + if (repo_parse_commit(repo, p->item)) { + ret = error(_("cannot parse commit")); + goto out; + } + o = &p->item->object; + seen = o->flags & SQUASH_SEEN; + /* + * Allow parents that match the parents of the + * squashed commit. + */ + for (q = oldest->parents; !seen && q; q = q->next) + if (p->item == q->item) + seen = true; + if (!seen) { + ret = error(_("parent %s of commit %s is " + "outside the revision range"), + repo_find_unique_abbrev(repo, &o->oid, + DEFAULT_ABBREV), + repo_find_unique_abbrev(repo, + &commit->object.oid, + DEFAULT_ABBREV)); + goto out; + } + if (o->flags & SQUASH_TIP) { + tip_count--; + o->flags &= ~SQUASH_TIP; + } + } + if (!oldest) + oldest = commit; + tip = commit; + tip->object.flags |= SQUASH_SEEN | SQUASH_TIP; + tip_count++; + } + + if (!tip_count) { + ret = error(_("the revision range is empty")); + goto out; + } else if (tip_count != 1) { + ret = error(_("the revision range contains more than one tip " + "commit")); + goto out; + } else if (oldest == tip) { + ret = error(_("the revision range holds a single commit; " + "nothing to squash")); + goto out; + } else if (!oldest->parents) { + BUG("an in-range commit must have a parent"); + } + + *oldest_out = oldest; + *tip_out = tip; + ret = 0; + +out: + clear_object_flags(repo, SQUASH_SEEN | SQUASH_TIP); + if (walk_started) + reset_revision_walk(); + release_revisions(&revs); + return ret; +} + static int cmd_history_squash(int argc, const char **argv, const char *prefix, @@ -1074,26 +1175,20 @@ static int cmd_history_squash(int argc, N_("edit the commit message")), OPT_END(), }; - struct rev_info revs = { 0 }; + struct commit *oldest, *tip; int ret; argc = parse_options(argc, argv, prefix, options, usage, PARSE_OPT_KEEP_UNKNOWN_OPT | PARSE_OPT_KEEP_ARGV0); - if (argc < 2) { - ret = error(_("command expects a revision range")); - goto out; - } + if (argc < 2) + return error(_("command expects a revision range")); repo_config(repo, git_default_config, NULL); - ret = setup_squash_revisions(repo, argc, argv, &revs); + ret = resolve_squash_range(repo, argc, argv, &oldest, &tip); if (ret < 0) - goto out; - - ret = error(_("squashing commits is not implemented yet")); + return ret; -out: - release_revisions(&revs); - return ret; + return error(_("squashing commits is not implemented yet")); } static int update_worktree(struct repository *repo, diff --git a/object.h b/object.h index 8fb03ff90a3e56..dcf30156ca6786 100644 --- a/object.h +++ b/object.h @@ -74,6 +74,7 @@ void object_array_init(struct object_array *array); * bisect.c: 16 * bundle.c: 16 * http-push.c: 11-----14 + * builtin/history.c: 1112 * commit-graph.c: 15 * commit-reach.c: 16-------20 * builtin/last-modified.c: 1617 diff --git a/t/meson.build b/t/meson.build index 8ae6ab6c5fe1e2..89cff164054b3d 100644 --- a/t/meson.build +++ b/t/meson.build @@ -405,6 +405,7 @@ integration_tests = [ 't3452-history-split.sh', 't3453-history-fixup.sh', 't3454-history-drop.sh', + 't3455-history-squash.sh', 't3500-cherry.sh', 't3501-revert-cherry-pick.sh', 't3502-cherry-pick-merge.sh', diff --git a/t/t3455-history-squash.sh b/t/t3455-history-squash.sh new file mode 100755 index 00000000000000..df92aa4f6c9e6c --- /dev/null +++ b/t/t3455-history-squash.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +test_description='tests for git-history squash subcommand' + +. ./test-lib.sh + +test_expect_success 'setup linear history' ' + test_commit base file a start && + test_commit one file b && + test_commit two file c && + test_commit three file d +' + +test_expect_success 'errors on missing range argument' ' + test_must_fail git history squash 2>err && + test_grep "expects a revision range" err +' + +test_expect_success 'errors on an empty range' ' + test_must_fail git history squash HEAD..HEAD 2>err && + test_grep "the revision range is empty" err +' + +test_expect_success 'errors on a single revision that is not a range' ' + test_must_fail git history squash HEAD 2>err && + test_grep "not a .*range" err && + test_must_fail git history squash HEAD~1 2>err && + test_grep "not a .*range" err +' + +test_expect_success 'errors on a range holding a single commit' ' + test_must_fail git history squash "HEAD^!" 2>err && + test_grep "single commit; nothing to squash" err +' + +test_expect_success 'rejects a root commit' ' + oid=$(git commit-tree -m root three^{tree}) && + test_must_fail git history squash \ + --ancestry-path=start "$oid..three" 2>err && + test_grep "cannot squash down to root commit" err +' + +test_expect_success 'rejects multiple tips' ' + oid=$(git commit-tree -m tip -p start^0 three^{tree}) && + test_must_fail git history squash ^start "$oid" three~1 2>err && + test_grep "revision range contains more than one tip" err +' + +test_expect_success 'rejects a merge parent outside the range' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + git checkout -b outside-parent && + test_commit --no-tag outside-parent outside x && + git checkout "$main" && + test_commit --no-tag outside-main file b && + base=$(git rev-parse HEAD) && + test_commit --no-tag outside-mid file c && + git merge --no-ff -m "merge outside-parent" outside-parent && + git branch -D outside-parent && + + test_must_fail git history squash "$base.." 2>err && + test_grep "parent .* of commit .* is outside the revision range" err +' + +test_done From 8a709ef6d2608e5a8da2cc1e8a00b54cf64e743f Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Fri, 7 Aug 2026 07:39:29 +0000 Subject: [PATCH 094/259] history: protect branches when squashing a range A local branch that descends from the selected graph without containing its tip cannot be replayed as a descendant of the squashed commit. Find those branches with ref-filter before creating any replacement objects and refuse the operation unless --update-refs=head was requested. Limit this protection to local branches, matching the refs that the default history rewrite mode updates; tags and remote-tracking refs remain untouched. Sort the blocking refs and print their short branch names so the user can decide whether to move them or leave them behind. Add advice.historyUpdateRefs for the hint that points to --update-refs=head. Helped-by: Phillip Wood Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- Documentation/config/advice.adoc | 4 ++ advice.c | 1 + advice.h | 1 + builtin/history.c | 70 ++++++++++++++++++++++++++++++-- t/t3455-history-squash.sh | 39 ++++++++++++++++++ 5 files changed, 111 insertions(+), 4 deletions(-) diff --git a/Documentation/config/advice.adoc b/Documentation/config/advice.adoc index 81f80a92745123..3d91c90eda72f9 100644 --- a/Documentation/config/advice.adoc +++ b/Documentation/config/advice.adoc @@ -59,6 +59,10 @@ all advice messages. forceDeleteBranch:: Shown when the user tries to delete a not fully merged branch without the force option set. + historyUpdateRefs:: + Shown when `git history squash` refuses because a local branch + cannot be rewritten as a descendant of the squashed commit, to + tell the user about `--update-refs=head`. ignoredHook:: Shown when a hook is ignored because the hook is not set as executable. diff --git a/advice.c b/advice.c index 63bf8b0c5f0481..401d0473917cd9 100644 --- a/advice.c +++ b/advice.c @@ -58,6 +58,7 @@ static struct { [ADVICE_FETCH_SHOW_FORCED_UPDATES] = { "fetchShowForcedUpdates" }, [ADVICE_FORCE_DELETE_BRANCH] = { "forceDeleteBranch" }, [ADVICE_GRAFT_FILE_DEPRECATED] = { "graftFileDeprecated" }, + [ADVICE_HISTORY_UPDATE_REFS] = { "historyUpdateRefs" }, [ADVICE_IGNORED_HOOK] = { "ignoredHook" }, [ADVICE_IMPLICIT_IDENTITY] = { "implicitIdentity" }, [ADVICE_MERGE_CONFLICT] = { "mergeConflict" }, diff --git a/advice.h b/advice.h index 66f6cd6a772d8c..3f0b4f0485c7fe 100644 --- a/advice.h +++ b/advice.h @@ -25,6 +25,7 @@ enum advice_type { ADVICE_FETCH_SHOW_FORCED_UPDATES, ADVICE_FORCE_DELETE_BRANCH, ADVICE_GRAFT_FILE_DEPRECATED, + ADVICE_HISTORY_UPDATE_REFS, ADVICE_IGNORED_HOOK, ADVICE_IMPLICIT_IDENTITY, ADVICE_MERGE_CONFLICT, diff --git a/builtin/history.c b/builtin/history.c index a7bf583862c048..f6a4205cd63bf2 100644 --- a/builtin/history.c +++ b/builtin/history.c @@ -1,6 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "builtin.h" +#include "advice.h" #include "cache-tree.h" #include "commit.h" #include "commit-reach.h" @@ -16,10 +17,12 @@ #include "path.h" #include "read-cache.h" #include "refs.h" +#include "ref-filter.h" #include "replay.h" #include "reset.h" #include "revision.h" #include "sequencer.h" +#include "string-list.h" #include "strvec.h" #include "tree.h" #include "tree-walk.h" @@ -1061,6 +1064,7 @@ static int setup_squash_revisions(struct repository *repo, * of the oldest commit. */ static int resolve_squash_range(struct repository *repo, + bool update_branches, int argc, const char **argv, struct commit **oldest_out, struct commit **tip_out) @@ -1069,6 +1073,8 @@ static int resolve_squash_range(struct repository *repo, struct commit *commit, *oldest = NULL, *tip = NULL; int ret, tip_count = 0; bool walk_started = false; + struct ref_filter filter = REF_FILTER_INIT; + struct ref_array refs = { 0 }; ret = setup_squash_revisions(repo, argc, argv, &revs); if (ret < 0) @@ -1101,9 +1107,12 @@ static int resolve_squash_range(struct repository *repo, * Allow parents that match the parents of the * squashed commit. */ - for (q = oldest->parents; !seen && q; q = q->next) - if (p->item == q->item) + for (q = oldest->parents; !seen && q; q = q->next) { + if (p->item == q->item) { seen = true; + commit_list_insert(commit, &filter.with_commit); + } + } if (!seen) { ret = error(_("parent %s of commit %s is " "outside the revision range"), @@ -1119,12 +1128,17 @@ static int resolve_squash_range(struct repository *repo, o->flags &= ~SQUASH_TIP; } } - if (!oldest) + if (!oldest) { + commit_list_insert(commit, &filter.with_commit); oldest = commit; + } tip = commit; tip->object.flags |= SQUASH_SEEN | SQUASH_TIP; tip_count++; } + clear_object_flags(repo, SQUASH_SEEN | SQUASH_TIP); + reset_revision_walk(); + walk_started = false; if (!tip_count) { ret = error(_("the revision range is empty")); @@ -1141,6 +1155,49 @@ static int resolve_squash_range(struct repository *repo, BUG("an in-range commit must have a parent"); } + commit_list_insert(tip, &filter.no_commit); + filter.kind = FILTER_REFS_BRANCHES; + if (update_branches && + filter_refs(&refs, &filter, filter.kind)) { + ret = error(_("could not filter refs")); + goto out; + } + if (refs.nr) { + struct ref_format format = REF_FORMAT_INIT; + struct ref_sorting *sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; + struct strbuf branches = STRBUF_INIT; + struct strbuf err = STRBUF_INIT; + + format.format = "%(refname:short)"; + if (verify_ref_format(&format)) + BUG("invalid branch format"); + string_list_append(&sorting_options, "refname"); + sorting = ref_sorting_options(&sorting_options); + ref_array_sort(sorting, &refs); + for (int i = 0; i < refs.nr; i++) { + strbuf_reset(&err); + strbuf_addstr(&branches, "\n "); + if (format_ref_array_item(refs.items[i], &format, + &branches, &err)) + BUG("could not format branch name: %s", err.buf); + } + /* + * TODO: also check HEADS from other worktrees. + */ + ret = error(_("the following branches cannot be rewritten as " + "descendants of the squashed commit:%s"), branches.buf); + advise_if_enabled(ADVICE_HISTORY_UPDATE_REFS, + _("Use --update-refs=head to rewrite only " + "the current branch and leave such branches " + "untouched.")); + strbuf_release(&err); + strbuf_release(&branches); + ref_sorting_release(sorting); + string_list_clear(&sorting_options, 0); + goto out; + } + *oldest_out = oldest; *tip_out = tip; ret = 0; @@ -1150,6 +1207,8 @@ static int resolve_squash_range(struct repository *repo, if (walk_started) reset_revision_walk(); release_revisions(&revs); + ref_filter_clear(&filter); + ref_array_clear(&refs); return ret; } @@ -1183,8 +1242,11 @@ static int cmd_history_squash(int argc, if (argc < 2) return error(_("command expects a revision range")); repo_config(repo, git_default_config, NULL); + if (action == REF_ACTION_DEFAULT) + action = REF_ACTION_BRANCHES; - ret = resolve_squash_range(repo, argc, argv, &oldest, &tip); + ret = resolve_squash_range(repo, action == REF_ACTION_BRANCHES, + argc, argv, &oldest, &tip); if (ret < 0) return ret; diff --git a/t/t3455-history-squash.sh b/t/t3455-history-squash.sh index df92aa4f6c9e6c..b1f65de5f507ad 100755 --- a/t/t3455-history-squash.sh +++ b/t/t3455-history-squash.sh @@ -62,4 +62,43 @@ test_expect_success 'rejects a merge parent outside the range' ' test_grep "parent .* of commit .* is outside the revision range" err ' +test_expect_success 'prints branches that cannot follow the squash' ' + test_when_finished \ + "git switch -f $GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME; \ + git branch -D feature" && + git checkout -f -b feature start && + test_commit C1 && + test_commit C2 && + git checkout -b topic-1 start && + test_commit C3 && + test_commit C4 && + git checkout C3 && + test_commit C5 && + git checkout feature && + git merge C5 && + test_commit C6 && + git checkout -b topic-2 C2 && + test_commit C7 && + git checkout feature && + + test_must_fail git history squash start.. 2>err && + test_grep "^error: the following branches cannot be rewritten" err && + test_grep "^ topic-1$" err && + test_grep "^ topic-2$" err && + test_grep "^hint: .* --update-refs=head" err +' + +test_expect_success 'advice.historyUpdateRefs silences the hint' ' + git reset --hard three && + git branch -f mid HEAD~1 && + + test_must_fail git -c advice.historyUpdateRefs=false \ + history squash start.. 2>err && + test_grep "^error: the following branches cannot be rewritten" err && + test_grep "^ mid$" err && + test_grep ! "hint:" err && + + git branch -D mid +' + test_done From 9ea34ffa0330eec93a2c101fe520b8da505028f6 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Fri, 7 Aug 2026 07:39:30 +0000 Subject: [PATCH 095/259] history: create squashed commits without editing Create one replacement commit from the resolved range when --no-edit is selected. Preserve the authorship and all parents of the oldest commit, use the tip tree, and replay descendants through the existing history rewrite machinery. Record the complete revision expression in the reflog and retain dry-run and update-refs behavior. Resolve fixup!, squash! and amend! subjects while walking the range. Reject markers whose targets are not selected and refuse any no-edit fold that would discard a squash! or amend! message. A range made entirely from related markers can still be consolidated, with the last applicable amend! body supplying the message. Inspired-by: Sergey Chernov Helped-by: Phillip Wood Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- Documentation/git-history.adoc | 44 ++- builtin/history.c | 324 ++++++++++++++++- object.h | 2 +- t/t3455-history-squash.sh | 633 +++++++++++++++++++++++++++++++-- 4 files changed, 968 insertions(+), 35 deletions(-) diff --git a/Documentation/git-history.adoc b/Documentation/git-history.adoc index b660baf94db033..fb04a6768513f0 100644 --- a/Documentation/git-history.adoc +++ b/Documentation/git-history.adoc @@ -44,8 +44,11 @@ at once. LIMITATIONS ----------- -This command does not (yet) work with histories that contain merges. You -should use linkgit:git-rebase[1] with the `--rebase-merges` flag instead. +This command does not (yet) replay merge commits onto the rewritten +history: if a commit that would be replayed is a merge, the operation is +rejected, and you should use linkgit:git-rebase[1] with the +`--rebase-merges` flag instead. The `squash` subcommand can still fold merges +that lie inside the selected range, subject to the restrictions below. Furthermore, the command does not support operations that can result in merge conflicts. This limitation is by design as history rewrites are not intended to @@ -114,6 +117,43 @@ linkgit:gitglossary[7]. It is invalid to select either all or no hunks, as that would lead to one of the commits becoming empty. +`squash `:: + Fold all commits in __ into the oldest commit of that + range. The resulting commit keeps the oldest commit's authorship and + takes the tree of the range's newest commit, so the whole range + collapses into a single commit. Commits above the range are replayed + on top of the result. ++ +The range is given in the usual `..` form, where __ is +the commit just below the oldest commit to squash. For example, `git +history squash HEAD~3..HEAD` folds the three most recent commits into +one, and `git history squash HEAD~5..HEAD~2` squashes an interior range +while leaving the two newest commits in place. Several revisions may be +given, for example `HEAD~3..HEAD ^topic` to additionally exclude what is +already on `topic`. Rev-list options may also be given, but any that would +change how the range is walked are overridden with a warning. ++ +With `--no-edit`, the oldest commit's message is preserved, except that an +`amend!` commit targeting it replaces its message. ++ +The selected commits must form a connected graph with a single tip and must +not include a root commit. Every parent of a commit after the oldest one must +either be selected or also be a parent of the oldest commit. When the oldest +commit is a merge, all of its parents are preserved in the squashed commit. ++ +A `fixup!`, `squash!`, or `amend!` commit is refused unless the commit it +targets is also in the range, so the fold does not silently absorb a +marker meant for a commit outside it. As an exception, a range made up entirely +of markers for one target is combined into a single commit. With `--no-edit`, +the last `amend!` message is used if there is one; a `squash!` or `amend!` is +otherwise refused if folding it would discard its message. ++ +A local branch descended from a selected commit but not from the range tip +cannot be rewritten as a descendant of the result, so with the default +`--update-refs=branches` the command refuses. Rerun with `--update-refs=head` +to rewrite only the current branch and leave such branches unchanged. Tags +and remote-tracking refs are always left unchanged. + OPTIONS ------- diff --git a/builtin/history.c b/builtin/history.c index f6a4205cd63bf2..41cef2c9763380 100644 --- a/builtin/history.c +++ b/builtin/history.c @@ -1011,6 +1011,260 @@ static int cmd_history_split(int argc, #define SQUASH_SEEN (1u << 11) #define SQUASH_TIP (1u << 12) +#define SQUASH_AMEND_TARGET (1u << 13) + +static bool is_autosquash_subject(const char *s) +{ + return starts_with(s, "amend!") || starts_with(s, "fixup!") || + starts_with(s, "squash!"); +} + +static bool skip_one_autosquash_prefix(const char *s, const char **out) +{ + if (skip_prefix(s, "amend!", out) || skip_prefix(s, "fixup!", out) || + skip_prefix(s, "squash!", out)) { + while (**out == ' ') + (*out)++; + return true; + } + return false; +} + +static void truncate_message_to_subject(struct strbuf *msg) +{ + const char *eos = strstr(msg->buf, "\n\n"); + + if (eos) + strbuf_setlen(msg, eos - msg->buf + 1); +} + +struct subject_data { + struct strintmap subjects; + struct strbuf subject; + struct strbuf squash_message; + const char *message; + bool edit_message; +}; + +#define SUBJECT_DATA_INIT { \ + .subjects = STRINTMAP_INIT, \ + .subject = STRBUF_INIT, \ + .squash_message = STRBUF_INIT, \ +} + +static void subject_data_clear(struct subject_data *data) +{ + strintmap_clear(&data->subjects); + strbuf_release(&data->subject); + strbuf_release(&data->squash_message); +} + +static int squash_amend_message(struct repository *repo, + struct commit *commit, + struct subject_data *data, + unsigned flags) +{ + const char *body = data->message + data->subject.len; + + while (isspace(*body)) + body++; + + if (!*body) { + warning(_("ignoring %s (%s): message body is empty"), + repo_find_unique_abbrev(repo, &commit->object.oid, + DEFAULT_ABBREV), + data->subject.buf); + return 0; + } + + if (data->edit_message) { + return 0; + } else if (flags & SQUASH_AMEND_TARGET) { + if (starts_with(data->squash_message.buf, "squash!")) + return error(_("squashing %s (%s) would overwrite " + "'squash!' message, please combine them " + "using '--edit'"), + repo_find_unique_abbrev(repo, + &commit->object.oid, + DEFAULT_ABBREV), + data->subject.buf); + if (starts_with(data->squash_message.buf, "fixup!")) + strbuf_splice(&data->squash_message, 0, 5, "amend!", 5); + if (starts_with(data->squash_message.buf, "amend!")) { + truncate_message_to_subject(&data->squash_message); + strbuf_addch(&data->squash_message, '\n'); + } else { + strbuf_reset(&data->squash_message); + } + strbuf_addstr(&data->squash_message, body); + strbuf_complete_line(&data->squash_message); + } else { + return error(_("cannot squash %s (%s) that does not target " + "base commit without '--edit'"), + repo_find_unique_abbrev(repo, &commit->object.oid, + DEFAULT_ABBREV), + data->subject.buf); + } + return 0; +} + +static int squash_squash_message(struct repository *repo, + struct commit *commit, + struct subject_data *data, + unsigned flags) +{ + const char *body = data->message + data->subject.len; + + while (isspace(*body)) + body++; + + if (data->edit_message) { + return 0; + } else if (flags & SQUASH_AMEND_TARGET) { + if (starts_with(data->squash_message.buf, "fixup!")) { + truncate_message_to_subject(&data->squash_message); + strbuf_splice(&data->squash_message, 0, 5, "squash", 6); + } + if (starts_with(data->squash_message.buf, "squash!")) { + strbuf_addch(&data->squash_message, '\n'); + strbuf_addstr(&data->squash_message, body); + strbuf_complete_line(&data->squash_message); + } else { + return error(_("squashing %s (%s) would discard its " + "message, please combine them using " + "'--edit'"), + repo_find_unique_abbrev(repo, + &commit->object.oid, + DEFAULT_ABBREV), + data->subject.buf); + } + } else { + return error(_("cannot squash %s (%s) that does not target " + "base commit without '--edit'"), + repo_find_unique_abbrev(repo, &commit->object.oid, + DEFAULT_ABBREV), + data->subject.buf); + } + return 0; +} + +static int squash_check_can_autosquash(struct repository *repo, + struct commit *commit, + struct subject_data *data, + unsigned flags) +{ + commit->object.flags |= flags & SQUASH_AMEND_TARGET; + if (starts_with(data->subject.buf, "amend!")) + return squash_amend_message(repo, commit, data, flags); + else if (starts_with(data->subject.buf, "squash!")) + return squash_squash_message(repo, commit, data, flags); + + return 0; +} + +static int squash_check_autosquash_subject(struct repository *repo, + struct commit *commit, + struct subject_data *data) +{ + const char* s = data->subject.buf; + struct commit *target; + struct hashmap_iter iter; + struct strmap_entry *entry; + /* Try skipping autosquash prefixes one at a time to allow + * squashing + * a commit + * fixup! fixup! a commit + * + * where we may have started with + * a commit + * fixup! a commit + * fixup! fixup! a commit + * + * and squashed the first fixup separately from the second + */ + while (skip_one_autosquash_prefix(s, &s)) { + unsigned flags = strintmap_get(&data->subjects, s); + if (flags) + return squash_check_can_autosquash(repo, commit, data, flags); + } + /* + * Allow "fixup! ", but not "fixup! HEAD^" or + * "fixup! main". If the target is not being squshed check the subject + * to allow "fixup! abc123" and "fixup! " to be + * squashed together. + */ + target = lookup_commit_reference_by_name(s); + if (target && istarts_with(oid_to_hex(&target->object.oid), s)) { + unsigned flags = + target->object.flags & (SQUASH_SEEN | SQUASH_AMEND_TARGET); + if (!flags) { + const char *subject_start; + const char *buffer = repo_logmsg_reencode(repo, target, + NULL, NULL); + size_t subject_len = find_commit_subject(buffer, + &subject_start); + char *subject = xmemdupz(subject_start, subject_len); + + flags = strintmap_get(&data->subjects, subject); + free(subject); + repo_unuse_commit_buffer(repo, target, buffer); + } + if (flags) + return squash_check_can_autosquash(repo, commit, + data, flags); + } + /* Try subject prefix matches */ + strintmap_for_each_entry(&data->subjects, &iter, entry) { + s = data->subject.buf; + while(skip_one_autosquash_prefix(s, &s)) { + if (starts_with(entry->key, s)) { + unsigned value = (intptr_t)entry->value; + + return squash_check_can_autosquash(repo, commit, + data, value); + } + } + } + return error(_("cannot squash %s (%s): its target is not being " + "squashed"), + repo_find_unique_abbrev(repo, &commit->object.oid, + DEFAULT_ABBREV), + data->subject.buf); +} + +static int squash_check_subject(struct repository *repo, + struct commit *commit, + struct subject_data *data) +{ + int ret = 0; + const char *buf = repo_logmsg_reencode(repo, commit, NULL, NULL); + size_t subject_len = find_commit_subject(buf, &data->message); + + strbuf_reset(&data->subject); + strbuf_add(&data->subject, data->message, subject_len); + + if (!strintmap_get_size(&data->subjects)) { + const char *s; + + strbuf_addstr(&data->squash_message, data->message); + strbuf_complete_line(&data->squash_message); + /* + * Strip a single autosquash prefix to allow squashing + * fixup! base + * amend! base + */ + s = data->subject.buf; + skip_one_autosquash_prefix(s, &s); + strintmap_set(&data->subjects, s, SQUASH_AMEND_TARGET | SQUASH_SEEN); + commit->object.flags |= SQUASH_AMEND_TARGET; + } else if (is_autosquash_subject(data->subject.buf)) { + ret = squash_check_autosquash_subject(repo, commit, data); + } else { + strintmap_set(&data->subjects, data->subject.buf, SQUASH_SEEN); + } + repo_unuse_commit_buffer(repo, commit, buf); + return ret; +} static int setup_squash_revisions(struct repository *repo, int argc, const char **argv, @@ -1067,9 +1321,11 @@ static int resolve_squash_range(struct repository *repo, bool update_branches, int argc, const char **argv, struct commit **oldest_out, - struct commit **tip_out) + struct commit **tip_out, + char **message_out) { struct rev_info revs; + struct subject_data subject_data = SUBJECT_DATA_INIT; struct commit *commit, *oldest = NULL, *tip = NULL; int ret, tip_count = 0; bool walk_started = false; @@ -1132,6 +1388,10 @@ static int resolve_squash_range(struct repository *repo, commit_list_insert(commit, &filter.with_commit); oldest = commit; } + if (squash_check_subject(repo, commit, &subject_data)) { + ret = -1; + goto out; + } tip = commit; tip->object.flags |= SQUASH_SEEN | SQUASH_TIP; tip_count++; @@ -1200,12 +1460,15 @@ static int resolve_squash_range(struct repository *repo, *oldest_out = oldest; *tip_out = tip; + *message_out = strbuf_detach(&subject_data.squash_message, NULL); ret = 0; out: - clear_object_flags(repo, SQUASH_SEEN | SQUASH_TIP); + clear_object_flags(repo, SQUASH_SEEN | SQUASH_TIP | + SQUASH_AMEND_TARGET); if (walk_started) reset_revision_walk(); + subject_data_clear(&subject_data); release_revisions(&revs); ref_filter_clear(&filter); ref_array_clear(&refs); @@ -1234,23 +1497,68 @@ static int cmd_history_squash(int argc, N_("edit the commit message")), OPT_END(), }; - struct commit *oldest, *tip; + struct strbuf reflog_msg = STRBUF_INIT; + struct commit *oldest, *tip, *rewritten; + const struct object_id *base_tree_oid, *tip_tree_oid; + char *message_template = NULL; + struct rev_info revs = { 0 }; int ret; argc = parse_options(argc, argv, prefix, options, usage, PARSE_OPT_KEEP_UNKNOWN_OPT | PARSE_OPT_KEEP_ARGV0); - if (argc < 2) - return error(_("command expects a revision range")); + if (argc < 2) { + ret = error(_("command expects a revision range")); + goto out; + } repo_config(repo, git_default_config, NULL); + if (action == REF_ACTION_DEFAULT) action = REF_ACTION_BRANCHES; + strbuf_addstr(&reflog_msg, "squash: updating "); + strbuf_join_argv(&reflog_msg, argc - 1, argv + 1, ' '); + ret = resolve_squash_range(repo, action == REF_ACTION_BRANCHES, - argc, argv, &oldest, &tip); + argc, argv, &oldest, &tip, + &message_template); if (ret < 0) - return ret; + goto out; + if (edit) { + ret = error(_("message editing is not supported yet; use '--no-edit'")); + goto out; + } + + ret = setup_revwalk(repo, action, tip, &revs); + if (ret < 0) + goto out; + + base_tree_oid = &repo_get_commit_tree(repo, + oldest->parents->item)->object.oid; + tip_tree_oid = &repo_get_commit_tree(repo, tip)->object.oid; + + ret = commit_tree_ext(repo, "squash", oldest, message_template, + oldest->parents, base_tree_oid, tip_tree_oid, + &rewritten, 0); + if (ret < 0) { + ret = error(_("failed writing squashed commit")); + goto out; + } + + ret = handle_reference_updates(&revs, action, tip, rewritten, + reflog_msg.buf, dry_run, + REPLAY_EMPTY_COMMIT_ABORT); + if (ret < 0) { + ret = error(_("failed replaying descendants")); + goto out; + } - return error(_("squashing commits is not implemented yet")); + ret = 0; + +out: + strbuf_release(&reflog_msg); + release_revisions(&revs); + free(message_template); + return ret; } static int update_worktree(struct repository *repo, diff --git a/object.h b/object.h index dcf30156ca6786..46cade33fb18fb 100644 --- a/object.h +++ b/object.h @@ -74,7 +74,7 @@ void object_array_init(struct object_array *array); * bisect.c: 16 * bundle.c: 16 * http-push.c: 11-----14 - * builtin/history.c: 1112 + * builtin/history.c: 11---13 * commit-graph.c: 15 * commit-reach.c: 16-------20 * builtin/last-modified.c: 1617 diff --git a/t/t3455-history-squash.sh b/t/t3455-history-squash.sh index b1f65de5f507ad..d2d8e5fdb9ce8f 100755 --- a/t/t3455-history-squash.sh +++ b/t/t3455-history-squash.sh @@ -4,11 +4,50 @@ test_description='tests for git-history squash subcommand' . ./test-lib.sh -test_expect_success 'setup linear history' ' +stage_file () { + printf "%s\n" "$1" >file && + git add file +} + +commit_with_message () { + printf "%b" "$1" >msg && + git commit --allow-empty -qF msg +} + +check_commit_count () { + git rev-list --count "$1" >actual && + echo "$2" >expect && + test_cmp expect actual +} + +check_log_subjects () { + git log --format="%s" "$1" >actual && + cat >expect && + test_cmp expect actual +} + +check_log_messages () { + git log --format="%B" "$1" >actual && + cat >expect && + test_cmp expect actual +} + +# Checks that the author data of two commits matches +# Usage: check_commit_author +check_commit_author () { + git show -s --format="%an <%ae> %ad" "$1" >expect && + git show -s --format="%an <%ae> %ad" "$2" >actual && + test_cmp expect actual +} + +test_expect_success 'setup linear history touching two files' ' test_commit base file a start && - test_commit one file b && - test_commit two file c && - test_commit three file d + GIT_AUTHOR_NAME=One GIT_AUTHOR_EMAIL=one@example.com \ + test_commit one other x && + GIT_AUTHOR_NAME=Two GIT_AUTHOR_EMAIL=two@example.com \ + test_commit two file c && + GIT_AUTHOR_NAME=Three GIT_AUTHOR_EMAIL=three@example.com \ + test_commit three file d ' test_expect_success 'errors on missing range argument' ' @@ -29,40 +68,331 @@ test_expect_success 'errors on a single revision that is not a range' ' ' test_expect_success 'errors on a range holding a single commit' ' + git reset --hard three && + head_before=$(git rev-parse HEAD) && + test_must_fail git history squash "HEAD^!" 2>err && - test_grep "single commit; nothing to squash" err + test_grep "single commit; nothing to squash" err && + test_cmp_rev "$head_before" HEAD ' -test_expect_success 'rejects a root commit' ' +test_expect_success 'rejects root commit' ' + # create a disconnected root commit oid=$(git commit-tree -m root three^{tree}) && - test_must_fail git history squash \ - --ancestry-path=start "$oid..three" 2>err && - test_grep "cannot squash down to root commit" err + # because we pass --ancestry-path when calling setup_revs() it the + # revision walk will only include commits decended from $oid so + # we need to give it another --ancestry-path commit to actually walk + # any commits. + test_must_fail git history squash --ancestry-path=start $oid..three 2>err && + echo "error: cannot squash down to root commit" >expect && + test_cmp expect err ' test_expect_success 'rejects multiple tips' ' oid=$(git commit-tree -m tip -p start^0 three^{tree}) && - test_must_fail git history squash ^start "$oid" three~1 2>err && - test_grep "revision range contains more than one tip" err + test_must_fail git history squash ^start $oid three~1 2>err && + echo "error: the revision range contains more than one tip commit" >expect && + test_cmp expect err && + + git reset --hard three && + git history squash --no-edit ^start three~1 three && + test_cmp_rev HEAD~1 start^0 && + test_cmp_rev HEAD^{tree} three^{tree} ' -test_expect_success 'rejects a merge parent outside the range' ' - git reset --hard start && +test_expect_success 'accepts multiple revision arguments with an exclusion' ' + git reset --hard three && + git branch -f keep HEAD~2 && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start..HEAD ^keep && + + git reflog -1 --format=%gs >actual && + echo "squash: updating start..HEAD ^keep" >expect && + test_cmp expect actual && + + check_log_subjects start..HEAD <<-\EOF && + two + one + EOF + test_cmp_rev keep HEAD~1 && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + + git branch -D keep +' + +test_expect_success 'squashes a branch the current branch is not on' ' + git reset --hard three && main=$(git symbolic-ref --short HEAD) && - git checkout -b outside-parent && - test_commit --no-tag outside-parent outside x && + head_before=$(git rev-parse HEAD) && + git checkout -b off-history start && + test_commit --no-tag off-one off a && + test_commit --no-tag off-two off b && git checkout "$main" && - test_commit --no-tag outside-main file b && - base=$(git rev-parse HEAD) && - test_commit --no-tag outside-mid file c && - git merge --no-ff -m "merge outside-parent" outside-parent && - git branch -D outside-parent && - test_must_fail git history squash "$base.." 2>err && - test_grep "parent .* of commit .* is outside the revision range" err + git history squash --no-edit start..off-history && + + check_commit_count start..off-history 1 && + test_cmp_rev "$head_before" HEAD && + + git branch -D off-history +' + +test_expect_success 'squashes a range into a single commit without changing the tree' ' + git reset --hard three && + head_before=$(git rev-parse HEAD) && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit --dry-run start.. >out && + predicted=$(awk "/^update refs\/heads\// {print \$3}" out) && + test_cmp_rev "$head_before" HEAD && + + git history squash --no-edit start.. && + + test "$predicted" = "$(git rev-parse HEAD)" && + check_commit_count start..HEAD 1 && + test_cmp_rev start HEAD^ && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + check_log_subjects -1 <<-\EOF && + one + EOF + git reflog >reflog && + test_grep "squash: updating" reflog +' + +test_expect_success 'sanitizes rev-list walk options, before and after --' ' + git reset --hard three && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit --date-order start.. 2>err && + test_grep "ignoring rev-list options" err && + test_cmp_rev start HEAD^ && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + + git reset --hard three && + git history squash --no-edit -- --reverse start.. 2>err && + test_grep "ignoring rev-list options" err && + test_cmp_rev start HEAD^ && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" +' + +test_expect_success 'squashes an interior range and replays descendants verbatim' ' + git reset --hard three && + final_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start..@~1 && + + check_log_subjects start..HEAD <<-\EOF && + three + one + EOF + + test_cmp_rev start HEAD~2 && + test "$final_tree" = "$(git rev-parse HEAD^{tree})" +' + +test_expect_success 'squashes when the base is the root commit' ' + git reset --hard three && + root=$(git rev-list --max-parents=0 HEAD) && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit "$root.." && + + check_commit_count "$root..HEAD" 1 && + test_cmp_rev "$root" HEAD^ && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" +' + +test_expect_success 'squashing a mix of fixups' ' + git reset --hard three && + echo fix >file && + git commit --fixup=two -a && + echo really fix >file && + git commit --fixup=one -a && + echo really really fix >file && + git commit --fixup=HEAD~1 -a && # fixup! two + echo really really really fix >file && + git commit --fixup=HEAD~1 -a && # fixup! one + + # squashing fixup! with a target that is not being squashed fails + test_must_fail git history squash one.. 2>err && + test_grep "^error: cannot squash .* (fixup! one): its target is not being squashed" err && + + # squashing fixup! into fixup! with a different target fails + test_must_fail git history squash HEAD~4.. 2>err && # HEAD~4 is fixup! two + test_grep "^error: cannot squash .* (fixup! one): its target is not being squashed" err && + + # squashing a sequence of fixup! commits into their targets + git history squash --no-edit start..HEAD~1 && + test_cmp_rev start HEAD~2 && + check_commit_author one HEAD~1 && + test_commit_message HEAD~1 -m one && + + # squashing "fixup! fixup! " into "" + git history squash --no-edit start.. && + test_cmp_rev start HEAD~1 && + check_commit_author one HEAD && + test_commit_message HEAD -m one +' + +test_expect_success 'squashing "squash!" messages' ' + git reset --hard two && + echo fix >file && + git commit --fixup=HEAD -a && + oldest=$(git rev-parse HEAD) && + echo better fix >file && + git commit -a -F - <<-EOF && + squash! $(git rev-parse two) + + Append this + EOF + + echo an even better fix >file && + git commit -a -F - <<-EOF && + squash! squash! two + + Append this as well + EOF + + # must edit when squashing "squash!" into its target + test_must_fail git history squash --no-edit two^.. 2>err && + test_grep "^error: squashing .* (squash! [a-f0-9]*) would discard its message" err && + + # squashing "squash!" into "fixup!" appends messages and changes + # subject prefix + git history squash --no-edit two.. && + test_cmp_rev HEAD^ two && + test_commit_message HEAD <<-\EOF && + squash! two + + Append this + + Append this as well + EOF + check_commit_author "$oldest" HEAD && + + git commit --allow-empty -F - <<-\EOF && + amend! two + + A new message + EOF + + # "amend!" does not replace "squash!" + test_must_fail git history squash --no-edit HEAD~2.. 2>err && + test_grep "^error: squashing .* (amend! two) would overwrite .squash!. message" err +' + +test_expect_success '--no-edit uses last "amend!" message without an editor' ' + git reset --hard three && + write_script editor <<-\EOF && + exit 1 + EOF + test_set_editor "$(pwd)/editor" && + echo fix >file && + git commit --author="Fix Me " --fixup=HEAD -a && + git commit --allow-empty -F - <<-EOF && + amend! $(git rev-parse --short HEAD) + + The first reword + + More detail + EOF + + git commit --allow-empty -F - <<-\EOF && + amend! three + + The second reword + + Extra detail + EOF + + test_commit WIP && + + cat >msg <<-EOF && + amend! $(git rev-parse HEAD^ | tr a-f A-F) + + The third reword + + Excruciating detail + EOF + + git commit --author="Someone Else " --allow-empty \ + -F msg && + + # squashing amend! updates the commit message + git history squash --no-edit three^.. && + sed -e 1,2d msg | test_commit_message HEAD && + check_commit_author three HEAD && + test_cmp_rev HEAD^ three^ && + + # squashing amend! into fixup! updates subject prefix + git reset --hard HEAD@{1} && + git history squash --no-edit three.. && + sed "1s/.*/amend! three/" msg | test_commit_message HEAD && + check_commit_author HEAD@{1}~4 HEAD && + test_cmp_rev HEAD^ three && + + # squashing amend! into amend! keeps original subject line + git reset --hard HEAD@{1} && + git history squash --no-edit HEAD~3.. && + sed "1s/.*/amend! three/" msg | test_commit_message HEAD && + test_cmp_rev HEAD~3 three && + + # all amend! messages must target the first commit + git reset --hard HEAD@{1} && + git commit --allow-empty -F - <<-\EOF && + amend! WIP + + The real message + EOF + + test_must_fail git history squash --no-edit HEAD~4.. 2>err && + test_grep "^error: cannot squash .* that does not target" err && + + # amend! message that targets commit that is not in range is rejected + test_must_fail git history squash --no-edit HEAD~3.. 2>err && + test_grep "^error: cannot squash .* target is not being squashed" err && + test_set_editor : ' -test_expect_success 'prints branches that cannot follow the squash' ' +test_expect_success 'squashing fixups into a merge' ' + test_when_finished \ + "git switch -f $GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME; \ + git branch -D feature" && + git checkout -f start && + test_commit F1 && + git checkout -b feature start && + test_commit F2 && + git merge F1 && + echo fixed >F1.t && + cat >msg <<-EOF && + amend! $(git rev-parse HEAD) + + merge F1 and F2 + + reworded + EOF + + git commit -a -F msg && + git history squash --no-edit HEAD^^! HEAD && + test_cmp_rev HEAD^1 F2 && + test_cmp_rev HEAD^2 F1 && + test_cmp_rev HEAD@{1}^{tree} HEAD^{tree} && + sed 1,2d msg | test_commit_message HEAD +' + +test_expect_success '--update-refs=head only moves HEAD' ' + git reset --hard three && + git branch -f other HEAD && + other_before=$(git rev-parse other) && + + git history squash --no-edit --update-refs=head start.. && + + check_commit_count start..HEAD 1 && + test_cmp_rev "$other_before" other +' + +test_expect_success 'refuses to fold a range a branch points into' ' test_when_finished \ "git switch -f $GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME; \ git branch -D feature" && @@ -85,20 +415,275 @@ test_expect_success 'prints branches that cannot follow the squash' ' test_grep "^error: the following branches cannot be rewritten" err && test_grep "^ topic-1$" err && test_grep "^ topic-2$" err && - test_grep "^hint: .* --update-refs=head" err + test_grep "^hint: .* --update-refs=head" err && + test_cmp_rev C6 HEAD && + + # squash succeeds with --update-refs=head + git history squash --no-edit --update-refs=head start.. && + test_cmp_rev start HEAD^ && + test_cmp_rev C6^{tree} HEAD^{tree} && + test_cmp_rev C6 HEAD@{1} ' test_expect_success 'advice.historyUpdateRefs silences the hint' ' git reset --hard three && git branch -f mid HEAD~1 && + head_before=$(git rev-parse HEAD) && test_must_fail git -c advice.historyUpdateRefs=false \ history squash start.. 2>err && test_grep "^error: the following branches cannot be rewritten" err && test_grep "^ mid$" err && test_grep ! "hint:" err && + test_cmp_rev "$head_before" HEAD && git branch -D mid ' +test_expect_success 'leaves tags and remote-tracking refs unchanged' ' + git reset --hard three && + git tag -f mark HEAD~1 && + git update-ref refs/remotes/origin/mark HEAD~1 && + mark_before=$(git rev-parse mark) && + + git history squash --no-edit start.. && + + test_cmp_rev "$mark_before" mark && + test_cmp_rev "$mark_before" refs/remotes/origin/mark && + + git tag -d mark && + git update-ref -d refs/remotes/origin/mark +' + +test_expect_success 'squashes a range whose internal merge has a single base' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + test_commit --no-tag before-side file b && + git checkout -b inner-side && + test_commit --no-tag on-inner-side inner x && + git checkout "$main" && + test_commit --no-tag after-side file c && + git merge --no-ff -m merge inner-side && + git branch -D inner-side && + test_commit --no-tag after-merge file d && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start.. && + + check_commit_count start..HEAD 1 && + check_log_subjects -1 <<-\EOF && + before-side + EOF + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + test_path_is_file inner +' + +test_expect_success 'folds a merge of a branch that forked at the base' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + git checkout -b base-fork-side && + test_commit --no-tag base-fork-side side x && + git checkout "$main" && + test_commit --no-tag base-fork-main file b && + git merge --no-ff -m "merge base-fork-side" base-fork-side && + git branch -D base-fork-side && + test_commit --no-tag base-fork-tail file c && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start.. && + + check_commit_count start..HEAD 1 && + test_cmp_rev start HEAD^ && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + test_path_is_file side +' + +test_expect_success 'refuses a merge whose other parent is outside the range' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + git checkout -b outside-parent && + test_commit --no-tag outside-parent outside x && + git checkout "$main" && + test_commit --no-tag outside-main file b && + base=$(git rev-parse HEAD) && + test_commit --no-tag outside-mid file c && + git merge --no-ff -m "merge outside-parent" outside-parent && + git branch -D outside-parent && + merged=$(git rev-parse HEAD) && + + test_must_fail git history squash "$base.." 2>err && + test_grep "parent .* of commit .* is outside the revision range" err && + test_cmp_rev "$merged" HEAD +' + +test_expect_success 'folds a range whose tip is a merge commit' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + test_commit --no-tag tipmerge-base file b && + git checkout -b tipmerge-side && + test_commit --no-tag tipmerge-side side x && + git checkout "$main" && + test_commit --no-tag tipmerge-main file c && + git merge --no-ff -m "merge tipmerge-side" tipmerge-side && + git branch -D tipmerge-side && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start.. && + + check_commit_count start..HEAD 1 && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + test_path_is_file side +' + +test_expect_success 'folds a range whose base is a merge commit' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + git checkout -b basemerge-side && + test_commit --no-tag basemerge-side side x && + git checkout "$main" && + test_commit --no-tag basemerge-main file b && + git merge --no-ff -m "merge basemerge-side" basemerge-side && + git branch -D basemerge-side && + base=$(git rev-parse HEAD) && + test_commit --no-tag basemerge-one file c && + test_commit --no-tag basemerge-two file d && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit "$base.." && + + check_commit_count "$base..HEAD" 1 && + test_cmp_rev "$base" HEAD^ && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" +' + +test_expect_success 'folds a range with two interior merges' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + test_commit --no-tag two-merge-a file a1 && + git checkout -b two-merge-s1 && + test_commit --no-tag two-merge-s1 s1 x && + git checkout "$main" && + git merge --no-ff -m "merge s1" two-merge-s1 && + test_commit --no-tag two-merge-b file b1 && + git checkout -b two-merge-s2 && + test_commit --no-tag two-merge-s2 s2 y && + git checkout "$main" && + git merge --no-ff -m "merge s2" two-merge-s2 && + git branch -D two-merge-s1 two-merge-s2 && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start.. && + + check_commit_count start..HEAD 1 && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + test_path_is_file s1 && + test_path_is_file s2 +' + +test_expect_success 'folds a range with a nested merge' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + git checkout -b nested-outer && + test_commit --no-tag nested-outer outer x && + git checkout -b nested-inner && + test_commit --no-tag nested-inner inner y && + git checkout nested-outer && + git merge --no-ff -m "merge inner" nested-inner && + git checkout "$main" && + test_commit --no-tag nested-main file b1 && + git merge --no-ff -m "merge outer" nested-outer && + git branch -D nested-outer nested-inner && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start.. && + + check_commit_count start..HEAD 1 && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + test_path_is_file outer && + test_path_is_file inner +' + +test_expect_success 'folds a range with an octopus merge' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + test_commit --no-tag octo-base file a1 && + git checkout -b octo-1 && + test_commit --no-tag octo-1 o1 x && + git checkout "$main" && + git checkout -b octo-2 && + test_commit --no-tag octo-2 o2 y && + git checkout "$main" && + git merge --no-ff -m octopus octo-1 octo-2 && + git branch -D octo-1 octo-2 && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start.. && + + check_commit_count start..HEAD 1 && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + test_path_is_file o1 && + test_path_is_file o2 +' + +test_expect_success 'refuses an octopus merge with an arm forked before the base' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + git checkout -b octo-pre && + test_commit octo-pre-side pside x && + git checkout "$main" && + test_commit octo-pre-main file b1 && + octo_base=$(git rev-parse HEAD) && + git checkout -b octo-within && + test_commit --no-tag octo-within wside y && + git checkout "$main" && + git merge --no-ff -m octopus octo-pre octo-within && + merged=$(git rev-parse HEAD) && + git branch -D octo-pre octo-within && + + test_must_fail git history squash "$octo_base.." 2>err && + test_grep "parent .* of commit .* is outside the revision range" err && + test_cmp_rev "$merged" HEAD +' + +test_expect_success 'refuses when a descendant above the range is a merge' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + test_commit --no-tag desc-one file b && + test_commit --no-tag desc-two file c && + git tag desc-tip && + git checkout -b desc-above && + test_commit --no-tag desc-above above x && + git checkout "$main" && + test_commit --no-tag desc-main file d && + git merge --no-ff -m "merge desc-above" desc-above && + git branch -D desc-above && + head_before=$(git rev-parse HEAD) && + + test_must_fail git history squash --no-edit start..desc-tip 2>err && + test_grep "merge commits is not supported" err && + test_cmp_rev "$head_before" HEAD +' + +test_expect_success 'refuses to fold a range a ref points into at a merge' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + test_commit --no-tag refmerge-base file b && + git checkout -b refmerge-side && + test_commit --no-tag refmerge-side side x && + git checkout "$main" && + test_commit --no-tag refmerge-main file c && + git merge --no-ff -m "interior merge" refmerge-side && + git branch -D refmerge-side && + git branch at-merge HEAD && + test_commit --no-tag refmerge-tail file d && + head_before=$(git rev-parse HEAD) && + + test_must_fail git history squash start.. 2>err && + test_grep "^error: the following branches cannot be rewritten" err && + test_grep "^ at-merge$" err && + test_cmp_rev "$head_before" HEAD && + + git branch -D at-merge +' + test_done From de2f6f3b6e710f22518f1f0f477551bb6c032297 Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Fri, 7 Aug 2026 07:39:31 +0000 Subject: [PATCH 096/259] history: support editing squashed commit messages Open the editor by default when squashing and provide --no-edit as the opt-out. Record the exact commits selected by the revision walk, rearrange that todo list with the sequencer's autosquash machinery, and build the message template from the resulting order. Match interactive rebase's treatment of marker messages: comment out fixup! messages, retain squash! bodies, and let amend! replace its target unless a preceding squash! requires both bodies. This keeps message editing aligned with the marker validation used by the no-edit path. Helped-by: Phillip Wood Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- Documentation/git-history.adoc | 18 ++- builtin/history.c | 97 +++++++++++++++- t/t3455-history-squash.sh | 197 +++++++++++++++++++++++++++++++++ 3 files changed, 305 insertions(+), 7 deletions(-) diff --git a/Documentation/git-history.adoc b/Documentation/git-history.adoc index fb04a6768513f0..2e2e31f521bee9 100644 --- a/Documentation/git-history.adoc +++ b/Documentation/git-history.adoc @@ -133,8 +133,9 @@ given, for example `HEAD~3..HEAD ^topic` to additionally exclude what is already on `topic`. Rev-list options may also be given, but any that would change how the range is walked are overridden with a warning. + -With `--no-edit`, the oldest commit's message is preserved, except that an -`amend!` commit targeting it replaces its message. +An editor opens pre-filled with the messages of all the folded commits so you +can combine them. With `--no-edit`, the oldest commit's message is preserved +instead, except that an `amend!` commit targeting it replaces its message. + The selected commits must form a connected graph with a single tip and must not include a root commit. Every parent of a commit after the oldest one must @@ -148,6 +149,13 @@ of markers for one target is combined into a single commit. With `--no-edit`, the last `amend!` message is used if there is one; a `squash!` or `amend!` is otherwise refused if folding it would discard its message. + +The editor template mirrors `git rebase -i --autosquash`: each `fixup!`, +`squash!`, or `amend!` is grouped under the commit it targets rather than +shown in commit order. A `fixup!` message is dropped (commented out in full), +a `squash!` keeps its body with only the marker subject commented, and an +`amend!` replaces its target's message, unless a `squash!` folded into that +target first, in which case it keeps its body like a `squash!`. ++ A local branch descended from a selected commit but not from the range tip cannot be rewritten as a descendant of the result, so with the default `--update-refs=branches` the command refuses. Rerun with `--update-refs=head` @@ -163,6 +171,12 @@ OPTIONS objects will be written into the repository, so applying these printed ref updates is generally safe. +`--edit`:: +`--no-edit`:: + For `squash`, open an editor to combine the messages of the folded commits. + This is the default; use `--no-edit` to keep the selected message without + opening an editor. + `--reedit-message`:: Open an editor to modify the target commit's message. diff --git a/builtin/history.c b/builtin/history.c index 41cef2c9763380..c6b097025c00f3 100644 --- a/builtin/history.c +++ b/builtin/history.c @@ -1266,6 +1266,10 @@ static int squash_check_subject(struct repository *repo, return ret; } +static int build_squash_message(struct repository *repo, + const struct strbuf *todo_buf, + struct strbuf *out); + static int setup_squash_revisions(struct repository *repo, int argc, const char **argv, struct rev_info *revs) @@ -1319,6 +1323,7 @@ static int setup_squash_revisions(struct repository *repo, */ static int resolve_squash_range(struct repository *repo, bool update_branches, + bool edit_message, int argc, const char **argv, struct commit **oldest_out, struct commit **tip_out, @@ -1327,11 +1332,13 @@ static int resolve_squash_range(struct repository *repo, struct rev_info revs; struct subject_data subject_data = SUBJECT_DATA_INIT; struct commit *commit, *oldest = NULL, *tip = NULL; + struct strbuf todo_buf = STRBUF_INIT; int ret, tip_count = 0; bool walk_started = false; struct ref_filter filter = REF_FILTER_INIT; struct ref_array refs = { 0 }; + subject_data.edit_message = edit_message; ret = setup_squash_revisions(repo, argc, argv, &revs); if (ret < 0) goto out; @@ -1344,6 +1351,10 @@ static int resolve_squash_range(struct repository *repo, while ((commit = get_revision(&revs))) { struct commit_list *p; + if (edit_message) + strbuf_addf(&todo_buf, "pick %s\n", + oid_to_hex(&commit->object.oid)); + if (!commit->parents) { ret = error(_("cannot squash down to root commit")); goto out; @@ -1457,6 +1468,13 @@ static int resolve_squash_range(struct repository *repo, string_list_clear(&sorting_options, 0); goto out; } + if (edit_message) { + strbuf_reset(&subject_data.squash_message); + ret = build_squash_message(repo, &todo_buf, + &subject_data.squash_message); + if (ret < 0) + goto out; + } *oldest_out = oldest; *tip_out = tip; @@ -1464,6 +1482,7 @@ static int resolve_squash_range(struct repository *repo, ret = 0; out: + strbuf_release(&todo_buf); clear_object_flags(repo, SQUASH_SEEN | SQUASH_TIP | SQUASH_AMEND_TARGET); if (walk_started) @@ -1475,6 +1494,76 @@ static int resolve_squash_range(struct repository *repo, return ret; } +static bool amend_replaces_target(struct todo_list *todo, int target) +{ + for (int i = target + 1; i < todo->nr && + todo->items[i].command != TODO_PICK; i++) { + if (todo->items[i].command == TODO_SQUASH) + return false; + if (todo->items[i].flags & TODO_REPLACE_FIXUP_MSG) + return true; + } + return false; +} + +static int build_squash_message(struct repository *repo, + const struct strbuf *todo_buf, + struct strbuf *out) +{ + struct todo_list todo = TODO_LIST_INIT; + struct replay_opts opts = REPLAY_OPTS_INIT; + int nr_commits, ret; + + if (todo_list_parse_insn_buffer(repo, &opts, todo_buf->buf, &todo) < 0 || + todo_list_rearrange_squash(&todo) < 0) { + ret = error(_("could not prepare the squash message")); + goto out; + } + + nr_commits = todo.nr; + for (int i = 0; i < nr_commits; i++) { + struct todo_item *item = &todo.items[i]; + const char *message, *body; + size_t commented_len; + bool skip, squashing; + + squashing = item->command == TODO_SQUASH || + (item->flags & TODO_REPLACE_FIXUP_MSG); + if (item->command == TODO_PICK) + skip = amend_replaces_target(&todo, i); + else + skip = !squashing; + + message = repo_logmsg_reencode(repo, item->commit, NULL, NULL); + find_commit_subject(message, &body); + + if (skip) + commented_len = strlen(body); + else if (squashing) + commented_len = squash_subject_comment_len(body, 1); + else + commented_len = 0; + + if (!i) + add_squash_combination_header(out, nr_commits); + strbuf_addch(out, '\n'); + add_squash_message_header(out, i + 1, skip); + strbuf_addstr(out, "\n\n"); + strbuf_add_commented_lines(out, body, commented_len, comment_line_str); + strbuf_addstr(out, body + commented_len); + strbuf_complete_line(out); + + repo_unuse_commit_buffer(repo, item->commit, message); + } + + ret = 0; + +out: + todo_list_release(&todo); + replay_opts_release(&opts); + return ret; +} + static int cmd_history_squash(int argc, const char **argv, const char *prefix, @@ -1519,14 +1608,11 @@ static int cmd_history_squash(int argc, strbuf_join_argv(&reflog_msg, argc - 1, argv + 1, ' '); ret = resolve_squash_range(repo, action == REF_ACTION_BRANCHES, + edit, argc, argv, &oldest, &tip, &message_template); if (ret < 0) goto out; - if (edit) { - ret = error(_("message editing is not supported yet; use '--no-edit'")); - goto out; - } ret = setup_revwalk(repo, action, tip, &revs); if (ret < 0) @@ -1538,7 +1624,8 @@ static int cmd_history_squash(int argc, ret = commit_tree_ext(repo, "squash", oldest, message_template, oldest->parents, base_tree_oid, tip_tree_oid, - &rewritten, 0); + &rewritten, + edit ? COMMIT_TREE_EDIT_MESSAGE : 0); if (ret < 0) { ret = error(_("failed writing squashed commit")); goto out; diff --git a/t/t3455-history-squash.sh b/t/t3455-history-squash.sh index d2d8e5fdb9ce8f..591463cb8681d2 100755 --- a/t/t3455-history-squash.sh +++ b/t/t3455-history-squash.sh @@ -381,6 +381,203 @@ test_expect_success 'squashing fixups into a merge' ' sed 1,2d msg | test_commit_message HEAD ' +test_expect_success 'edits every message and aborts on an empty result' ' + git reset --hard start && + stage_file b && + git commit -m "re-one subject" -m "re-one body line" && + test_commit --no-tag re-two file c && + test_commit re-three file d && + head_before=$(git rev-parse HEAD) && + + write_script empty-editor <<-\EOF && + >"$1" + EOF + test_set_editor "$(pwd)/empty-editor" && + test_must_fail git history squash start.. 2>err && + test_grep "Aborting commit due to empty commit message" err && + test_cmp_rev "$head_before" HEAD && + + write_script editor <<-\EOF && + cat "$1" >edited && + echo combined >"$1" + EOF + test_set_editor "$(pwd)/editor" && + git history squash start.. && + + cat >expect <<-EOF && + # This is a combination of 3 commits. + # This is the 1st commit message: + + re-one subject + + re-one body line + + # This is the commit message #2: + + re-two + + # This is the commit message #3: + + re-three + + # Please enter the commit message for the squash changes. Lines starting + # with ${SQ}#${SQ} will be ignored, and an empty message aborts the commit. + # Changes to be committed: + # modified: file + # + EOF + test_cmp expect edited && + check_log_subjects -1 <<-\EOF + combined + EOF +' + +test_expect_success 'handles fixup!, squash! and amend! like rebase' ' + git reset --hard start && + test_commit --no-tag mark-base file b && + stage_file c && + commit_with_message "fixup! mark-base\n\nfixup body\n" && + stage_file d && + commit_with_message "squash! mark-base\n\nsquash remark\n" && + stage_file e && + commit_with_message "amend! mark-base\n\namended message\n" && + + write_script editor <<-\EOF && + cat "$1" >edited + EOF + test_set_editor "$(pwd)/editor" && + git history squash start.. && + + cat >expect <<-EOF && + # This is a combination of 4 commits. + # This is the 1st commit message: + + mark-base + + # The commit message #2 will be skipped: + + # fixup! mark-base + # + # fixup body + + # This is the commit message #3: + + # squash! mark-base + + squash remark + + # This is the commit message #4: + + # amend! mark-base + + amended message + + # Please enter the commit message for the squash changes. Lines starting + # with ${SQ}#${SQ} will be ignored, and an empty message aborts the commit. + # Changes to be committed: + # modified: file + # + EOF + test_cmp expect edited && + check_log_messages -1 <<-\EOF + mark-base + + squash remark + + amended message + + EOF +' + +test_expect_success 'groups fixups under their targets in the editor' ' + git reset --hard start && + test_commit --no-tag alpha file a1 && + test_commit --no-tag beta file b1 && + stage_file a2 && + commit_with_message "fixup! alpha\n" && + stage_file b2 && + commit_with_message "fixup! beta\n" && + + write_script editor <<-\EOF && + cat "$1" >edited + EOF + test_set_editor "$(pwd)/editor" && + git history squash start.. && + + cat >expect <<-EOF && + # This is a combination of 4 commits. + # This is the 1st commit message: + + alpha + + # The commit message #2 will be skipped: + + # fixup! alpha + + # This is the commit message #3: + + beta + + # The commit message #4 will be skipped: + + # fixup! beta + + # Please enter the commit message for the squash changes. Lines starting + # with ${SQ}#${SQ} will be ignored, and an empty message aborts the commit. + # Changes to be committed: + # modified: file + # + EOF + test_cmp expect edited +' + +test_expect_success 'lets amend! replace its target message in the editor' ' + git reset --hard start && + test_commit --no-tag mark-base file b && + stage_file c && + commit_with_message "amend! mark-base\n\namended message\n" && + stage_file d && + commit_with_message "squash! mark-base\n\nsquash remark\n" && + + write_script editor <<-\EOF && + cat "$1" >edited + EOF + test_set_editor "$(pwd)/editor" && + git history squash start.. && + + cat >expect <<-EOF && + # This is a combination of 3 commits. + # The 1st commit message will be skipped: + + # mark-base + + # This is the commit message #2: + + # amend! mark-base + + amended message + + # This is the commit message #3: + + # squash! mark-base + + squash remark + + # Please enter the commit message for the squash changes. Lines starting + # with ${SQ}#${SQ} will be ignored, and an empty message aborts the commit. + # Changes to be committed: + # modified: file + # + EOF + test_cmp expect edited && + check_log_messages -1 <<-\EOF + amended message + + squash remark + + EOF +' + test_expect_success '--update-refs=head only moves HEAD' ' git reset --hard three && git branch -f other HEAD && From 862564c961f173bc7bccff1848c96eb1fbb00247 Mon Sep 17 00:00:00 2001 From: Tian Yuchen Date: Fri, 7 Aug 2026 16:59:30 +0800 Subject: [PATCH 097/259] environment: drop redundant NULL checks in config getters These repository config getters require a valid repository pointer. While an uninitialized repository is a valid state and is handled by returning default values, passing NULL is a programming error. Drop the NULL checks so that invalid callers are not silently accepted. Mentored-by: Christian Couder Mentored-by: Ayush Chandekar Mentored-by: Olamide Caleb Bello Signed-off-by: Tian Yuchen Signed-off-by: Junio C Hamano --- environment.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/environment.c b/environment.c index 76ee65e62b4823..f5628b67580a1e 100644 --- a/environment.c +++ b/environment.c @@ -119,23 +119,23 @@ int is_bare_repository(struct repository *repo) int repo_protect_ntfs(struct repository *repo) { - return (repo && repo->initialized) ? - repo_config_values(repo)->protect_ntfs : - PROTECT_NTFS_DEFAULT; + return repo->initialized + ? repo_config_values(repo)->protect_ntfs + : PROTECT_NTFS_DEFAULT; } int repo_protect_hfs(struct repository *repo) { - return (repo && repo->initialized) ? - repo_config_values(repo)->protect_hfs : - PROTECT_HFS_DEFAULT; + return repo->initialized + ? repo_config_values(repo)->protect_hfs + : PROTECT_HFS_DEFAULT; } int repo_ignore_case(struct repository *repo) { - return (repo && repo->initialized) ? - repo_config_values(repo)->ignore_case : - 0; + return repo->initialized + ? repo_config_values(repo)->ignore_case + : 0; } int repo_trust_executable_bit(struct repository *repo) From a095d70a19a83d2b74a33060b779733c1b5bceac Mon Sep 17 00:00:00 2001 From: Tian Yuchen Date: Fri, 7 Aug 2026 16:59:31 +0800 Subject: [PATCH 098/259] environment: clarify repository config getter documentation Update the comment above repository config getters to describe their common behavior. The getters handle repositories that are not fully initialized by returning the corresponding default values. Mentored-by: Christian Couder Mentored-by: Ayush Chandekar Mentored-by: Olamide Caleb Bello Signed-off-by: Tian Yuchen Signed-off-by: Junio C Hamano --- environment.h | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/environment.h b/environment.h index e7ec5b0437342d..6f864c1635bb88 100644 --- a/environment.h +++ b/environment.h @@ -175,22 +175,15 @@ int git_default_core_config(const char *var, const char *value, const struct config_context *ctx, void *cb); /* - * Getters for the `protect_hfs` and `protect_ntfs` fields of `struct repo_config_values`. - * They check `repo->initialized` to prevent calling `repo_config_values()` - * before the repository setup is fully complete or in non-git environments. + * Getters for configuration variables in `struct repo_config_values`. + * These functions require a non-NULL repository pointer and handle + * repositories that are not fully initialized by returning appropriate + * default values. */ int repo_protect_hfs(struct repository *repo); int repo_protect_ntfs(struct repository *repo); - -/* - * Getter for the `ignore_case` field of `struct repo_config_values`. - * It checks `repo->initialized` to prevent calling repo_config_values()` - * before the repository setup is fully complete or in non-git environments. - */ int repo_ignore_case(struct repository *repo); - int repo_trust_executable_bit(struct repository *repo); - int repo_has_symlinks(struct repository *repo); const char *repo_excludes_file(struct repository *repo); From 9e50340a23aa9a67ba8c3f947c594e453269d378 Mon Sep 17 00:00:00 2001 From: Tian Yuchen Date: Fri, 7 Aug 2026 16:59:32 +0800 Subject: [PATCH 099/259] environment: remove inaccurate repo_config_values comments The section comments in struct repo_config_values do not accurately describe all members grouped under them. Remove them rather than implying a relationship that does not exist. Mentored-by: Christian Couder Mentored-by: Ayush Chandekar Mentored-by: Olamide Caleb Bello Signed-off-by: Tian Yuchen Signed-off-by: Junio C Hamano --- environment.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/environment.h b/environment.h index 6f864c1635bb88..67fd387d3571c3 100644 --- a/environment.h +++ b/environment.h @@ -115,7 +115,6 @@ enum object_creation_mode { }; struct repo_config_values { - /* section "core" config values */ char *attributes_file; char *excludes_file; char *editor_program; @@ -139,11 +138,7 @@ struct repo_config_values { int ignore_case; int trust_executable_bit; int has_symlinks; - - /* section "sparse" config values */ int sparse_expect_files_outside_of_patterns; - - /* section "branch" config values */ enum branch_track branch_track; }; From 0f7085824c29dc62f30381ad0e9fe503515131b9 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Fri, 7 Aug 2026 20:26:47 +0200 Subject: [PATCH 100/259] revision: move bloom keyvec precondition into function There are currently two callsites calling check_maybe_different_in_bloom_filter(). They both check if revs->bloom_keyvecs_nr is not zero before they call that function. Move bloom_keyvecs_nr precondition into check_maybe_different_in_bloom_filter() to simplify the code. Note that this changes `bloom_ret` to become -1 when there are no Bloom key vectors, which results in `count_bloom_filter_false_positive` not being incremented. This is unobservable, as the Bloom statistics are only reported when key vectors were set up. Signed-off-by: Toon Claes Signed-off-by: Junio C Hamano --- revision.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/revision.c b/revision.c index ccbe2e03d1406a..087d64d572c141 100644 --- a/revision.c +++ b/revision.c @@ -752,6 +752,9 @@ static int check_maybe_different_in_bloom_filter(struct rev_info *revs, struct bloom_filter *filter; int result = 0; + if (!revs->bloom_keyvecs_nr) + return -1; + if (commit_graph_generation(commit) == GENERATION_NUMBER_INFINITY) return -1; @@ -806,7 +809,7 @@ static int rev_compare_tree(struct rev_info *revs, return REV_TREE_SAME; } - if (revs->bloom_keyvecs_nr && !nth_parent) { + if (!nth_parent) { bloom_ret = check_maybe_different_in_bloom_filter(revs, commit); if (bloom_ret == 0) @@ -833,7 +836,7 @@ static int rev_same_tree_as_empty(struct rev_info *revs, struct commit *commit, if (!t1) return 0; - if (!nth_parent && revs->bloom_keyvecs_nr) { + if (!nth_parent) { bloom_ret = check_maybe_different_in_bloom_filter(revs, commit); if (!bloom_ret) return 1; From e44488eb16d03ac94211d84a6796befa1139bd72 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Fri, 7 Aug 2026 20:26:48 +0200 Subject: [PATCH 101/259] revision: expose check for paths maybe changed in Bloom filter check_maybe_different_in_bloom_filter() looks up a commit's changed-path Bloom filter and consults it to see whether the commit might have modified any of the paths in the pathspec that `revs` was set up with. In a follow-up commit we want to reuse this logic from another builtin. That caller, however, has already looked up the commit's Bloom filter for its own purposes, so having the function look it up again would mean a redundant lookup. Extract the filter-consulting part into a new public function, revs_maybe_changed_in_bloom(). This function takes an already looked-up `struct bloom_filter` instead of a commit. The existing check_maybe_different_in_bloom_filter() becomes a thin wrapper that looks up the filter and delegates. Expose the new function via revision.h so other builtins can reuse the exact same filtering that `git log ` performs. The existing function check_maybe_different_in_bloom_filter() returns a tristate value. This returns either: * `-1` : No Bloom filter was used. * `0` : The commit definitely did not change any of the paths. * `1` : The commit maybe changed one of the paths. These return values are used to keep count of false-positives. But because the new function revs_maybe_changed_in_bloom() is not involved in counting statistics, it returns a boolean value telling whether the commit definitely did not change any of the paths, or maybe changed some of them. Signed-off-by: Toon Claes Signed-off-by: Junio C Hamano --- revision.c | 30 ++++++++++++++++++++---------- revision.h | 12 ++++++++++++ 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/revision.c b/revision.c index 087d64d572c141..3628ef963da40d 100644 --- a/revision.c +++ b/revision.c @@ -750,7 +750,6 @@ static int check_maybe_different_in_bloom_filter(struct rev_info *revs, struct commit *commit) { struct bloom_filter *filter; - int result = 0; if (!revs->bloom_keyvecs_nr) return -1; @@ -765,18 +764,29 @@ static int check_maybe_different_in_bloom_filter(struct rev_info *revs, return -1; } - for (size_t nr = 0; !result && nr < revs->bloom_keyvecs_nr; nr++) { - result = bloom_filter_contains_vec(filter, - revs->bloom_keyvecs[nr], - revs->bloom_filter_settings); + if (revs_maybe_changed_in_bloom(revs, filter)) { + count_bloom_filter_maybe++; + return 1; } - if (result) - count_bloom_filter_maybe++; - else - count_bloom_filter_definitely_not++; + count_bloom_filter_definitely_not++; + + return 0; +} + +bool revs_maybe_changed_in_bloom(struct rev_info *revs, + struct bloom_filter *filter) +{ + if (!revs->bloom_keyvecs_nr || !filter) + return true; + + for (size_t nr = 0; nr < revs->bloom_keyvecs_nr; nr++) + if (bloom_filter_contains_vec(filter, + revs->bloom_keyvecs[nr], + revs->bloom_filter_settings)) + return true; - return result; + return false; } static int rev_compare_tree(struct rev_info *revs, diff --git a/revision.h b/revision.h index 569b3fa1cb5a33..14be39e20c53c6 100644 --- a/revision.h +++ b/revision.h @@ -68,6 +68,7 @@ struct string_list; struct saved_parents; struct follow_pathspec_slab; struct bloom_keyvec; +struct bloom_filter; struct bloom_filter_settings; struct option; struct parse_opt_ctx_t; @@ -493,6 +494,17 @@ void reset_revision_walk(void); */ int prepare_revision_walk(struct rev_info *revs); +/** + * Consult a changed-path Bloom filter to determine if the commit to which the + * filter belongs might have changed any of the paths in the `revs`. + * prepare_revision_walk() needs to be called in advance to ensure + * pathspec key vectors are set up. + * + * Returns false iff the commit definitely did not change any of the paths. + */ +bool revs_maybe_changed_in_bloom(struct rev_info *revs, + struct bloom_filter *filter); + /* Drain the commits linked list into the priority queue. */ void rev_info_commit_list_to_queue(struct rev_info *revs); /** From 814c55e1285cd91d15005ce27b40328faeb54535 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Fri, 7 Aug 2026 20:26:49 +0200 Subject: [PATCH 102/259] bloom: add helper to check if any key in a vector is present The changed-path Bloom filter of a commit stores a key for every changed path together with each of its leading directories. To query if a path was changed, bloom_keyvec_new() fills a key vector the same way: a key for the given path and one for each of its leading directories. For example, for "a/b/c" the vector holds keys for "a/b/c", "a/b" and "a". A Bloom filter can only ever prove absence. When a key is not in the filter, the path it was made for definitely did not change. When it is in the filter, the path may have changed, as the key can be a false positive. bloom_filter_contains_vec() looks up all keys of a vector and reports whether all of them are present. That answers: Is this path maybe changed by this commit? A caller that also cares about the directories containing the path asks a different question: Is this path, or any directory leading up to it, maybe changed by this commit? Consider the Bloom filter of a commit that changed "a/b/d". It holds keys for "a/b/d", "a/b" and "a", so looking up the vector of "a/b/c" with bloom_filter_contains_vec() reports that nothing changed, even though "a/b" and "a" did. Add bloom_filter_contains_any_vec(), which reports whether any key in the vector is present. It returns 0 only when none of the keys are in the filter, which means the path and all directories leading up to it definitely did not change. There are no callers yet, one is added in a subsequent commit. Signed-off-by: Toon Claes Signed-off-by: Junio C Hamano --- bloom.c | 12 ++++++++++++ bloom.h | 11 +++++++++++ 2 files changed, 23 insertions(+) diff --git a/bloom.c b/bloom.c index c98d1672adb71a..192692e548f029 100644 --- a/bloom.c +++ b/bloom.c @@ -607,6 +607,18 @@ int bloom_filter_contains_vec(const struct bloom_filter *filter, return ret; } +int bloom_filter_contains_any_vec(const struct bloom_filter *filter, + const struct bloom_keyvec *vec, + const struct bloom_filter_settings *settings) +{ + int ret = 0; + + for (size_t nr = 0; !ret && nr < vec->count; nr++) + ret = bloom_filter_contains(filter, &vec->key[nr], settings); + + return ret; +} + uint32_t test_bloom_murmur3_seeded(uint32_t seed, const char *data, size_t len, int version) { diff --git a/bloom.h b/bloom.h index 92ab2100d390e7..f508db23addf51 100644 --- a/bloom.h +++ b/bloom.h @@ -164,6 +164,17 @@ int bloom_filter_contains_vec(const struct bloom_filter *filter, const struct bloom_keyvec *v, const struct bloom_filter_settings *settings); +/* + * bloom_filter_contains_any_vec - Check if any key in a key vector is in the + * Bloom filter. + * + * Returns 1 if **any** key in the vector is present in the filter, 0 if none + * of them are. + */ +int bloom_filter_contains_any_vec(const struct bloom_filter *filter, + const struct bloom_keyvec *v, + const struct bloom_filter_settings *settings); + uint32_t test_bloom_murmur3_seeded(uint32_t seed, const char *data, size_t len, int version); From 7fc4c80a867948787d6a975ab691fa50a37e0483 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Fri, 7 Aug 2026 20:26:50 +0200 Subject: [PATCH 103/259] revision: add Bloom check that includes parent directories revs_maybe_changed_in_bloom() reports whether a commit may have changed any of the paths in the pathspec. It uses bloom_filter_contains_vec(), which requires all keys of a path's key vector to be present, so it only answers for the paths themselves. A caller may track more than those paths. git-last-modified(1) with --show-trees reports the last modifying commit for the tree entries containing the paths as well, up to the root. For a pathspec "a/b/c/" that means it reports "a" and "a/b" next to "a/b/c" and its entries, and those can each resolve to a different commit. A commit that only changed "a/top" is the answer for "a", even though it touched nothing under "a/b". Such a caller needs to know whether the path, or any of the directories leading up to it, may have changed. Add revs_maybe_changed_in_bloom_with_parents(), which asks that question by using bloom_filter_contains_any_vec() instead. A key vector holds a key for the path and one for each of its leading directories, so looking up any of them answers it. There are no callers yet, one is added in a subsequent commit. Signed-off-by: Toon Claes Signed-off-by: Junio C Hamano --- revision.c | 15 +++++++++++++++ revision.h | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/revision.c b/revision.c index 3628ef963da40d..b56608d144d282 100644 --- a/revision.c +++ b/revision.c @@ -789,6 +789,21 @@ bool revs_maybe_changed_in_bloom(struct rev_info *revs, return false; } +bool revs_maybe_changed_in_bloom_with_parents(struct rev_info *revs, + struct bloom_filter *filter) +{ + if (!revs->bloom_keyvecs_nr || !filter) + return true; + + for (size_t nr = 0; nr < revs->bloom_keyvecs_nr; nr++) + if (bloom_filter_contains_any_vec(filter, + revs->bloom_keyvecs[nr], + revs->bloom_filter_settings)) + return true; + + return false; +} + static int rev_compare_tree(struct rev_info *revs, struct commit *parent, struct commit *commit, int nth_parent) { diff --git a/revision.h b/revision.h index 14be39e20c53c6..58b60f9b3e7edc 100644 --- a/revision.h +++ b/revision.h @@ -505,6 +505,14 @@ int prepare_revision_walk(struct rev_info *revs); bool revs_maybe_changed_in_bloom(struct rev_info *revs, struct bloom_filter *filter); +/** + * Same as revs_maybe_changed_in_bloom(), but a change to any of the directories + * leading up to a path counts as well. Callers that track the tree entries + * containing the paths, and not just the paths themselves, need this. + */ +bool revs_maybe_changed_in_bloom_with_parents(struct rev_info *revs, + struct bloom_filter *filter); + /* Drain the commits linked list into the priority queue. */ void rev_info_commit_list_to_queue(struct rev_info *revs); /** From 885ea73988550def5d56a0346f39109a9f108684 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Fri, 7 Aug 2026 20:26:51 +0200 Subject: [PATCH 104/259] last-modified: check pathspec against Bloom filter first When git-last-modified(1) starts, it builds a list of all the paths matching the pathspec it needs to find the last modifying commit for. For example, every file and subdirectory listed by: $ git last-modified -t --max-depth=0 -- src/ As it resolves a commit for each path during the revision walk, it drops that path from the list. To avoid diffing trees for every commit, Bloom filters are used when available. For each remaining path, the commit's Bloom filter is checked to see whether the commit changed that path. The Bloom filter says either "no" or "maybe", and only in the latter case is the diff calculated. git-log(1) does this differently. It does not expand the pathspec but checks the Bloom filter against the pathspec itself. This way, commits not touching any path matching the pathspec can be discarded as a whole. Apply this same check to git-last-modified(1). In a previous commit the function revs_maybe_changed_in_bloom(), used by git-log(1), was made public. Use this as a pre-filter in git-last-modified(1). After this pre-filter, paths are still checked one-by-one to only find those which don't have a "last commit" yet. With `--show-trees` the list holds more than the paths matching the pathspec. It also holds each parent tree entry, up to the root. Each of those can resolve to a different commit. Thus for the pathspec "a/b/c", the list will also hold "a" and "a/b". When a commit touches "a/other", that commit could be the last commit for "a", but revs_maybe_changed_in_bloom() would discard it, because it doesn't match the full pathspec. Instead, when `--show-trees` is given, use revs_maybe_changed_in_bloom_with_parents(), which indicates the commit maybe changed any of the paths leading up to the path in the pathspec. Signed-off-by: Toon Claes Signed-off-by: Junio C Hamano --- builtin/last-modified.c | 12 ++++++++++++ t/t8020-last-modified.sh | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/builtin/last-modified.c b/builtin/last-modified.c index 5478182f2e95c2..5678731a0487de 100644 --- a/builtin/last-modified.c +++ b/builtin/last-modified.c @@ -272,6 +272,18 @@ static bool maybe_changed_path(struct last_modified *lm, if (!filter) return true; + /* + * With --show-trees we also track the tree entries containing the + * paths, so a change to any of those parent directories matters too. + */ + if (lm->show_trees) { + if (!revs_maybe_changed_in_bloom_with_parents(&lm->rev, filter)) + return false; + } else { + if (!revs_maybe_changed_in_bloom(&lm->rev, filter)) + return false; + } + hashmap_for_each_entry(&lm->paths, &iter, ent, hashent) { if (active && !bitmap_get(active, ent->diff_idx)) continue; diff --git a/t/t8020-last-modified.sh b/t/t8020-last-modified.sh index 9dba4b9d906808..df73c7d0d0982a 100755 --- a/t/t8020-last-modified.sh +++ b/t/t8020-last-modified.sh @@ -269,6 +269,27 @@ test_expect_success 'last-modified merge undoes changes' ' EOF ' +test_expect_success 'last-modified with Bloom filters and --show-trees' ' + test_when_finished rm -rf bloom && + git init bloom && + ( + cd bloom && + mkdir d && + test_commit base-a d/a && + test_commit base-b d/b && + test_commit touch-a d/a && + test_commit touch-b d/b && + + git commit-graph write --reachable --changed-paths && + git -c core.commitGraph=false last-modified -t HEAD -- d/a \ + >expect && + git -c core.commitGraph=true last-modified -t HEAD -- d/a \ + >actual && + + test_cmp expect actual + ) +' + test_expect_success 'cannot run last-modified on two commits' ' test_must_fail git last-modified HEAD HEAD~1 2>err && test_grep "last-modified can only operate on one commit at a time" err From 57bc4cb31c67e03db901c860db19fd9a98561654 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Fri, 7 Aug 2026 20:26:52 +0200 Subject: [PATCH 105/259] last-modified: keep per-path Bloom filters for wildcard pathspecs The last-modified builtin expands the pathspec to a set of literal paths and builds a Bloom key for each. During the walk it looks those keys up in the commit's filter to decide whether the commit is worth diffing. These lookups need `bloom_filter_settings` for the key hashing. prepare_revision_walk() runs prepare_to_use_bloom_filter() to build the pathspec key vectors. For a pathspec that cannot be turned into a Bloom key, such as a top-level wildcard like "*.c", that function gives up and clears `bloom_filter_settings`. Restore `bloom_filter_settings` after prepare_revision_walk() so the per-path check keeps working for wildcard pathspecs. Signed-off-by: Toon Claes Signed-off-by: Junio C Hamano --- builtin/last-modified.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/builtin/last-modified.c b/builtin/last-modified.c index 5678731a0487de..35d9dccd9bca6d 100644 --- a/builtin/last-modified.c +++ b/builtin/last-modified.c @@ -369,6 +369,14 @@ static int last_modified_run(struct last_modified *lm) prepare_revision_walk(&lm->rev); + /* + * prepare_revision_walk() clears bloom_filter_settings for pathspecs + * without a Bloom key. Restore it so the per-path check keeps working. + */ + if (!lm->rev.bloom_filter_settings) + lm->rev.bloom_filter_settings = + get_bloom_filter_settings(lm->rev.repo); + max_count = lm->rev.max_count; init_active_paths_for_commit(&lm->active_paths); From c3a8f4303c9e7b72ab506d58177648cd90026be6 Mon Sep 17 00:00:00 2001 From: Pablo Sabater Date: Sat, 8 Aug 2026 02:02:16 +0200 Subject: [PATCH 106/259] t5701: use test_file_size() to get the size of a file The 'basics of object-info' test runs 'wc -c | xargs' twice to get the size of two.t. The pipe to xargs is only there to strip the blanks that some platforms pad the output of wc with. Use the test_file_size() helper, which outputs the size directly, and store the result in a variable. Because 'git rev-parse two:two.t' is also run multiple times, store its output in a variable as well. Storing them in variables outside the HERE-document has the added benefit of preserving their exit statuses. Mentored-by: Karthik Nayak Mentored-by: Chandra Pratap Signed-off-by: Pablo Sabater Signed-off-by: Junio C Hamano --- t/t5701-git-serve.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh index 9a575aa098afd3..51d5dd1ae6f389 100755 --- a/t/t5701-git-serve.sh +++ b/t/t5701-git-serve.sh @@ -344,20 +344,23 @@ test_expect_success 'unexpected lines are not allowed in fetch request' ' test_expect_success 'basics of object-info' ' test_config transfer.advertiseObjectInfo true && + two_oid=$(git rev-parse two:two.t) && + two_size=$(test_file_size two.t) && + test-tool pkt-line pack >in <<-EOF && command=object-info object-format=$(test_oid algo) 0001 size - oid $(git rev-parse two:two.t) - oid $(git rev-parse two:two.t) + oid $two_oid + oid $two_oid 0000 EOF cat >expect <<-EOF && size - $(git rev-parse two:two.t) $(wc -c Date: Sat, 8 Aug 2026 02:02:17 +0200 Subject: [PATCH 107/259] fetch-object-info: detect malformed server responses The loop reading the object-info response stops as soon as the reader returns something other than PACKET_READ_NORMAL, or once it has read as many lines as we requested. Neither end is checked. A server that answers with fewer objects leaves the end of the result arrays empty, and the caller trusts that every requested object was filled in. A server that answers with more leaves the extra packets unread. On stateless transports check_stateless_delimiter() notices, but on the others it passes unnoticed. Check both limits by extracting the packet_reader_read() from the loop condition, so the loop no longer consumes the last packet (flush). If while looping the read is different from a PACKET_READ_NORMAL, die() meaning there are fewer objects than expected. After iterating, we only expect a flush, so if the last packet is not a flush, die(). Helped-by: Junio C Hamano Mentored-by: Karthik Nayak Mentored-by: Chandra Pratap Signed-off-by: Pablo Sabater Signed-off-by: Junio C Hamano --- fetch-object-info.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fetch-object-info.c b/fetch-object-info.c index ba7e179c44ee54..287f668a3cc07b 100644 --- a/fetch-object-info.c +++ b/fetch-object-info.c @@ -106,12 +106,13 @@ int fetch_object_info(const enum protocol_version version, struct object_info_ar } } - for (size_t i = 0; - packet_reader_read(reader) == PACKET_READ_NORMAL && - i < args->oids->nr; - i++) { + for (size_t i = 0; i < args->oids->nr; i++) { struct string_list object_info_values = STRING_LIST_INIT_DUP; + if (packet_reader_read(reader) != PACKET_READ_NORMAL) + die(_("object-info: expected %" PRIuMAX " objects, got %" PRIuMAX), + (uintmax_t)args->oids->nr, (uintmax_t)i); + string_list_split(&object_info_values, reader->line, " ", -1); if (strcmp(object_info_values.items[0].string, @@ -150,6 +151,11 @@ int fetch_object_info(const enum protocol_version version, struct object_info_ar string_list_clear(&object_info_values, 0); } + + if (packet_reader_read(reader) != PACKET_READ_FLUSH) + die(_("object-info: expected flush after %" PRIuMAX " objects"), + (uintmax_t)args->oids->nr); + check_stateless_delimiter(stateless_rpc, reader, "stateless delimiter expected"); return 0; From d43b4d3fa10ee596c1bb6517f30d6f4daad906e4 Mon Sep 17 00:00:00 2001 From: Pablo Sabater Date: Sat, 8 Aug 2026 02:02:18 +0200 Subject: [PATCH 108/259] fetch-object-info: pass arguments directly instead of a struct struct object_info_args groups three pointers that already live in the transport and are given to fetch_object_info(). Grouping them into a struct reduces the number of parameters, but it suggests that the three belong together, when they are unrelated and end up being accessed as args->* independently. Drop the struct and pass those parameters directly to fetch_object_info() and send_object_info_request(). This should have no change in behavior. Helped-by: Jeff King Helped-by: Junio C Hamano Mentored-by: Karthik Nayak Mentored-by: Chandra Pratap Signed-off-by: Pablo Sabater Signed-off-by: Junio C Hamano --- fetch-object-info.c | 53 ++++++++++++++++++++++++++------------------- fetch-object-info.h | 17 +++++++-------- transport.c | 11 +++++----- 3 files changed, 44 insertions(+), 37 deletions(-) diff --git a/fetch-object-info.c b/fetch-object-info.c index 287f668a3cc07b..53eec88cf0de10 100644 --- a/fetch-object-info.c +++ b/fetch-object-info.c @@ -9,20 +9,24 @@ #include "string-list.h" /* Sends object-info command and its arguments into the request buffer. */ -static void send_object_info_request(const int fd_out, struct object_info_args *args) +static void send_object_info_request(const int fd_out, + const struct string_list *server_options, + struct oid_array *oids, + struct string_list *object_info_options) { struct strbuf req_buf = STRBUF_INIT; - write_command_and_capabilities(&req_buf, "object-info", args->server_options); + write_command_and_capabilities(&req_buf, "object-info", server_options); - if (unsorted_string_list_has_string(args->object_info_options, "size")) + if (unsorted_string_list_has_string(object_info_options, "size")) packet_buf_write(&req_buf, "size"); - else if (args->object_info_options->nr) + else if (object_info_options->nr) BUG("only size should be in object_info_options"); - if (args->oids) - for (size_t i = 0; i < args->oids->nr; i++) - packet_buf_write(&req_buf, "oid %s", oid_to_hex(&args->oids->oid[i])); + if (oids) + for (size_t i = 0; i < oids->nr; i++) + packet_buf_write(&req_buf, "oid %s", + oid_to_hex(&oids->oid[i])); packet_buf_flush(&req_buf); if (write_in_full(fd_out, req_buf.buf, req_buf.len) < 0) @@ -45,8 +49,12 @@ static int parse_object_size(const char *s, size_t *res) return 0; } -int fetch_object_info(const enum protocol_version version, struct object_info_args *args, - struct packet_reader *reader, struct object_info *object_info_data, +int fetch_object_info(const enum protocol_version version, + const struct string_list *server_options, + struct oid_array *oids, + struct string_list *object_info_options, + struct packet_reader *reader, + struct object_info *object_info_data, const int stateless_rpc, const int fd_out) { int size_index = -1; @@ -64,16 +72,17 @@ int fetch_object_info(const enum protocol_version version, struct object_info_ar * because the number of options is a small known number (the * supported placeholders which currently are size and type). */ - for (int i = (int)args->object_info_options->nr - 1; i >= 0; i--) + for (int i = (int)object_info_options->nr - 1; i >= 0; i--) if (!server_supports_feature("object-info", - args->object_info_options->items[i].string, 0)) - unsorted_string_list_delete_item(args->object_info_options, i, 0); + object_info_options->items[i].string, 0)) + unsorted_string_list_delete_item(object_info_options, i, 0); /* * Even if no options are left, we still send the oid so we get * at least an existence check. */ - send_object_info_request(fd_out, args); + send_object_info_request(fd_out, server_options, oids, + object_info_options); break; case protocol_v1: case protocol_v0: @@ -82,14 +91,14 @@ int fetch_object_info(const enum protocol_version version, struct object_info_ar BUG("unknown protocol version"); } - for (size_t i = 0; i < args->object_info_options->nr; i++) { + for (size_t i = 0; i < object_info_options->nr; i++) { if (packet_reader_read(reader) != PACKET_READ_NORMAL) { check_stateless_delimiter(stateless_rpc, reader, "stateless delimiter expected"); return -1; } - if (!unsorted_string_list_has_string(args->object_info_options, reader->line)) + if (!unsorted_string_list_has_string(object_info_options, reader->line)) return -1; if (!strcmp(reader->line, "size")) { @@ -98,7 +107,7 @@ int fetch_object_info(const enum protocol_version version, struct object_info_ar * is only size. No risk of overflow. */ size_index = (int)i; - for (size_t j = 0; j < args->oids->nr; j++) + for (size_t j = 0; j < oids->nr; j++) object_info_data[j].sizep = xcalloc(1, sizeof(*object_info_data[j].sizep)); } else { @@ -106,19 +115,19 @@ int fetch_object_info(const enum protocol_version version, struct object_info_ar } } - for (size_t i = 0; i < args->oids->nr; i++) { + for (size_t i = 0; i < oids->nr; i++) { struct string_list object_info_values = STRING_LIST_INIT_DUP; if (packet_reader_read(reader) != PACKET_READ_NORMAL) die(_("object-info: expected %" PRIuMAX " objects, got %" PRIuMAX), - (uintmax_t)args->oids->nr, (uintmax_t)i); + (uintmax_t)oids->nr, (uintmax_t)i); string_list_split(&object_info_values, reader->line, " ", -1); if (strcmp(object_info_values.items[0].string, - oid_to_hex(&args->oids->oid[i]))) + oid_to_hex(&oids->oid[i]))) die(_("object-info: expected OID: %s, got %s"), - oid_to_hex(&args->oids->oid[i]), + oid_to_hex(&oids->oid[i]), object_info_values.items[0].string); /* @@ -138,7 +147,7 @@ int fetch_object_info(const enum protocol_version version, struct object_info_ar * the server we expect the server to answer with the same * number of attributes requested. */ - if (args->object_info_options->nr + 1 != object_info_values.nr) + if (object_info_options->nr + 1 != object_info_values.nr) die("object-info: unexpected number of attributes: %s", reader->line); @@ -154,7 +163,7 @@ int fetch_object_info(const enum protocol_version version, struct object_info_ar if (packet_reader_read(reader) != PACKET_READ_FLUSH) die(_("object-info: expected flush after %" PRIuMAX " objects"), - (uintmax_t)args->oids->nr); + (uintmax_t)oids->nr); check_stateless_delimiter(stateless_rpc, reader, "stateless delimiter expected"); diff --git a/fetch-object-info.h b/fetch-object-info.h index 269cebb3f7df48..316bf917ce2d5c 100644 --- a/fetch-object-info.h +++ b/fetch-object-info.h @@ -4,22 +4,21 @@ #include "pkt-line.h" #include "protocol.h" -struct object_info_args { - struct string_list *object_info_options; - const struct string_list *server_options; - struct oid_array *oids; -}; - struct object_info; +struct oid_array; /* * Sends git-cat-file object-info command into the request buf and read the * results from packets. * - * Modifies args->object_info_options, on return it contains only the supported + * Modifies object_info_options, on return it contains only the supported * options by the server. */ -int fetch_object_info(enum protocol_version version, struct object_info_args *args, - struct packet_reader *reader, struct object_info *object_info_data, +int fetch_object_info(enum protocol_version version, + const struct string_list *server_options, + struct oid_array *oids, + struct string_list *object_info_options, + struct packet_reader *reader, + struct object_info *object_info_data, int stateless_rpc, int fd_out); #endif /* FETCH_OBJECT_INFO_H */ diff --git a/transport.c b/transport.c index f0a6a455479800..c6df56129d7c8d 100644 --- a/transport.c +++ b/transport.c @@ -438,11 +438,6 @@ static int fetch_object_info_via_pack(struct transport *transport) int ret = 0; struct git_transport_data *data = transport->data; struct packet_reader reader; - struct object_info_args args = { 0 }; - - args.server_options = transport->server_options; - args.oids = transport->smart_options->object_info_oids; - args.object_info_options = transport->smart_options->object_info_options; connect_setup(transport, 0); packet_reader_init(&reader, data->fd[0], NULL, 0, @@ -453,7 +448,11 @@ static int fetch_object_info_via_pack(struct transport *transport) data->version = discover_version(&reader); transport->hash_algo = reader.hash_algo; - ret = fetch_object_info(data->version, &args, &reader, + ret = fetch_object_info(data->version, + transport->server_options, + transport->smart_options->object_info_oids, + transport->smart_options->object_info_options, + &reader, data->options.object_info_data, transport->stateless_rpc, data->fd[1]); From c5c971d967617592d27c1b7db72455f9277fad47 Mon Sep 17 00:00:00 2001 From: Pablo Sabater Date: Sat, 8 Aug 2026 02:02:19 +0200 Subject: [PATCH 109/259] fetch-object-info: use dedicated struct for the results fetch_object_info() collects information about N objects, but it stores the results in an array of object_info. That struct holds the extended parameters of read_object_info() (The optional outputs the caller wants filled). Its pointers tell that function where to write the answers for a single object. object_info is not meant to be the final storage, and since fetch_object_info() does not call read_object_info(), there is no reason to use it. Using it means allocating one scalar per object per attribute just to have those pointers somewhere to point at. Add struct fetch_object_info_results. The caller sets the wants_* flags to say what it is interested in, and fetch_object_info() allocates one array per attribute. A set wants_* flag means "asked for", while a non-NULL array means "available". The caller releases the arrays with free_fetch_object_info_results(). The object_info_options string list is no longer needed. Filtering against the server's advertisement now sets local ask_* flags, and send_object_info_request() turns those into the v2 protocol option strings. remote_atom_map[] existed only to map those strings back into atom names, so drop it and build remote_allowed_atoms from the result arrays. Currently for wants_* and ask_* there is only the 'size' variant but a subsequent commit will add '*_type'. free_object_info_contents() loses its only caller and is dropped. Dropping the allow-list check makes the final else reachable from the wire, so die() instead of BUG(): an unknown attribute is the server's error, not ours. Helped-by: Jeff King Helped-by: Junio C Hamano Mentored-by: Karthik Nayak Mentored-by: Chandra Pratap Signed-off-by: Pablo Sabater Signed-off-by: Junio C Hamano --- builtin/cat-file.c | 59 ++++++++------------------------- fetch-object-info.c | 81 ++++++++++++++++++++++----------------------- fetch-object-info.h | 27 +++++++++++---- object-file.c | 10 ------ odb.h | 3 -- transport.c | 3 +- transport.h | 5 +-- 7 files changed, 77 insertions(+), 111 deletions(-) diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 884b6d5ad348b5..e1650b2921ffcf 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -31,6 +31,7 @@ #include "alias.h" #include "remote.h" #include "transport.h" +#include "fetch-object-info.h" /* * Maximum length for a remote URL. While no universal standard exists, @@ -681,9 +682,8 @@ static void batch_one_object(const char *obj_name, static int get_remote_info(int argc, const char **argv, - struct object_info **remote_object_info, - struct oid_array *object_info_oids, - struct string_list *object_info_options) + struct fetch_object_info_results *results, + struct oid_array *object_info_oids) { int retval = 0; struct remote *remote = NULL; @@ -724,11 +724,9 @@ static int get_remote_info(int argc, goto cleanup; } - CALLOC_ARRAY(*remote_object_info, object_info_oids->nr); gtransport->smart_options->object_info_oids = object_info_oids; - gtransport->smart_options->object_info_options = object_info_options; - gtransport->smart_options->object_info_data = *remote_object_info; + gtransport->smart_options->object_info_results = results; retval = transport_fetch_object_info(gtransport); cleanup: transport_disconnect(gtransport); @@ -816,21 +814,6 @@ static void parse_cmd_mailmap(struct batch_options *opt UNUSED, load_mailmap(); } -struct protocol_placeholder_entry { - const char *option; - const char *atom; -}; - -static const struct protocol_placeholder_entry remote_atom_map[] = { - {"size", "objectsize"}, - {"type", "objecttype"}, - /* - * Add new protocol options here. Even if the server doesn't support - * them the allow_list will drop them if the server doesn't advertise - * them. - */ -}; - static void parse_cmd_remote_object_info(struct batch_options *opt, const char *line, struct strbuf *output, struct expand_data *data) @@ -838,9 +821,8 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, int count; const char **argv; char *line_to_split; - struct object_info *remote_object_info = NULL; + struct fetch_object_info_results results = FETCH_OBJECT_INFO_RESULTS_INIT; struct oid_array object_info_oids = OID_ARRAY_INIT; - struct string_list object_info_options = STRING_LIST_INIT_NODUP; const char *saved_format = opt->format; if (strlen(line) >= MAX_REMOTE_OBJ_INFO_LINE) @@ -861,26 +843,21 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, MAX_ALLOWED_OBJ_LIMIT); if (data->info.sizep) - string_list_append(&object_info_options, "size"); - if (data->info.typep) - string_list_append(&object_info_options, "type"); + results.wants_size = 1; - if (get_remote_info(count, argv, &remote_object_info, - &object_info_oids, &object_info_options)) + if (get_remote_info(count, argv, &results, &object_info_oids)) die(_("failed to get object info from the remote: %s"), argv[0]); string_list_clear(&data->remote_allowed_atoms, 0); string_list_append(&data->remote_allowed_atoms, "objectname"); - for (size_t i = 0; i < ARRAY_SIZE(remote_atom_map); i++) - if (unsorted_string_list_has_string(&object_info_options, remote_atom_map[i].option)) - string_list_append(&data->remote_allowed_atoms, - remote_atom_map[i].atom); + if (results.sizes) + string_list_append(&data->remote_allowed_atoms, "objectsize"); data->skip_object_info = 1; - for (size_t i = 0; i < object_info_oids.nr; i++) { + for (size_t i = 0; i < results.nr; i++) { data->oid = object_info_oids.oid[i]; - if (remote_object_info[i].unrecognized) { + if (results.unrecognized[i]) { report_object_status(opt, oid_to_hex(&data->oid), &data->oid, "missing"); continue; @@ -890,13 +867,8 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, * When reaching here, it means remote-object-info can retrieve * information from server without downloading them. */ - if (remote_object_info[i].sizep) { - data->size = *remote_object_info[i].sizep; - } - - if (remote_object_info[i].typep) { - data->type = *remote_object_info[i].typep; - } + if (results.sizes) + data->size = results.sizes[i]; opt->batch_mode = BATCH_MODE_INFO; data->is_remote = 1; @@ -906,12 +878,9 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, data->skip_object_info = 0; opt->format = saved_format; - for (size_t i = 0; i < object_info_oids.nr; i++) - free_object_info_contents(&remote_object_info[i]); - string_list_clear(&object_info_options, 0); + free_fetch_object_info_results(&results); free(line_to_split); free(argv); - free(remote_object_info); oid_array_clear(&object_info_oids); } diff --git a/fetch-object-info.c b/fetch-object-info.c index 53eec88cf0de10..5f53dbd6b90109 100644 --- a/fetch-object-info.c +++ b/fetch-object-info.c @@ -12,16 +12,14 @@ static void send_object_info_request(const int fd_out, const struct string_list *server_options, struct oid_array *oids, - struct string_list *object_info_options) + unsigned ask_size) { struct strbuf req_buf = STRBUF_INIT; write_command_and_capabilities(&req_buf, "object-info", server_options); - if (unsorted_string_list_has_string(object_info_options, "size")) + if (ask_size) packet_buf_write(&req_buf, "size"); - else if (object_info_options->nr) - BUG("only size should be in object_info_options"); if (oids) for (size_t i = 0; i < oids->nr; i++) @@ -52,37 +50,32 @@ static int parse_object_size(const char *s, size_t *res) int fetch_object_info(const enum protocol_version version, const struct string_list *server_options, struct oid_array *oids, - struct string_list *object_info_options, struct packet_reader *reader, - struct object_info *object_info_data, - const int stateless_rpc, const int fd_out) + struct fetch_object_info_results *results, + const int stateless_rpc, + const int fd_out) { + unsigned ask_size = 0; int size_index = -1; + size_t wanted; + + results->nr = oids->nr; + CALLOC_ARRAY(results->unrecognized, results->nr); switch (version) { case protocol_v2: if (!server_supports_v2("object-info")) die(_("object-info capability is not enabled on the server")); - /* - * When removing an element from the list it gets swapped by the - * last element, iterate backwards to prevent elements skipping - * evaluation. - * - * object_info_options->nr can be safely casted without overflow - * because the number of options is a small known number (the - * supported placeholders which currently are size and type). - */ - for (int i = (int)object_info_options->nr - 1; i >= 0; i--) - if (!server_supports_feature("object-info", - object_info_options->items[i].string, 0)) - unsorted_string_list_delete_item(object_info_options, i, 0); + + if (results->wants_size && + server_supports_feature("object-info", "size", 0)) + ask_size = 1; /* * Even if no options are left, we still send the oid so we get * at least an existence check. */ - send_object_info_request(fd_out, server_options, oids, - object_info_options); + send_object_info_request(fd_out, server_options, oids, ask_size); break; case protocol_v1: case protocol_v0: @@ -90,28 +83,25 @@ int fetch_object_info(const enum protocol_version version, case protocol_unknown_version: BUG("unknown protocol version"); } + wanted = ask_size; - for (size_t i = 0; i < object_info_options->nr; i++) { + for (size_t i = 0; i < wanted; i++) { if (packet_reader_read(reader) != PACKET_READ_NORMAL) { check_stateless_delimiter(stateless_rpc, reader, "stateless delimiter expected"); return -1; } - if (!unsorted_string_list_has_string(object_info_options, reader->line)) - return -1; - if (!strcmp(reader->line, "size")) { - /* - * i is the number of supported options which currently - * is only size. No risk of overflow. - */ + if (!ask_size) + die(_("object-info: unrequested 'size' attribute")); + if (results->sizes) + die(_("object-info: duplicate 'size' attribute")); size_index = (int)i; - for (size_t j = 0; j < oids->nr; j++) - object_info_data[j].sizep = - xcalloc(1, sizeof(*object_info_data[j].sizep)); + CALLOC_ARRAY(results->sizes, results->nr); } else { - BUG("only size is supported"); + die(_("object-info: unknown attribute '%s'"), + reader->line); } } @@ -137,24 +127,24 @@ int fetch_object_info(const enum protocol_version version, */ if (object_info_values.nr >= 2 && !strcmp(object_info_values.items[1].string, "")) { - object_info_data[i].unrecognized = 1; + results->unrecognized[i] = 1; string_list_clear(&object_info_values, 0); continue; } /* - * Because we filter the options to be only the supported by - * the server we expect the server to answer with the same - * number of attributes requested. + * Because we only ask for attributes the server said it + * supports, we expect the answer to have one value per + * requested attribute, plus the OID. */ - if (object_info_options->nr + 1 != object_info_values.nr) + if (wanted + 1 != object_info_values.nr) die("object-info: unexpected number of attributes: %s", reader->line); - if (size_index >= 0 && + if (results->sizes && parse_object_size(object_info_values.items[size_index + 1].string, - object_info_data[i].sizep)) - die("object-info: ref %s has invalid size %s", + &results->sizes[i])) + die("object-info: object %s has invalid size %s", object_info_values.items[0].string, object_info_values.items[size_index + 1].string); @@ -169,3 +159,10 @@ int fetch_object_info(const enum protocol_version version, return 0; } + +void free_fetch_object_info_results(struct fetch_object_info_results *results) +{ + free(results->sizes); + free(results->unrecognized); + memset(results, 0, sizeof(*results)); +} diff --git a/fetch-object-info.h b/fetch-object-info.h index 316bf917ce2d5c..9f72e91155336f 100644 --- a/fetch-object-info.h +++ b/fetch-object-info.h @@ -4,21 +4,34 @@ #include "pkt-line.h" #include "protocol.h" -struct object_info; +struct fetch_object_info_results { + size_t *sizes; + uint8_t *unrecognized; + size_t nr; + unsigned wants_size:1; +}; + +#define FETCH_OBJECT_INFO_RESULTS_INIT { 0 } + struct oid_array; /* - * Sends git-cat-file object-info command into the request buf and read the + * Sends git-cat-file object-info command into the request buf and reads the * results from packets. * - * Modifies object_info_options, on return it contains only the supported - * options by the server. + * The caller sets the wants_* flags in "results" to indicate which attributes + * it is interested in. On return, "results" holds one array per attribute that + * the server both advertised and answered with. An array left NULL means the + * attribute is not available. + * Release them with free_fetch_object_info_results(). */ int fetch_object_info(enum protocol_version version, const struct string_list *server_options, struct oid_array *oids, - struct string_list *object_info_options, struct packet_reader *reader, - struct object_info *object_info_data, - int stateless_rpc, int fd_out); + struct fetch_object_info_results *results, + int stateless_rpc, + int fd_out); + +void free_fetch_object_info_results(struct fetch_object_info_results *results); #endif /* FETCH_OBJECT_INFO_H */ diff --git a/object-file.c b/object-file.c index c5809db598703d..7ff2b730ac0ead 100644 --- a/object-file.c +++ b/object-file.c @@ -1740,13 +1740,3 @@ int odb_transaction_files_begin(struct odb_source *source, return 0; } - -void free_object_info_contents(struct object_info *object_info) -{ - if (!object_info) - return; - free(object_info->typep); - free(object_info->sizep); - free(object_info->disk_sizep); - free(object_info->delta_base_oid); -} diff --git a/odb.h b/odb.h index 3f7c48365675d1..b7bc0ee8443e9e 100644 --- a/odb.h +++ b/odb.h @@ -635,7 +635,4 @@ void parse_alternates(const char *string, const char *relative_base, struct strvec *out); -/* Free pointers inside of object_info, but not object_info itself */ -void free_object_info_contents(struct object_info *object_info); - #endif /* ODB_H */ diff --git a/transport.c b/transport.c index c6df56129d7c8d..35d3e98d9739a7 100644 --- a/transport.c +++ b/transport.c @@ -451,9 +451,8 @@ static int fetch_object_info_via_pack(struct transport *transport) ret = fetch_object_info(data->version, transport->server_options, transport->smart_options->object_info_oids, - transport->smart_options->object_info_options, &reader, - data->options.object_info_data, + data->options.object_info_results, transport->stateless_rpc, data->fd[1]); close(data->fd[0]); diff --git a/transport.h b/transport.h index a7869d18e020fb..6948b65db984be 100644 --- a/transport.h +++ b/transport.h @@ -7,6 +7,8 @@ #include "string-list.h" #include "connect.h" +struct fetch_object_info_results; + struct git_transport_options { unsigned thin : 1; unsigned keep : 1; @@ -57,8 +59,7 @@ struct git_transport_options { struct oidset *acked_commits; struct oid_array *object_info_oids; - struct object_info *object_info_data; - struct string_list *object_info_options; + struct fetch_object_info_results *object_info_results; }; enum transport_family { From 50dd6d370cd6421b523347ffa94bdd35bd264833 Mon Sep 17 00:00:00 2001 From: Pablo Sabater Date: Sat, 8 Aug 2026 02:02:20 +0200 Subject: [PATCH 110/259] fetch-object-info: die() on the remaining error path Every failure in fetch_object_info() dies except one: a short read while parsing the attribute lines returns -1. That -1 is then passed through fetch_object_info_via_pack() and get_remote_info() up to cat-file, only to die() with a generic message. Die in fetch_object_info() instead, consistently with the rest of its error paths, and make fetch_object_info() void. Mentored-by: Karthik Nayak Mentored-by: Chandra Pratap Signed-off-by: Pablo Sabater Signed-off-by: Junio C Hamano --- fetch-object-info.c | 19 +++++++++---------- fetch-object-info.h | 14 +++++++------- transport.c | 12 ++++++------ 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/fetch-object-info.c b/fetch-object-info.c index 5f53dbd6b90109..4db879c2dc7cc5 100644 --- a/fetch-object-info.c +++ b/fetch-object-info.c @@ -47,13 +47,13 @@ static int parse_object_size(const char *s, size_t *res) return 0; } -int fetch_object_info(const enum protocol_version version, - const struct string_list *server_options, - struct oid_array *oids, - struct packet_reader *reader, - struct fetch_object_info_results *results, - const int stateless_rpc, - const int fd_out) +void fetch_object_info(const enum protocol_version version, + const struct string_list *server_options, + struct oid_array *oids, + struct packet_reader *reader, + struct fetch_object_info_results *results, + const int stateless_rpc, + const int fd_out) { unsigned ask_size = 0; int size_index = -1; @@ -89,7 +89,8 @@ int fetch_object_info(const enum protocol_version version, if (packet_reader_read(reader) != PACKET_READ_NORMAL) { check_stateless_delimiter(stateless_rpc, reader, "stateless delimiter expected"); - return -1; + die(_("object-info: expected %" PRIuMAX " attributes, got %" PRIuMAX), + (uintmax_t)wanted, (uintmax_t)i); } if (!strcmp(reader->line, "size")) { @@ -156,8 +157,6 @@ int fetch_object_info(const enum protocol_version version, (uintmax_t)oids->nr); check_stateless_delimiter(stateless_rpc, reader, "stateless delimiter expected"); - - return 0; } void free_fetch_object_info_results(struct fetch_object_info_results *results) diff --git a/fetch-object-info.h b/fetch-object-info.h index 9f72e91155336f..97ee5314c99b00 100644 --- a/fetch-object-info.h +++ b/fetch-object-info.h @@ -24,13 +24,13 @@ struct oid_array; * attribute is not available. * Release them with free_fetch_object_info_results(). */ -int fetch_object_info(enum protocol_version version, - const struct string_list *server_options, - struct oid_array *oids, - struct packet_reader *reader, - struct fetch_object_info_results *results, - int stateless_rpc, - int fd_out); +void fetch_object_info(enum protocol_version version, + const struct string_list *server_options, + struct oid_array *oids, + struct packet_reader *reader, + struct fetch_object_info_results *results, + int stateless_rpc, + int fd_out); void free_fetch_object_info_results(struct fetch_object_info_results *results); diff --git a/transport.c b/transport.c index 35d3e98d9739a7..242fdea95b5fa4 100644 --- a/transport.c +++ b/transport.c @@ -448,12 +448,12 @@ static int fetch_object_info_via_pack(struct transport *transport) data->version = discover_version(&reader); transport->hash_algo = reader.hash_algo; - ret = fetch_object_info(data->version, - transport->server_options, - transport->smart_options->object_info_oids, - &reader, - data->options.object_info_results, - transport->stateless_rpc, data->fd[1]); + fetch_object_info(data->version, + transport->server_options, + transport->smart_options->object_info_oids, + &reader, + data->options.object_info_results, + transport->stateless_rpc, data->fd[1]); close(data->fd[0]); if (data->fd[1] >= 0) From 567e62b1b946407224dedfb85f8c9220983a5e13 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Sat, 8 Aug 2026 02:02:21 +0200 Subject: [PATCH 111/259] transport: drop remote object-info fields from transport struct A remote object-info request needs three things: the transport for contacting the remote, the list of oids to request, and a place to store the output. Rather than take these as function parameters, we take only the transport object, and expect the caller to have placed the other two into special fields in the transport struct. But this doesn't make much sense. The set of oids and results are really only valid for one request. There is no reason the transport would need to hang on to them outside of the single function call. Even though we save a few lines passing the parameters around through the various vtable functions, the result is harder to understand (for example, who is responsible for cleaning up results, and when should it happen?). It also opens up the possibility of a subtle bug. A caller is likely to point those fields to stack variables which could go out of scope, and the transport struct would be left holding invalid pointers. This is mostly harmless now, as we disconnect the transport immediately after the sole caller of transport_fetch_object_info(). But conceptually we could keep the transport open and make multiple fetch calls (and reuse the same connection to the helper, to a remote HTTP server, and so on). So let's pull these out of the struct and pass them as function parameters. It's a little more verbose, but I think more clearly illustrates the intent. I've also tweaked a few function signatures to mark the input oid array as const, since it is purely an input to the function. Signed-off-by: Jeff King Signed-off-by: Pablo Sabater Signed-off-by: Junio C Hamano --- builtin/cat-file.c | 6 ++---- fetch-object-info.c | 4 ++-- fetch-object-info.h | 2 +- transport-helper.c | 7 +++++-- transport-internal.h | 6 +++++- transport.c | 14 +++++++++----- transport.h | 7 +++---- 7 files changed, 27 insertions(+), 19 deletions(-) diff --git a/builtin/cat-file.c b/builtin/cat-file.c index e1650b2921ffcf..8dcad2f5ebf9f4 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -724,10 +724,8 @@ static int get_remote_info(int argc, goto cleanup; } - gtransport->smart_options->object_info_oids = object_info_oids; - - gtransport->smart_options->object_info_results = results; - retval = transport_fetch_object_info(gtransport); + retval = transport_fetch_object_info(gtransport, object_info_oids, + results); cleanup: transport_disconnect(gtransport); return retval; diff --git a/fetch-object-info.c b/fetch-object-info.c index 4db879c2dc7cc5..fe26bf4bbc9dc2 100644 --- a/fetch-object-info.c +++ b/fetch-object-info.c @@ -11,7 +11,7 @@ /* Sends object-info command and its arguments into the request buffer. */ static void send_object_info_request(const int fd_out, const struct string_list *server_options, - struct oid_array *oids, + const struct oid_array *oids, unsigned ask_size) { struct strbuf req_buf = STRBUF_INIT; @@ -49,7 +49,7 @@ static int parse_object_size(const char *s, size_t *res) void fetch_object_info(const enum protocol_version version, const struct string_list *server_options, - struct oid_array *oids, + const struct oid_array *oids, struct packet_reader *reader, struct fetch_object_info_results *results, const int stateless_rpc, diff --git a/fetch-object-info.h b/fetch-object-info.h index 97ee5314c99b00..10cf9f5f63a455 100644 --- a/fetch-object-info.h +++ b/fetch-object-info.h @@ -26,7 +26,7 @@ struct oid_array; */ void fetch_object_info(enum protocol_version version, const struct string_list *server_options, - struct oid_array *oids, + const struct oid_array *oids, struct packet_reader *reader, struct fetch_object_info_results *results, int stateless_rpc, diff --git a/transport-helper.c b/transport-helper.c index 623463dcea891a..b69cb733d85108 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -784,11 +784,14 @@ static int fetch_refs(struct transport *transport, return -1; } -static int fetch_object_info_helper(struct transport *transport) +static int fetch_object_info_helper(struct transport *transport, + const struct oid_array *oids, + struct fetch_object_info_results *results) { get_helper(transport); if (process_connect(transport, 0)) - return transport->vtable->fetch_object_info(transport); + return transport->vtable->fetch_object_info(transport, oids, + results); die(_("object-info requires protocol v2")); } diff --git a/transport-internal.h b/transport-internal.h index 60db0bedcdb9ae..a10b27cc81f511 100644 --- a/transport-internal.h +++ b/transport-internal.h @@ -7,6 +7,8 @@ struct ref; struct transport; struct strvec; struct transport_ls_refs_options; +struct oid_array; +struct fetch_object_info_results; struct transport_vtable { /** @@ -51,7 +53,9 @@ struct transport_vtable { * * Uses object-info capability of v2 protocol. */ - int (*fetch_object_info)(struct transport *transport); + int (*fetch_object_info)(struct transport *transport, + const struct oid_array *oids, + struct fetch_object_info_results *results); /** * Push the objects and refs. Send the necessary objects, and diff --git a/transport.c b/transport.c index 242fdea95b5fa4..abca9ac29aa9d7 100644 --- a/transport.c +++ b/transport.c @@ -433,7 +433,9 @@ static int get_bundle_uri(struct transport *transport) transport->bundles, stateless_rpc); } -static int fetch_object_info_via_pack(struct transport *transport) +static int fetch_object_info_via_pack(struct transport *transport, + const struct oid_array *oids, + struct fetch_object_info_results *results) { int ret = 0; struct git_transport_data *data = transport->data; @@ -450,9 +452,9 @@ static int fetch_object_info_via_pack(struct transport *transport) fetch_object_info(data->version, transport->server_options, - transport->smart_options->object_info_oids, + oids, &reader, - data->options.object_info_results, + results, transport->stateless_rpc, data->fd[1]); close(data->fd[0]); @@ -465,11 +467,13 @@ static int fetch_object_info_via_pack(struct transport *transport) return ret; } -int transport_fetch_object_info(struct transport *transport) +int transport_fetch_object_info(struct transport *transport, + const struct oid_array *oids, + struct fetch_object_info_results *results) { if (!transport->vtable->fetch_object_info) die(_("remote does not support object-info")); - return transport->vtable->fetch_object_info(transport); + return transport->vtable->fetch_object_info(transport, oids, results); } static int fetch_refs_via_pack(struct transport *transport, diff --git a/transport.h b/transport.h index 6948b65db984be..39193d0077a312 100644 --- a/transport.h +++ b/transport.h @@ -57,9 +57,6 @@ struct git_transport_options { * common commits to this oidset instead of fetching any packfiles. */ struct oidset *acked_commits; - - struct oid_array *object_info_oids; - struct fetch_object_info_results *object_info_results; }; enum transport_family { @@ -317,7 +314,9 @@ int transport_fetch_refs(struct transport *transport, struct ref *refs); /* * Fetch the object info from remote */ -int transport_fetch_object_info(struct transport *transport); +int transport_fetch_object_info(struct transport *transport, + const struct oid_array *oids, + struct fetch_object_info_results *results); /* * If this flag is set, unlocking will avoid to call non-async-signal-safe From 7692fa90199c623629f07e8883f5df8cfa859e03 Mon Sep 17 00:00:00 2001 From: Pablo Sabater Date: Sat, 8 Aug 2026 02:02:22 +0200 Subject: [PATCH 112/259] protocol-caps: add type support to object-info Teach the server-side object-info handler to accept type as a requested field. When the client includes type in its object-info request, the server returns the requested object type. While touching send_info(), wrap an over-long line and fix the bit field style of requested_info.size. Mentored-by: Karthik Nayak Mentored-by: Chandra Pratap Signed-off-by: Pablo Sabater Signed-off-by: Junio C Hamano --- protocol-caps.c | 21 ++++++++++++++++++--- t/t5701-git-serve.sh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/protocol-caps.c b/protocol-caps.c index 02261be14d817a..27e0f85b100cb9 100644 --- a/protocol-caps.c +++ b/protocol-caps.c @@ -11,7 +11,8 @@ #include "strbuf.h" struct requested_info { - unsigned size : 1; + unsigned size:1; + unsigned type:1; }; /* @@ -73,15 +74,20 @@ static void send_info(struct repository *r, struct packet_writer *writer, if (info->size) packet_writer_write(writer, "size"); + if (info->type) + packet_writer_write(writer, "type"); + for_each_string_list_item (item, oid_str_list) { const char *oid_str = item->string; + enum object_type object_type; struct object_id oid; size_t object_size; if (get_oid_hex_algop(oid_str, &oid, r->hash_algo) < 0) { packet_writer_error( writer, - "object-info: protocol error, expected to get oid, not '%s'", + "object-info: protocol error, expected to get " + "oid, not '%s'", oid_str); continue; } @@ -93,7 +99,8 @@ static void send_info(struct repository *r, struct packet_writer *writer, * If an object is not recognized by the server append SP to * the response. */ - if (get_object_info(r->objects, &oid, &object_size) <= OBJ_NONE) { + object_type = get_object_info(r->objects, &oid, &object_size); + if (object_type <= OBJ_NONE) { strbuf_addstr(&send_buffer, " "); goto write; } @@ -103,6 +110,9 @@ static void send_info(struct repository *r, struct packet_writer *writer, (uintmax_t)object_size); } + if (info->type) + strbuf_addf(&send_buffer, " %s", type_name(object_type)); + write: packet_writer_write(writer, "%s", send_buffer.buf); strbuf_reset(&send_buffer); @@ -124,6 +134,11 @@ int cap_object_info(struct repository *r, struct packet_reader *request) continue; } + if (!strcmp("type", request->line)) { + info.type = 1; + continue; + } + if (parse_oid(request->line, &oid_str_list)) continue; diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh index 51d5dd1ae6f389..f57e36a88d3cfb 100755 --- a/t/t5701-git-serve.sh +++ b/t/t5701-git-serve.sh @@ -369,6 +369,36 @@ test_expect_success 'basics of object-info' ' test_cmp expect actual ' +test_expect_success 'object-info supports type' ' + test_config transfer.advertiseObjectInfo true && + + two_oid=$(git rev-parse two:two.t) && + two_size=$(test_file_size two.t) && + + test-tool pkt-line pack >in <<-EOF && + command=object-info + object-format=$(test_oid algo) + 0001 + size + type + oid $two_oid + oid $two_oid + 0000 + EOF + + cat >expect <<-EOF && + size + type + $two_oid $two_size blob + $two_oid $two_size blob + 0000 + EOF + + test-tool serve-v2 --stateless-rpc out && + test-tool pkt-line unpack actual && + test_cmp expect actual +' + test_expect_success 'bare OID request' ' test_config transfer.advertiseObjectInfo true && From afa4ea56fedc3ca04d3b00a3ef930833e7ad68ea Mon Sep 17 00:00:00 2001 From: Pablo Sabater Date: Sat, 8 Aug 2026 02:02:23 +0200 Subject: [PATCH 113/259] fetch-object-info: parse type from server response The server can handle type requests but does not advertise the capability yet. Prepare the client to know how to parse the server response once the server advertises the type capability. Mentored-by: Karthik Nayak Mentored-by: Chandra Pratap Signed-off-by: Pablo Sabater Signed-off-by: Junio C Hamano --- builtin/cat-file.c | 7 +++++++ fetch-object-info.c | 38 +++++++++++++++++++++++++++++++++++--- fetch-object-info.h | 3 +++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 8dcad2f5ebf9f4..85020200835337 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -842,6 +842,8 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, if (data->info.sizep) results.wants_size = 1; + if (data->info.typep) + results.wants_type = 1; if (get_remote_info(count, argv, &results, &object_info_oids)) die(_("failed to get object info from the remote: %s"), argv[0]); @@ -850,6 +852,8 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, string_list_append(&data->remote_allowed_atoms, "objectname"); if (results.sizes) string_list_append(&data->remote_allowed_atoms, "objectsize"); + if (results.types) + string_list_append(&data->remote_allowed_atoms, "objecttype"); data->skip_object_info = 1; for (size_t i = 0; i < results.nr; i++) { @@ -868,6 +872,9 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, if (results.sizes) data->size = results.sizes[i]; + if (results.types) + data->type = results.types[i]; + opt->batch_mode = BATCH_MODE_INFO; data->is_remote = 1; batch_object_write(argv[i + 1], output, opt, data, NULL, 0); diff --git a/fetch-object-info.c b/fetch-object-info.c index fe26bf4bbc9dc2..0a58308f9b2559 100644 --- a/fetch-object-info.c +++ b/fetch-object-info.c @@ -1,6 +1,7 @@ #include "git-compat-util.h" #include "gettext.h" #include "hex.h" +#include "object.h" #include "pkt-line.h" #include "connect.h" #include "oid-array.h" @@ -12,7 +13,8 @@ static void send_object_info_request(const int fd_out, const struct string_list *server_options, const struct oid_array *oids, - unsigned ask_size) + unsigned ask_size, + unsigned ask_type) { struct strbuf req_buf = STRBUF_INIT; @@ -21,6 +23,9 @@ static void send_object_info_request(const int fd_out, if (ask_size) packet_buf_write(&req_buf, "size"); + if (ask_type) + packet_buf_write(&req_buf, "type"); + if (oids) for (size_t i = 0; i < oids->nr; i++) packet_buf_write(&req_buf, "oid %s", @@ -56,7 +61,9 @@ void fetch_object_info(const enum protocol_version version, const int fd_out) { unsigned ask_size = 0; + unsigned ask_type = 0; int size_index = -1; + int type_index = -1; size_t wanted; results->nr = oids->nr; @@ -71,11 +78,16 @@ void fetch_object_info(const enum protocol_version version, server_supports_feature("object-info", "size", 0)) ask_size = 1; + if (results->wants_type && + server_supports_feature("object-info", "type", 0)) + ask_type = 1; + /* * Even if no options are left, we still send the oid so we get * at least an existence check. */ - send_object_info_request(fd_out, server_options, oids, ask_size); + send_object_info_request(fd_out, server_options, oids, ask_size, + ask_type); break; case protocol_v1: case protocol_v0: @@ -83,7 +95,7 @@ void fetch_object_info(const enum protocol_version version, case protocol_unknown_version: BUG("unknown protocol version"); } - wanted = ask_size; + wanted = ask_size + ask_type; for (size_t i = 0; i < wanted; i++) { if (packet_reader_read(reader) != PACKET_READ_NORMAL) { @@ -100,6 +112,13 @@ void fetch_object_info(const enum protocol_version version, die(_("object-info: duplicate 'size' attribute")); size_index = (int)i; CALLOC_ARRAY(results->sizes, results->nr); + } else if (!strcmp(reader->line, "type")) { + if (!ask_type) + die(_("object-info: unrequested 'type' attribute")); + if (results->types) + die(_("object-info: duplicate 'type' attribute")); + type_index = (int)i; + CALLOC_ARRAY(results->types, results->nr); } else { die(_("object-info: unknown attribute '%s'"), reader->line); @@ -149,6 +168,18 @@ void fetch_object_info(const enum protocol_version version, object_info_values.items[0].string, object_info_values.items[size_index + 1].string); + if (results->types) { + const char *type_str = + object_info_values.items[type_index + 1].string; + int type = type_from_string_gently(type_str, -1, 1); + + if (type < 0) + die(_("object-info: object %s has invalid type '%s'"), + object_info_values.items[0].string, type_str); + + results->types[i] = type; + } + string_list_clear(&object_info_values, 0); } @@ -162,6 +193,7 @@ void fetch_object_info(const enum protocol_version version, void free_fetch_object_info_results(struct fetch_object_info_results *results) { free(results->sizes); + free(results->types); free(results->unrecognized); memset(results, 0, sizeof(*results)); } diff --git a/fetch-object-info.h b/fetch-object-info.h index 10cf9f5f63a455..2fba96c6f7de52 100644 --- a/fetch-object-info.h +++ b/fetch-object-info.h @@ -1,14 +1,17 @@ #ifndef FETCH_OBJECT_INFO_H #define FETCH_OBJECT_INFO_H +#include "object.h" #include "pkt-line.h" #include "protocol.h" struct fetch_object_info_results { size_t *sizes; + enum object_type *types; uint8_t *unrecognized; size_t nr; unsigned wants_size:1; + unsigned wants_type:1; }; #define FETCH_OBJECT_INFO_RESULTS_INIT { 0 } From 4c842de0e4580dc217d9ba5519e8348915f9c816 Mon Sep 17 00:00:00 2001 From: Pablo Sabater Date: Sat, 8 Aug 2026 02:02:24 +0200 Subject: [PATCH 114/259] serve: advertise type capability The server and the client can handle type requests but the client won't ask for it until the server advertises it. Add type to the advertised capabilities so the client knows that it can request it. Mentored-by: Karthik Nayak Mentored-by: Chandra Pratap Signed-off-by: Pablo Sabater Signed-off-by: Junio C Hamano --- serve.c | 4 ++-- t/t1017-cat-file-remote-object-info.sh | 26 ++++++++++++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/serve.c b/serve.c index 2b07d922b3dde1..2ce513cf2d5892 100644 --- a/serve.c +++ b/serve.c @@ -97,9 +97,9 @@ static int object_info_advertise(struct repository *r, struct strbuf *value) /* disabled by default */ advertise_object_info = 0; } - /* Currently only size is supported */ + /* Currently only size and type are supported */ if (value && advertise_object_info) - strbuf_addstr(value, "size"); + strbuf_addstr(value, "size type"); return advertise_object_info; } diff --git a/t/t1017-cat-file-remote-object-info.sh b/t/t1017-cat-file-remote-object-info.sh index 116862f9d0b447..190c45eefc21bd 100755 --- a/t/t1017-cat-file-remote-object-info.sh +++ b/t/t1017-cat-file-remote-object-info.sh @@ -7,6 +7,7 @@ test_description='git cat-file --batch-command with remote-object-info command' hello_content="Hello World" hello_size=$(strlen "$hello_content") +hello_type="blob" hello_oid=$(echo_without_newline "$hello_content" | git hash-object --stdin) hello_short_oid=$(git rev-parse --short "$hello_oid") @@ -19,6 +20,7 @@ unstored_oid=$(echo_without_newline "$unstored_content" | git hash-object --stdi # file name is hello, which is 5 characters # a space is 1 character and a null is 1 character tree_size=$(($(test_oid rawsz) + 13)) +tree_type="tree" commit_message="Initial commit" @@ -31,6 +33,7 @@ commit_message="Initial commit" # An easier way to calculate is: 1. use `git cat-file commit | wc -c`, # to get 177, 2. then deduct 40 hex characters to get 137 commit_size=$(($(test_oid hexsz) + 137)) +commit_type="commit" tag_header_without_oid="type blob tag hellotag @@ -44,6 +47,7 @@ $tag_description" tag_oid=$(echo_without_newline "$tag_content" | git hash-object -t tag --stdin -w) tag_size=$(strlen "$tag_content") +tag_type="tag" set_transport_variables () { hello_oid=$(echo_without_newline "$hello_content" | git hash-object --stdin) @@ -256,14 +260,12 @@ test_expect_success 'remote-object-info does not die on missing oid like info' ' ) ' -# This tests depends on %(objecttype) not being supported yet, once supported -# it needs to be updated. -test_expect_success 'unsupported placeholder on remote returns empty string' ' +test_expect_success 'objecttype is supported by remote-object-info' ' ( set_transport_variables "$daemon_parent" && cd "$daemon_parent/daemon_client_empty" && - echo "" >expect && + echo "$hello_type" >expect && git cat-file --batch-command="%(objecttype)" >actual <<-EOF && remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid EOF @@ -271,6 +273,22 @@ test_expect_success 'unsupported placeholder on remote returns empty string' ' ) ' +test_expect_success 'unsupported placeholders on remote return empty string' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + fmt="%(objectmode) %(objectsize:disk) %(rest) %(deltabase)" && + + # The hardcoded SPs between the atoms are respected. + echo " " >expect && + git cat-file --batch-command="$fmt" >actual <<-EOF && + remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid + EOF + test_cmp expect actual + ) +' + test_expect_success 'requesting only objectname echoes back' ' ( set_transport_variables "$daemon_parent" && From 245f2a8b2efb1cf93358cd4c143d0a91c23e0e1d Mon Sep 17 00:00:00 2001 From: Pablo Sabater Date: Sat, 8 Aug 2026 02:02:25 +0200 Subject: [PATCH 115/259] cat-file: unify default format %(objecttype) is supported both by the client and by the server. Change the temporary default format to the unified version that the other commands use. Update documentation to remove %(objecttype) from the caveats of remote-object-info and show %(objecttype) support. Now that type is supported and the default format unified, update the tests to expect the new default format. Mentored-by: Karthik Nayak Mentored-by: Chandra Pratap Signed-off-by: Pablo Sabater Signed-off-by: Junio C Hamano --- Documentation/git-cat-file.adoc | 17 ++++----- Documentation/gitprotocol-v2.adoc | 18 +++++++-- builtin/cat-file.c | 7 ---- t/t1017-cat-file-remote-object-info.sh | 52 +++++++++++++------------- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/Documentation/git-cat-file.adoc b/Documentation/git-cat-file.adoc index ac3b528c6f00f6..514bfc00328caf 100644 --- a/Documentation/git-cat-file.adoc +++ b/Documentation/git-cat-file.adoc @@ -348,15 +348,12 @@ newline. The available atoms are: after that first run of whitespace (i.e., the "rest" of the line) are output in place of the `%(rest)` atom. -The command `remote-object-info` only supports the `%(objectname)` and -`%(objectsize)` placeholders. See `CAVEATS` below for more information. +The command `remote-object-info` only supports the `%(objectname)`, +`%(objectsize)` and `%(objecttype)` placeholders. See `CAVEATS` below for more +information. If no format is specified, the default format is `%(objectname) -%(objecttype) %(objectsize)`, except for `remote-object-info` commands which -use `%(objectname) %(objectsize)` because `%(objecttype)` is not supported yet. - -WARNING: When "%(objecttype)" is supported, the default format WILL be unified, -so DO NOT RELY on the current default format to stay the same!!! +%(objecttype) %(objectsize)`. If `--batch` is specified, or if `--batch-command` is used with the `contents` command, the object information is followed by the object contents (consisting @@ -453,9 +450,9 @@ scripting purposes. CAVEATS ------- -Note that only `%(objectname)` and `%(objectsize)` are currently -supported by the `remote-object-info` command. Using any other placeholder in -the format string will return an empty string in its position. +Note that only `%(objectname)`, `%(objectsize)` and `%(objecttype)` are +currently supported by the `remote-object-info` command. Using any other +placeholder in the format string will return an empty string in its position. Note that the sizes of objects on disk are reported accurately, but care should be taken in drawing conclusions about which refs or objects are diff --git a/Documentation/gitprotocol-v2.adoc b/Documentation/gitprotocol-v2.adoc index 7bf62014c3917d..dd52fd8110dcf1 100644 --- a/Documentation/gitprotocol-v2.adoc +++ b/Documentation/gitprotocol-v2.adoc @@ -558,14 +558,17 @@ object-info `object-info` is the command to retrieve information about one or more objects. Its main purpose is to allow a client to make decisions based on this -information without having to fully fetch objects. Object size is the only -information that is currently supported. +information without having to fully fetch objects. Currently only object size +and type are supported. An `object-info` request takes the following arguments: size Requests size information to be returned for each listed object id. + type + Requests type information to be returned for each listed object id. + oid Indicates to the server an object which the client wants to obtain information for. They must be full OIDs. @@ -580,11 +583,18 @@ space. info = *PKT-LINE(attr LF) *PKT-LINE(obj-info LF) - attr = "size" + attr = "size" | "type" obj-size = 1*DIGIT - obj-info = obj-id [SP [obj-size]] + obj-type = "blob" | "tree" | "commit" | "tag" + + obj-val = obj-size | obj-type + + obj-info = obj-id [SP [obj-val *(SP obj-val)]] + +The values in `obj-info` appear in the same order as the corresponding `attr` +lines, with exactly one value per requested attribute. If the server does not recognize the OID, the response will be ` SP` regardless of the number of attributes requested. diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 85020200835337..011acdec09ef61 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -821,15 +821,9 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, char *line_to_split; struct fetch_object_info_results results = FETCH_OBJECT_INFO_RESULTS_INIT; struct oid_array object_info_oids = OID_ARRAY_INIT; - const char *saved_format = opt->format; if (strlen(line) >= MAX_REMOTE_OBJ_INFO_LINE) die(_("remote-object-info command too long")); - /* - * TODO: Use the default format once %(objecttype) is supported. - */ - if (!opt->format) - opt->format = "%(objectname) %(objectsize)"; line_to_split = xstrdup(line); count = split_cmdline(line_to_split, &argv); @@ -881,7 +875,6 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, data->is_remote = 0; } data->skip_object_info = 0; - opt->format = saved_format; free_fetch_object_info_results(&results); free(line_to_split); diff --git a/t/t1017-cat-file-remote-object-info.sh b/t/t1017-cat-file-remote-object-info.sh index 190c45eefc21bd..e2919aa061830a 100755 --- a/t/t1017-cat-file-remote-object-info.sh +++ b/t/t1017-cat-file-remote-object-info.sh @@ -139,10 +139,10 @@ test_expect_success 'batch-command remote-object-info git:// default filter' ' set_transport_variables "$daemon_parent" && cd "$daemon_parent/daemon_client_empty" && - echo "$hello_oid $hello_size" >expect && - echo "$tree_oid $tree_size" >>expect && - echo "$commit_oid $commit_size" >>expect && - echo "$tag_oid $tag_size" >>expect && + echo "$hello_oid $hello_type $hello_size" >expect && + echo "$tree_oid $tree_type $tree_size" >>expect && + echo "$commit_oid $commit_type $commit_size" >>expect && + echo "$tag_oid $tag_type $tag_size" >>expect && git cat-file --batch-command >actual <<-EOF && remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid $tree_oid @@ -152,7 +152,7 @@ test_expect_success 'batch-command remote-object-info git:// default filter' ' ) ' -test_expect_success 'remote-object-info does not change the default format of info' ' +test_expect_success 'remote-object-info and info can be mixed using the unified default format' ' ( set_transport_variables "$daemon_parent" && cd "$daemon_parent/daemon_client_empty" && @@ -162,7 +162,7 @@ test_expect_success 'remote-object-info does not change the default format of in local_size=$(strlen "$local_content") && echo "$local_oid blob $local_size" >expect && - echo "$hello_oid $hello_size" >>expect && + echo "$hello_oid blob $hello_size" >>expect && echo "$local_oid blob $local_size" >>expect && git cat-file --batch-command >actual <<-EOF && @@ -209,10 +209,10 @@ test_expect_success 'batch-command -Z remote-object-info git:// default filter' set_transport_variables "$daemon_parent" && cd "$daemon_parent/daemon_client_empty" && - printf "%s\0" "$hello_oid $hello_size" >expect && - printf "%s\0" "$tree_oid $tree_size" >>expect && - printf "%s\0" "$commit_oid $commit_size" >>expect && - printf "%s\0" "$tag_oid $tag_size" >>expect && + printf "%s\0" "$hello_oid $hello_type $hello_size" >expect && + printf "%s\0" "$tree_oid $tree_type $tree_size" >>expect && + printf "%s\0" "$commit_oid $commit_type $commit_size" >>expect && + printf "%s\0" "$tag_oid $tag_type $tag_size" >>expect && printf "%s\0" "$hello_oid missing" >>expect && printf "%s\0" "$tree_oid missing" >>expect && @@ -448,10 +448,10 @@ test_expect_success 'batch-command remote-object-info file:// default filter' ' server_path="$(pwd)/server" && cd file_client_empty && - echo "$hello_oid $hello_size" >expect && - echo "$tree_oid $tree_size" >>expect && - echo "$commit_oid $commit_size" >>expect && - echo "$tag_oid $tag_size" >>expect && + echo "$hello_oid $hello_type $hello_size" >expect && + echo "$tree_oid $tree_type $tree_size" >>expect && + echo "$commit_oid $commit_type $commit_size" >>expect && + echo "$tag_oid $tag_type $tag_size" >>expect && git cat-file --batch-command >actual <<-EOF && remote-object-info "file://${server_path}" $hello_oid $tree_oid @@ -467,10 +467,10 @@ test_expect_success 'batch-command -Z remote-object-info file:// default filter' server_path="$(pwd)/server" && cd file_client_empty && - printf "%s\0" "$hello_oid $hello_size" >expect && - printf "%s\0" "$tree_oid $tree_size" >>expect && - printf "%s\0" "$commit_oid $commit_size" >>expect && - printf "%s\0" "$tag_oid $tag_size" >>expect && + printf "%s\0" "$hello_oid $hello_type $hello_size" >expect && + printf "%s\0" "$tree_oid $tree_type $tree_size" >>expect && + printf "%s\0" "$commit_oid $commit_type $commit_size" >>expect && + printf "%s\0" "$tag_oid $tag_type $tag_size" >>expect && printf "%s\0" "$hello_oid missing" >>expect && printf "%s\0" "$tree_oid missing" >>expect && @@ -618,10 +618,10 @@ test_expect_success 'batch-command remote-object-info http:// default filter' ' set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && cd "$HTTPD_DOCUMENT_ROOT_PATH/http_client_empty" && - echo "$hello_oid $hello_size" >expect && - echo "$tree_oid $tree_size" >>expect && - echo "$commit_oid $commit_size" >>expect && - echo "$tag_oid $tag_size" >>expect && + echo "$hello_oid $hello_type $hello_size" >expect && + echo "$tree_oid $tree_type $tree_size" >>expect && + echo "$commit_oid $commit_type $commit_size" >>expect && + echo "$tag_oid $tag_type $tag_size" >>expect && git cat-file --batch-command >actual <<-EOF && remote-object-info "$HTTPD_URL/smart/http_parent" $hello_oid $tree_oid @@ -636,10 +636,10 @@ test_expect_success 'batch-command -Z remote-object-info http:// default filter' set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && cd "$HTTPD_DOCUMENT_ROOT_PATH/http_client_empty" && - printf "%s\0" "$hello_oid $hello_size" >expect && - printf "%s\0" "$tree_oid $tree_size" >>expect && - printf "%s\0" "$commit_oid $commit_size" >>expect && - printf "%s\0" "$tag_oid $tag_size" >>expect && + printf "%s\0" "$hello_oid $hello_type $hello_size" >expect && + printf "%s\0" "$tree_oid $tree_type $tree_size" >>expect && + printf "%s\0" "$commit_oid $commit_type $commit_size" >>expect && + printf "%s\0" "$tag_oid $tag_type $tag_size" >>expect && batch_input="remote-object-info $HTTPD_URL/smart/http_parent $hello_oid $tree_oid remote-object-info $HTTPD_URL/smart/http_parent $commit_oid $tag_oid From bc27e75a0e9a75b9543ff4cd2a12443fce558a79 Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Sun, 9 Aug 2026 22:06:25 +0200 Subject: [PATCH 116/259] doc: interpret-trailers: stop fixating on RFC 822 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This command handles the trailer metadata format. But the command isn’t introduced as such; it is instead introduced by stating that these trailer lines look similar to RFC 822 email headers. This is overwrought; most people do not deal directly with email headers, and certainly not email RFCs. Trailers are just key–value pairs that, like email headers, use colon as the separator. The format in its simplest form is easy to describe directly without comparing it to anything else; we will do that in the upcoming commit “explain the format after the intro”. For now, let’s: • remove the first mention of email headers; • keep the second, innocuous comparison with email line folding in the middle; and • remove the now-unneeded disclaimer that trailers do not share many of the features of RFC 822 email headers—there is no invitation to speculate that trailers would follow any other email format rules since we do not compare them directly any more. *** Talking about trailers as an RFC 822/2822-like format seems to go back to the `--fixes`/`Fixes:` trailer topic,[1] the thread that precipitated this command and in turn the first trailer support in git(1) beyond adding s-o-b lines. † 1: https://lore.kernel.org/all/20131027071407.GA11683@leaf/ Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-interpret-trailers.adoc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index 77b4f63b05cf5b..1878848ad2acb9 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -14,9 +14,9 @@ git interpret-trailers [--in-place] [--trim-empty] DESCRIPTION ----------- -Add or parse _trailer_ lines that look similar to RFC 822 e-mail -headers, at the end of the otherwise free-form part of a commit -message. For example, in the following commit message +Add or parse _trailer_ lines at the end of the otherwise +free-form part of a commit message. For example, in the following commit +message ------------------------------------------------ subject @@ -107,9 +107,6 @@ key: This is a very long value, with spaces and newlines in it. ------------------------------------------------ -Note that trailers do not follow (nor are they intended to follow) many of the -rules for RFC 822 headers. For example they do not follow the encoding rule. - OPTIONS ------- `--in-place`:: From abb0d859f8132634d77f1205e7862b9e7a14b479 Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Sun, 9 Aug 2026 22:06:26 +0200 Subject: [PATCH 117/259] =?UTF-8?q?doc:=20interpret-trailers:=20replace=20?= =?UTF-8?q?=E2=80=9Clines=E2=80=9D=20with=20=E2=80=9Cmetadata=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We removed the initial comparison to email headers in the previous commit. Now the introduction paragraph just says “trailer lines”, and the only hint that this is metadata/structured information is the “otherwise free-form” phrase. Let’s replace “lines” with “metadata” since that is their purpose. This also makes the introduction more consistent with how I chose to define trailers in the glossary:[1] “Key-value metadata”. (We will introduce “key–value” in the upcoming commit “explain the format after the intro”.) † 1: 68e3c69e (Documentation/glossary: describe "trailer", 2024-11-17) Let’s not emphasize “trailer” here since we are going to define the term in the upcoming commit “explain the format after the intro”. Let’s call it “trailer metadata” rather than “trailers metadata”. At first it seemed better to use the latter: 1. We’re introducing the jargon, and the format is often discussed as plural “trailers”, with its constituent parts being singular “trailer” 2. What this replaces uses “trailer”, but it rescues the plural mood with “lines” 3. This is very soon going to go into the constituent parts, including each trailer, so we’re contrasting the concept name (trailers) with its parts But: 1. The former reads better (most important) 2. “Trailer *metadata*” suggests plurality, similar to “trailer *lines*” Helped-by: Matt Hunter Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-interpret-trailers.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index 1878848ad2acb9..c8950d3babc0f3 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -14,7 +14,7 @@ git interpret-trailers [--in-place] [--trim-empty] DESCRIPTION ----------- -Add or parse _trailer_ lines at the end of the otherwise +Add or parse trailer metadata at the end of the otherwise free-form part of a commit message. For example, in the following commit message From 500257fd2bc6f81181039bc7420383d14ecf980b Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Sun, 9 Aug 2026 22:06:27 +0200 Subject: [PATCH 118/259] =?UTF-8?q?doc:=20interpret-trailers:=20use=20?= =?UTF-8?q?=E2=80=9Cmetadata=E2=80=9D=20in=20Name=20as=20well?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now since the previous commit introduce the format as “trailer metadata”. We can replace “structured information” with “metadata” in the “Name” section to be consistent. While “structured information” does emphasize that the data is not loosely structured, we also say that this command adds to or parses this format. I don’t think that we need to emphasize that it is structured since clearly there is some structure there. Both “metadata” and “structured information” can convey the same information. But “metadata” is shorter and easier to deploy since it’s just one word. Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-interpret-trailers.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index c8950d3babc0f3..5e776f0059a11d 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -3,7 +3,7 @@ git-interpret-trailers(1) NAME ---- -git-interpret-trailers - Add or parse structured information in commit messages +git-interpret-trailers - Add or parse metadata in commit messages SYNOPSIS -------- From 33691bc9d75560299568ebdf9dfbf38539087be3 Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Sun, 9 Aug 2026 22:06:28 +0200 Subject: [PATCH 119/259] doc: interpret-trailers: not just for commit messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This command doesn’t interface with commits directly. You can interpret or modify any kind of text, even though commit messages are the most relevant. The git(1) suite also isn’t restricted to only direct commit support since git-tag(1) learned `--trailer` in 066cef77 (builtin/tag: add --trailer option, 2024-05-05) Now, we already introduce the command in the “Name” section as dealing with commit messages as well. That is fine since that intro line needs to remain pretty short. Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-interpret-trailers.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index 5e776f0059a11d..ab3627c2cba953 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -15,8 +15,8 @@ git interpret-trailers [--in-place] [--trim-empty] DESCRIPTION ----------- Add or parse trailer metadata at the end of the otherwise -free-form part of a commit message. For example, in the following commit -message +free-form part of a commit message, or any other kind of text. +For example, in the following commit message ------------------------------------------------ subject From c88d60db4438b05ac29d07b5373e6fb7a37d56af Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Sun, 9 Aug 2026 22:06:29 +0200 Subject: [PATCH 120/259] doc: interpret-trailers: explain the format after the intro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit You need to read the entire “Description” section in order to understand the full trailer format. But there are many nuances, so that’s fine. As a starter though we have an introductory example.[1] That turns out to be crucial; the rest of this section talks about the mechanics of the command and only incidentally the format itself. Now, although the example might arguably be self-explanatory, we can add a little preamble which defines the format in its simplest form as well as define the most important terms. Note that we name the “blank line” rule since I want to use that term every time it comes up. It gets very mildly obfuscated if you call it a “blank line” in one place[2] and “empty (or whitespace-only) ...” in another one.[3] We will define the format of the *key* in the next commit. † 1: from d57fa7fc (doc: trailer: add more examples in DESCRIPTION, 2023-06-15) † 2: `Documentation/git-interpret-trailers.adoc:86` in 5361983c (The 22nd batch, 2026-03-27) † 3: `Documentation/git-interpret-trailers.adoc:93` in 5361983c (The 22nd batch, 2026-03-27) Suggested-by: D. Ben Knoble Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-interpret-trailers.adoc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index ab3627c2cba953..109059f11edc66 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -16,7 +16,12 @@ DESCRIPTION ----------- Add or parse trailer metadata at the end of the otherwise free-form part of a commit message, or any other kind of text. -For example, in the following commit message + +A _trailer_ in its simplest form is a key-value pair with a colon as a +separator. A _trailer block_ consists of one or more trailers. The +trailer block needs to be preceded by a blank line, where a _blank line_ +is either an empty or a whitespace-only line. For example, in the +following commit message ------------------------------------------------ subject From fd39e5a481154cdb0e42a8a89c0062bbf6e3ab2b Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Sun, 9 Aug 2026 22:06:30 +0200 Subject: [PATCH 121/259] doc: interpret-trailers: explain key format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A trailer key must consist of ASCII alphanumeric characters and hyphens *only*. Let’s document it explicitly instead of relying on readers being conservative and only basing their trailer keys on the documentation examples.[1] The previous commit provided us with an appropriate paragraph to describe the key format. † 1: Technically they would then miss out on using digits in them since all of the example keys just use letters and hyphens Reported-by: Brendan Jackman Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-interpret-trailers.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index 109059f11edc66..fb503cbe9528ce 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -18,7 +18,8 @@ Add or parse trailer metadata at the end of the otherwise free-form part of a commit message, or any other kind of text. A _trailer_ in its simplest form is a key-value pair with a colon as a -separator. A _trailer block_ consists of one or more trailers. The +separator. The _key_ consists of ASCII alphanumeric characters and +hyphens (`-`). A _trailer block_ consists of one or more trailers. The trailer block needs to be preceded by a blank line, where a _blank line_ is either an empty or a whitespace-only line. For example, in the following commit message From fddec1fe112470d35696322de65666a53b5bac5c Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Sun, 9 Aug 2026 22:06:31 +0200 Subject: [PATCH 122/259] doc: interpret-trailers: add key format example All of the examples speak of the Happy Path where everything works as intended. But failure examples can also be instructive. Especially for explaining again, by example, the key format (see previous commit). This also allows us to demonstrate trailer block detection with a concrete example. Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-interpret-trailers.adoc | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index fb503cbe9528ce..a0f7ed6fdd9bd7 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -405,6 +405,29 @@ mv "\$1.new" "\$1" $ chmod +x .git/hooks/commit-msg ------------ +* Here we try to use three different trailer keys. But it fails because + two of them are not recognized as trailer keys. ++ +---- +$ cat msg.txt +subject + +Skapad-på: some-branch +Hash-in-v6.11: 45c12d3269fe48f22834320c782ffe86c3560f2c +Reviewed-by: Alice +$ git interpret-trailers --only-trailers Date: Sun, 9 Aug 2026 22:06:32 +0200 Subject: [PATCH 123/259] doc: interpret-trailers: join new-trailers again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are three paragraphs that talk about how a new trailer is added. But the first one is separated from the other two by two paragraphs about how `key-alias` can make using `--trailer` more convenient. This short how-to does not follow thematically from the previous paragraph, and can wait until we have fully described how a new trailer is added. So let’s move the three paragraphs about the new-trailer topic together and move the how-to paragraphs after that. *** Let’s now review the history of the document. Even if the document is not quite correct in its current state, just doing the apparently obvious edit without considering the history does not respect the effort that went into changing the document in the past. These three paragraphs were originally next to each other, in the first version of the doc.[1] But extra sentences about this how-to topic was added to the first paragraph nine years later:[2] [...] `': '` (one colon followed by one space). For convenience, the can be a shortened string key (e.g., "sign") instead of the full string which should [...] And then it was split into it’s own paragraph a little later.[3] This evolution shows, in my opinion, that this how-to never followed thematically from the existing topic. Which means that there is nothing that was potentially lost to time that we need to restore or respect. † 1: dfd66ddf (Documentation: add documentation for 'git interpret-trailers', 2014-10-13) † 2: eda2c44c (doc: trailer: mention 'key' in DESCRIPTION, 2023-06-15) † 3: 6ccbc667 (trailer doc: is a or , not both, 2023-09-07) Suggested-by: D. Ben Knoble Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-interpret-trailers.adoc | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index a0f7ed6fdd9bd7..616f479a3670a0 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -74,19 +74,6 @@ key: value This means that the trimmed __ and __ will be separated by "`:`{nbsp}" (one colon followed by one space). -For convenience, a __ can be configured to make using `--trailer` -shorter to type on the command line. This can be configured using the -`trailer..key` configuration variable. The __ must be a prefix -of the full __ string, although case sensitivity does not matter. For -example, if you have - ------------------------------------------------- -trailer.sign.key "Signed-off-by: " ------------------------------------------------- - -in your configuration, you only need to specify `--trailer="sign: foo"` -on the command line instead of `--trailer="Signed-off-by: foo"`. - By default the new trailer will appear at the end of all the existing trailers. If there is no existing trailer, the new trailer will appear at the end of the input. A blank line will be added before the new @@ -101,6 +88,19 @@ The group must either be at the end of the input or be the last non-whitespace lines before a line that starts with `---` (followed by a space or the end of the line). +For convenience, a __ can be configured to make using `--trailer` +shorter to type on the command line. This can be configured using the +`trailer..key` configuration variable. The __ must be a prefix +of the full __ string, although case sensitivity does not matter. For +example, if you have + +------------------------------------------------ +trailer.sign.key "Signed-off-by: " +------------------------------------------------ + +in your configuration, you only need to specify `--trailer="sign: foo"` +on the command line instead of `--trailer="Signed-off-by: foo"`. + When reading trailers, there can be no whitespace before or inside the __, but any number of regular space and tab characters are allowed between the __ and the separator. There can be whitespaces before, From 4d45e571ae9a8dc47af95a4698cd28b15723bf52 Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Sun, 9 Aug 2026 22:06:33 +0200 Subject: [PATCH 124/259] =?UTF-8?q?doc:=20interpret-trailers:=20commit=20t?= =?UTF-8?q?o=20=E2=80=9Ctrailer=20block=E2=80=9D=20term?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We chose to introduce the term “trailer block” into the documentation a few commits ago.[1] It is used in the code though, so it is not a newly invented term. That term was useful to explain where the trailers are found (they *trail* the message). But it is also useful here, where we explain how trailers are added to existing messages, how trailer blocks are found (beyond the simple case in the introduction), and how the end of the message is found. Also note that we simplify the “blank line” point. The text says: A blank line will be added before the new trailer if there isn't one already. But this isn’t quite coherent. The previous sentence says “If there is no existing trailer”, so we are in one of these modes: 1. discussing trailer blocks in general; or 2. discussing creating a new trailer block in particular. If (1), then we shouldn’t add a blank line before the new trailer if there exists a trailer block already. And if (2), then the “if there isn’t one already” is redundant.[2] So just talking about the higher- level “trailer block” simplifies the text, since we don’t have to worry about the different contexts that *trailers* can find themselves in. † 1: in commit “explain the format after the intro” † 2: Note that non-trailer lines don’t matter here; if you have a trailer block consisting of `(cherry picked from commit )`, then you still shouldn’t insert a blank line before the new trailer since that would create a new trailer block Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-interpret-trailers.adoc | 26 ++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index 616f479a3670a0..a1adab20fefd61 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -74,19 +74,21 @@ key: value This means that the trimmed __ and __ will be separated by "`:`{nbsp}" (one colon followed by one space). -By default the new trailer will appear at the end of all the existing -trailers. If there is no existing trailer, the new trailer will appear -at the end of the input. A blank line will be added before the new -trailer if there isn't one already. - -Existing trailers are extracted from the input by looking for -a group of one or more lines that (i) is all trailers, or (ii) contains at -least one Git-generated or user-configured trailer and consists of at +By default the new trailer will appear at the end of the trailer block. +A trailer block will be created with only that trailer if a trailer +block does not already exist. Recall that a trailer block needs to be +preceded by a blank line, so a blank line will be inserted before the +new trailer block in that case. + +Existing trailers are extracted from the input by looking for the +trailer block. A trailer block is a group of one or more lines that (i) +is all trailers, or (ii) contains at least one Git-generated or +user-configured trailer and consists of at least 25% trailers. -The group must be preceded by one or more empty (or whitespace-only) lines. -The group must either be at the end of the input or be the last -non-whitespace lines before a line that starts with `---` (followed by a -space or the end of the line). +The trailer block is by definition at the end of the commit message. +The message in turn is either (i) at the end of the input, or (ii) the +last non-whitespace lines before a line that starts with `---` (followed +by a space or the end of the line). For convenience, a __ can be configured to make using `--trailer` shorter to type on the command line. This can be configured using the From cb657364d5c3b82bc32e22fe1eca445d9960032b Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Sun, 9 Aug 2026 22:06:34 +0200 Subject: [PATCH 125/259] doc: interpret-trailers: rewrite new-trailers paragraphs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two commits ago we moved new-trailers paragraph next to each other. But there is something curious about two of them: By default the new trailer will appear at the end of the trailer block. [...] Then a source block and a paragraph later: By default, a `=` or `:` argument given using `--trailer` will be appended after the existing trailers only if [...] Why are there two paragraphs that talk about how “By default” a trailer will be appended? We can make these paragraphs flow better, and with a more distinct character each, by dividing the flow like this: 1. Declare that we are about to talk about `--trailer` appending 2. Explain the default behavior 3. Explain how this affects the trailer block 4. Then discuss what each trailer line will look like Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-interpret-trailers.adoc | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index a1adab20fefd61..ac59ef51f806f5 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -60,10 +60,18 @@ are applied to each input and the way any existing trailer in the input is changed. They also make it possible to automatically add some trailers. -By default, a `=` or `:` argument given -using `--trailer` will be appended after the existing trailers only if -the last trailer has a different (__, __) pair (or if there -is no existing trailer). The __ and __ parts will be trimmed +Let's consider new trailers added with `--trailer`. +By default, the new trailer will appear at the end of the trailer block. +Also by default, this new trailer will only be added +if the last trailer is different to it. +A trailer block will be created with only that trailer if a trailer +block does not already exist. Recall that a trailer block needs to be +preceded by a blank line, so a blank line will be inserted before the +new trailer block in that case. + +This is how the new trailer is added: a `=` or +`:` argument given using `--trailer` will be appended after +the existing trailers. The __ and __ parts will be trimmed to remove starting and trailing whitespace, and the resulting trimmed __ and __ will appear in the output like this: @@ -74,12 +82,6 @@ key: value This means that the trimmed __ and __ will be separated by "`:`{nbsp}" (one colon followed by one space). -By default the new trailer will appear at the end of the trailer block. -A trailer block will be created with only that trailer if a trailer -block does not already exist. Recall that a trailer block needs to be -preceded by a blank line, so a blank line will be inserted before the -new trailer block in that case. - Existing trailers are extracted from the input by looking for the trailer block. A trailer block is a group of one or more lines that (i) is all trailers, or (ii) contains at least one Git-generated or From 4515c86fd95ef8de8a1cbea90bb275538bfc87ee Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Sun, 9 Aug 2026 22:06:35 +0200 Subject: [PATCH 126/259] doc: interpret-trailers: document comment line treatment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment lines have always been ignored but this is not documented. The primary motivation here is to be reasonably complete in the documentation of how trailers are parsed; this is after all the only documentation page that documents this format. However, and going beyond that point, we could imagine that someone would want to use this format outside a commit (or tag) message context, like say in Git notes. On the other hand, it seems far-fetched that someone would be caught off guard by this considering that comment characters/strings are not likely to be alphanumeric,[1] which would mean that these comment lines would be treated as non-trailer lines if they were *not* detected and removed as comment lines. † 1: A notable exception is that Jujutsu VCS uses `JJ:` as the comment string Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-interpret-trailers.adoc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index ac59ef51f806f5..b4988d39eab0e9 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -117,6 +117,16 @@ key: This is a very long value, with spaces and newlines in it. ------------------------------------------------ +OTHER RULES +----------- + +What was covered in the previous section are the rules that are relevant +for regular use. The following points are included for completeness. + +This command ignores comment lines (see `core.commentString` in +linkgit:git-config[1]). This is for use with the `prepare-commit-msg` +and `commit-msg` hooks. + OPTIONS ------- `--in-place`:: From 09d63ccb06dcd687c023405dec0abbe790faed18 Mon Sep 17 00:00:00 2001 From: Kristoffer Haugsbakk Date: Sun, 2 Aug 2026 21:57:17 +0200 Subject: [PATCH 127/259] trailers: stop recognizing URLs as trailers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An HTTPS URL starts with an alphanumeric scheme followed by a colon. That means that they will be recognized as trailers in a trailer block. That turns out to be a problem in practice. Let’s stop recognizing these as trailers by failing the trailer parsing when we: 1. find the separator; 2. the separator and the next two characters form `://`; and 3. we haven’t parsed any whitespace yet. The simplest example of how this can be a problem is for people who do not use trailers but may leave URLs at the end of the commit message. Now, while these authors might not use trailers themselves, other authors may have used trailers and this metadata confusion can become a problem once someone tries to extract that metadata (and non-metadata). Let’s now look at some examples in the Linux Kernel[1] to see how this is a problem in practice. There are commits which contain intended non-trailer lines which start with URLs. These are comments. Example with just the trailers:[2] Signed-off-by: Shuai Xue [bhelgaas: squash fixes: https://lore.kernel.org/r/20260108013956.14351-2-bagasdotme@gmail.com https://lore.kernel.org/r/20260108013956.14351-3-bagasdotme@gmail.com] Signed-off-by: Bjorn Helgaas Reviewed-by: Ilpo Järvinen Link: https://patch.msgid.link/20251210132907.58799-4-xueshuai@linux.alibaba.com Those `[]` pairs delimit the “squash fixes” comment. Now, any of these two commands: git log --format='%(trailers:only)' -1 git log -1 --format=%B | git interpret-trailers --only-trailers Will both wrongly (according to the surmised user intent) include these two URL lines as trailers and also mangle the URLs, e.g.: https: //lore.kernel.org/r/20260108013956.14351-2-bagasdotme@gmail.com Because the `--only-trailers` mode (or `only` for the git-log(1) format) normalizes the output to a colon and a space. Another example is linewrapping mistakes; a `Link` trailer with a URL where the URL ended up on the next line, presumably because the user’s editor linewrapped the “too long” line. Example with just the trailers:[3] Link: https://patch.msgid.link/20260216-work-xattr-socket-v1-4-c2efa4f74cb7@kernel.org Link: https://lore.kernel.org/3cnmtqmakpbb2uwhenrj7kdqu3uefykiykjllgfbtpkiwhaa4s@sghkevv7jned [1] Acked-by: Darrick J. Wong Reviewed-by: Jan Kara Signed-off-by: Christian Brauner Now, this intended trailer is already ruined, but interpreting the URL as a standalone trailer only compounds the mistake. Yet another example is the trailer machinery normalizing the trailer block before application, resulting in a `https` trailer key in the commit message itself. Example with just the trailers:[4] https: //sashiko.dev/#/patchset/20260429114208.941011-1-holger.brunck%40hitachienergy.com Fixes: c19b6d246a35 ("drivers/net: support hdlc function for QE-UCC") Signed-off-by: Holger Brunck Link: https://patch.msgid.link/20260507155332.3452319-1-holger.brunck@hitachienergy.com Signed-off-by: Jakub Kicinski We have a helpful `Link` that points to the original patch.[5] Following it we can see that that `https` trailer was indeed a URL originally (again just the trailer block here): https://sashiko.dev/#/patchset/20260429114208.941011-1-holger.brunck%40hitachienergy.com Fixes: c19b6d246a35 ("drivers/net: support hdlc function for QE-UCC") Signed-off-by: Holger Brunck So how did it end up as a `https` trailer? My theory is that the trailer block was normalized on patch application, causing a URL comment to be wrongly normalized and cemented in the commit message as a trailer.[6] † 1: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/ † 2: commit 8236fc613d44e59f6736d6c3e9efffaf26ab7f00 † 3: commit 5bd97f5c5f241a5610c4412d1b93995a26241f81 † 4: commit 496c0c4c53bbe1bad97e82cd12103df61a6e459d † 5: https://patch.msgid.link/20260507155332.3452319-1-holger.brunck@hitachienergy.com † 6: There are only four commits in the Linux Kernel of this kind, and three of them have the same recurring person in the signoff chain. Helped-by: Jeff King Signed-off-by: Kristoffer Haugsbakk Signed-off-by: Junio C Hamano --- Documentation/git-interpret-trailers.adoc | 13 ++++-- t/t7513-interpret-trailers.sh | 19 +++++++++ t/unit-tests/u-trailer.c | 52 +++++++++++++++++++++++ trailer.c | 7 ++- 4 files changed, 87 insertions(+), 4 deletions(-) diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index b4988d39eab0e9..903d598dcb0595 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -123,9 +123,16 @@ OTHER RULES What was covered in the previous section are the rules that are relevant for regular use. The following points are included for completeness. -This command ignores comment lines (see `core.commentString` in -linkgit:git-config[1]). This is for use with the `prepare-commit-msg` -and `commit-msg` hooks. +-- +* This command ignores comment lines (see `core.commentString` in + linkgit:git-config[1]). This is for use with the `prepare-commit-msg` + and `commit-msg` hooks. + +* Candidate trailer lines that have `:` as the separator, that have no + whitespace before the value part, and that start with `//` are not + recognized as trailers. This is to avoid accidentally interpreting + URLs as trailers (e.g. lines that start with `https://`). +-- OPTIONS ------- diff --git a/t/t7513-interpret-trailers.sh b/t/t7513-interpret-trailers.sh index 818a8dafbd29e5..e3555b6d51d134 100755 --- a/t/t7513-interpret-trailers.sh +++ b/t/t7513-interpret-trailers.sh @@ -1989,4 +1989,23 @@ test_expect_success 'handling of --- lines in conjunction with cut-lines' ' test_cmp expected actual ' +test_expect_success 'URLs and lines that are not quite URLs' ' + cat >expect <<-\EOF && + https: //www.a-trailer.org + https: //www.another-trailer.org + Signed-off-by: somebody + EOF + git interpret-trailers --only-trailers >actual <<-\EOF && + subject + + body + + https://www.not-a-trailer.org + https ://www.a-trailer.org + https: //www.another-trailer.org + Signed-off-by: somebody + EOF + test_cmp expect actual +' + test_done diff --git a/t/unit-tests/u-trailer.c b/t/unit-tests/u-trailer.c index 3d60ea1603dbda..7404b165fac8fd 100644 --- a/t/unit-tests/u-trailer.c +++ b/t/unit-tests/u-trailer.c @@ -318,3 +318,55 @@ void test_trailer__one_non_trailer_no_git_trailers(void) 0, expected_contents); } + +void test_trailer__URL(void) +{ + struct contents expected_contents[] = { 0 }; + + t_trailer_iterator("Subject: foo bar\n" + "\n" + /* + * We do not want to match URLs as trailers. + */ + "https://www.example.org\n", + 0, + expected_contents); +} + +void test_trailer__not_a_URL_space_after_separator(void) +{ + struct contents expected_contents[] = { + { .raw = "https: //www.example.org\n", + .key = "https", + .val = "//www.example.org" }, + { 0 }, + }; + + t_trailer_iterator("Subject: foo bar\n" + "\n" + /* + * This has a space after ':' so it's not a URL. + */ + "https: //www.example.org\n", + 1, + expected_contents); +} + +void test_trailer__not_a_URL_space_before_separator(void) +{ + struct contents expected_contents[] = { + { .raw = "https ://www.example.org\n", + .key = "https", + .val = "//www.example.org" }, + { 0 }, + }; + + t_trailer_iterator("Subject: foo bar\n" + "\n" + /* + * This has a space before ':' so it's not a URL. + */ + "https ://www.example.org\n", + 1, + expected_contents); +} diff --git a/trailer.c b/trailer.c index 6d8ec7fa8d88b5..971ae45959618f 100644 --- a/trailer.c +++ b/trailer.c @@ -635,8 +635,13 @@ static ssize_t find_separator(const char *line, const char *separators) int whitespace_found = 0; const char *c; for (c = line; *c; c++) { - if (strchr(separators, *c)) + if (strchr(separators, *c)) { + /* avoid accidental URL matches (://) */ + if (*c == ':' && c[1] == '/' && c[2] == '/' && + !whitespace_found) + return -1; return c - line; + } if (!whitespace_found && (isalnum(*c) || *c == '-')) continue; if (c != line && (*c == ' ' || *c == '\t')) { From b0b304c10a65882a8e36abbb694603648a78edfd Mon Sep 17 00:00:00 2001 From: Harald Nordgren Date: Mon, 10 Aug 2026 17:53:32 +0000 Subject: [PATCH 128/259] send-email: clarify missing subject error Clarify that a message file is missing a 'Subject:' line. Terminate the error with a newline so Perl does not append its internal source location. Signed-off-by: Harald Nordgren Signed-off-by: Junio C Hamano --- git-send-email.perl | 2 +- t/t9001-send-email.sh | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/git-send-email.perl b/git-send-email.perl index bb8ddd1eef2c25..2071cff6ae3f0a 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -863,7 +863,7 @@ sub get_patch_subject { return "GIT: $1\n"; } close $fh; - die sprintf(__("No subject line in %s?"), $fn); + die sprintf(__("No 'Subject:' line in '%s'\n"), $fn); } if ($compose) { diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh index e9d814a34aa86c..d1393ef1978761 100755 --- a/t/t9001-send-email.sh +++ b/t/t9001-send-email.sh @@ -1422,6 +1422,21 @@ test_expect_success $PREREQ 'detects ambiguous reference/file conflict' ' test_grep disambiguate errors ' +test_expect_success $PREREQ 'missing subject omits Perl location' ' + cat >no-subject.patch <<-\EOF && + This is the body. + EOF + test_must_fail git send-email \ + --dry-run \ + --from="Example " \ + --to=nobody@example.com \ + no-subject.patch 2>actual && + cat >expect <<-\EOF && + No '\''Subject:'\'' line in '\''no-subject.patch'\'' + EOF + test_cmp expect actual +' + test_expect_success $PREREQ 'feed two files' ' rm -fr outdir && git format-patch -2 -o outdir && From 5ab204e7dfe4e5affb779c406a24b3b6e46ae54e Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 11 Aug 2026 10:33:03 +0200 Subject: [PATCH 129/259] parse-options: introduce OPT_HIDDEN_GROUP Hidden options are not shown by `git -h`, but are still shown by `git --help-all`. If there are a lot of hidden options or if they don't belong to the same categories as other options, there is currently no way to properly group them. Using `OPT_GROUP("Foo")` means that "Foo" will always be shown which we don't want if that group contains only hidden options. To provide a way to have groups shown only when hidden options are shown, let's implement an OPT_HIDDEN_GROUP macro. To test this new macro, let's also improve `test-tool parse-options` and test its output with `--help-all`. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano --- parse-options.c | 4 ++-- parse-options.h | 5 +++++ t/helper/test-parse-options.c | 4 ++++ t/t0040-parse-options.sh | 25 ++++++++++++++++++++++++- 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/parse-options.c b/parse-options.c index 08c21d9fc0a585..4519ead9dc77b8 100644 --- a/parse-options.c +++ b/parse-options.c @@ -1414,6 +1414,8 @@ static enum parse_opt_result usage_with_options_internal(struct parse_opt_ctx_t if (opts->type == OPTION_SUBCOMMAND) continue; + if (!full && (opts->flags & PARSE_OPT_HIDDEN)) + continue; if (opts->type == OPTION_GROUP) { fputc('\n', outfile); need_newline = 0; @@ -1421,8 +1423,6 @@ static enum parse_opt_result usage_with_options_internal(struct parse_opt_ctx_t fprintf(outfile, "%s\n", _(opts->help)); continue; } - if (!full && (opts->flags & PARSE_OPT_HIDDEN)) - continue; if (need_newline) { fputc('\n', outfile); diff --git a/parse-options.h b/parse-options.h index 3ec8ba5cc83c60..d7f896a9337024 100644 --- a/parse-options.h +++ b/parse-options.h @@ -237,6 +237,11 @@ struct option { .type = OPTION_GROUP, \ .help = (h), \ } +#define OPT_HIDDEN_GROUP(h) { \ + .type = OPTION_GROUP, \ + .help = (h), \ + .flags = PARSE_OPT_HIDDEN, \ +} #define OPT_BIT(s, l, v, h, b) OPT_BIT_F(s, l, v, h, b, 0) #define OPT_BITOP(s, l, v, h, set, clear) { \ .type = OPTION_BITOP, \ diff --git a/t/helper/test-parse-options.c b/t/helper/test-parse-options.c index 68579d83f3939e..f181f0c02d355a 100644 --- a/t/helper/test-parse-options.c +++ b/t/helper/test-parse-options.c @@ -209,6 +209,10 @@ int cmd__parse_options(int argc, const char **argv) OPT_GROUP("Alias"), OPT_STRING('A', "alias-source", &string, "string", "get a string"), OPT_ALIAS('Z', "alias-target", "alias-source"), + OPT_HIDDEN_GROUP("Hidden options"), + OPT_HIDDEN_BOOL(0, "hidden-bool", &boolean, "get a boolean"), + OPT_INTEGER_F('k', "hidden-integer", &integer, "get a integer", + PARSE_OPT_HIDDEN), OPT_END(), }; int ret = 0; diff --git a/t/t0040-parse-options.sh b/t/t0040-parse-options.sh index a22533f9ed6d16..449fff4d34b172 100755 --- a/t/t0040-parse-options.sh +++ b/t/t0040-parse-options.sh @@ -7,7 +7,7 @@ test_description='our own option parser' . ./test-lib.sh -cat >expect <<\EOF +cat >expect-part1 <<\EOF usage: test-tool parse-options A helper function for the parse-options API. @@ -41,6 +41,9 @@ String options --[no-]string2 get another string --[no-]st get another string (pervert ordering) -o get another string +EOF + +cat >expect-part2 <<\EOF --longhelp help text of this entry spans multiple lines --[no-]list add str to list @@ -67,12 +70,32 @@ Alias EOF +cat >expect-noop <<\EOF + --[no-]obsolete no-op (backward compatibility) +EOF + +cat >expect-hidden <<\EOF +Hidden options + --[no-]hidden-bool get a boolean + -k, --[no-]hidden-integer + get a integer + +EOF + test_expect_success 'test help' ' + cat expect-part1 expect-part2 >expect && test-tool parse-options -h >output 2>output.err && test_must_be_empty output.err && test_cmp expect output ' +test_expect_success 'test --help-all shows hidden group and options' ' + cat expect-part1 expect-noop expect-part2 expect-hidden >expect-help-all && + test-tool parse-options --help-all >output 2>output.err && + test_must_be_empty output.err && + test_cmp expect-help-all output +' + mv expect expect.err check () { From e455bade3a5167c36b42b3b8aa3c690722597c5b Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 11 Aug 2026 10:33:04 +0200 Subject: [PATCH 130/259] api-parse-options.adoc: document per-option flags The "Flags" section in "Documentation/technical/api-parse-options.adoc" documents the flags that can be passed to parse_options() itself. It does not, however, document the flags that can be set on individual options through the `flags` member of `struct option` (and through the `OPT_*_F()` macro variants). These per-option flags are used throughout the codebase (for example `PARSE_OPT_HIDDEN` is used to hide an option from `-h` while still showing it with `--help-all`), but a reader currently has to dig into "parse-options.h" to find them. To remediate that, let's add an "Option flags" subsection to the "Data Structure" section, just before the list of option macros. Let's also make it explicit that these are distinct from the parse_options() flags described earlier, and let's describe the `-h` versus `--help-all` behavior for `PARSE_OPT_HIDDEN`. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano --- .../technical/api-parse-options.adoc | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/Documentation/technical/api-parse-options.adoc b/Documentation/technical/api-parse-options.adoc index 880eb94642587a..5602cd44b23783 100644 --- a/Documentation/technical/api-parse-options.adoc +++ b/Documentation/technical/api-parse-options.adoc @@ -150,6 +150,68 @@ Data Structure The main data structure is an array of the `option` struct, say `static struct option builtin_add_options[]`. + +Option flags +~~~~~~~~~~~~ + +Each option can carry flags in the `flags` field of its `option` +struct. These are per-option flags and are distinct from the +`parse_options()` flags described above; they are usually set through +the `OPT_*_F()` macro variants (see below) rather than by hand. They +are the bitwise-or of: + +`PARSE_OPT_OPTARG`:: + The option's argument is optional, i.e. both `--option` and + `--option=` are accepted. + +`PARSE_OPT_NOARG`:: + The option takes no argument at all. Using `--option=` + is rejected. + +`PARSE_OPT_NONEG`:: + Disable the automatically generated negated `--no-option` + form. + +`PARSE_OPT_HIDDEN`:: + Hide the option: it is omitted from the usage shown by + `git -h`, but is still shown by `git --help-all`. + The option is parsed as usual either way. This is meant for + deprecated, advanced or otherwise uncommon options. + +`PARSE_OPT_LASTARG_DEFAULT`:: + The no-argument form is only accepted when the option is the + last token on the command line; used earlier, it still + requires an argument. Should not be combined with + `PARSE_OPT_OPTARG`. + +`PARSE_OPT_NODASH`:: + The option is a single character without a leading dash, such + as the `+` used by some commands. + +`PARSE_OPT_LITERAL_ARGHELP`:: + Use the argument help string (`argh`) verbatim in the usage + output instead of surrounding it with `<>` or `[]`. Useful when + `argh` already contains a hand-formatted description. + +`PARSE_OPT_FROM_ALIAS`:: + Internal flag, set on options that were expanded from a + configured alias. It should not be set by callers. + +`PARSE_OPT_NOCOMPLETE`:: + Do not offer this option for completion. + +`PARSE_OPT_COMP_ARG`:: + The option's argument, rather than the option itself, is what + should be completed. + +`PARSE_OPT_CMDMODE`:: + The option is one of several mutually exclusive "command mode" + options that share the same variable. Using more than one of + them at once is rejected. + +Macros +~~~~~~ + There are some macros to easily define options: `OPT__ABBREV(&int_var)`:: From 08b7e358e377ec2d7cd3cfe0679261b1ffffc8a7 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 11 Aug 2026 10:33:05 +0200 Subject: [PATCH 131/259] api-parse-options.adoc: document hidden and OPT_*_F option macros In "Documentation/technical/api-parse-options.adoc", the list of option macros does not mention the `OPT_*_F()` macro variants that take a trailing `flags` argument, nor the `OPT_HIDDEN_GROUP()` and `OPT_HIDDEN_BOOL()` convenience macros. Now that a previous commit documents the per-option flags, let's document these macros too: - Add a paragraph explaining the `OPT_*_F` convention and how it relates to the per-option flags. - Document `OPT_HIDDEN_GROUP()`, introduced in a previous commit, right after `OPT_GROUP()`. - Document `OPT_HIDDEN_BOOL()` right after `OPT_BOOL()`. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano --- Documentation/technical/api-parse-options.adoc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Documentation/technical/api-parse-options.adoc b/Documentation/technical/api-parse-options.adoc index 5602cd44b23783..95b7924e84e2c8 100644 --- a/Documentation/technical/api-parse-options.adoc +++ b/Documentation/technical/api-parse-options.adoc @@ -214,6 +214,13 @@ Macros There are some macros to easily define options: +Many of the macros below have an `_F` variant (for example `OPT_BOOL_F`, +`OPT_STRING_F`, `OPT_INTEGER_F`, `OPT_SET_INT_F`, `OPT_BIT_F` and +`OPT_CALLBACK_F`) that takes an additional trailing `flags` argument. +That argument is the bitwise-or of the per-option flags described in the +"Option flags" section above; the non-`_F` macros are simply defined +with `flags` set to `0`. + `OPT__ABBREV(&int_var)`:: Add `--abbrev[=]`. @@ -237,10 +244,21 @@ There are some macros to easily define options: describes the group or an empty string. Start the description with an upper-case letter. +`OPT_HIDDEN_GROUP(description)`:: + Like `OPT_GROUP()`, but the group header carries + `PARSE_OPT_HIDDEN`, so it is only shown by `--help-all` and not + by `-h`. Use it to label a group that contains only hidden + options, which would otherwise show an empty header under `-h`. + `OPT_BOOL(short, long, &int_var, description)`:: Introduce a boolean option. `int_var` is set to one with `--option` and set to zero with `--no-option`. +`OPT_HIDDEN_BOOL(short, long, &int_var, description)`:: + Like `OPT_BOOL()`, but the option carries `PARSE_OPT_HIDDEN`, + so it is hidden from `-h` while still being shown by + `--help-all`. + `OPT_COUNTUP(short, long, &int_var, description)`:: Introduce a count-up option. Each use of `--option` increments `int_var`, starting from zero From 62c1a58eebe81708f9c2408918f76713285e62a0 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 11 Aug 2026 10:33:06 +0200 Subject: [PATCH 132/259] fast-import: localize 'i' into the 'for' loops using it In cmd_fast_import(), a local variable 'i' is defined as an `unsigned int` and then used as a loop counter in four different `for (i = ...; i < ...; i++) { ... }` loops. But in three out of the four cases, `unsigned int` isn't the best type to use. To give each loop counter the type matching its bound (int/unsigned/size_t), let's localize 'i' into each loop that uses it. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano --- builtin/fast-import.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 6692f7cd812d0e..9fc9ebe65a6072 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -3937,8 +3937,6 @@ int cmd_fast_import(int argc, const char *prefix, struct repository *repo) { - unsigned int i; - show_usage_if_asked(argc, argv, fast_import_usage); reset_pack_idx_option(&pack_idx_opts); @@ -3959,7 +3957,7 @@ int cmd_fast_import(int argc, * line to override stream data). But we must do an early parse of any * command-line options that impact how we interpret the feature lines. */ - for (i = 1; i < argc; i++) { + for (int i = 1; i < argc; i++) { const char *arg = argv[i]; if (*arg != '-' || !strcmp(arg, "--")) break; @@ -3972,7 +3970,7 @@ int cmd_fast_import(int argc, global_prefix = prefix; rc_free = mem_pool_alloc(&fi_mem_pool, cmd_save * sizeof(*rc_free)); - for (i = 0; i < (cmd_save - 1); i++) + for (unsigned int i = 0; i < (cmd_save - 1); i++) rc_free[i].next = &rc_free[i + 1]; rc_free[cmd_save - 1].next = NULL; @@ -4035,9 +4033,9 @@ int cmd_fast_import(int argc, if (show_stats) { uintmax_t total_count = 0, duplicate_count = 0; - for (i = 0; i < ARRAY_SIZE(object_count_by_type); i++) + for (size_t i = 0; i < ARRAY_SIZE(object_count_by_type); i++) total_count += object_count_by_type[i]; - for (i = 0; i < ARRAY_SIZE(duplicate_count_by_type); i++) + for (size_t i = 0; i < ARRAY_SIZE(duplicate_count_by_type); i++) duplicate_count += duplicate_count_by_type[i]; fprintf(stderr, "%s statistics:\n", argv[0]); From a895a37c1bccbfc19892df14ea83c56ea4f50e87 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 11 Aug 2026 10:33:07 +0200 Subject: [PATCH 133/259] fast-import: use int for some bool flags The `show_stats` and `quiet` flags are meant to be parsed and used as boolean flags. To easily parse them using OPT_BOOL in a following commit, let's change their type from 'unsigned int' to just 'int'. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano --- builtin/fast-import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 9fc9ebe65a6072..9c8edd7c8936e6 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -182,8 +182,8 @@ static unsigned long branch_count; static unsigned long branch_load_count; static int failure; static FILE *pack_edges; -static unsigned int show_stats = 1; -static unsigned int quiet; +static int show_stats = 1; +static int quiet; static int global_argc; static const char **global_argv; static const char *global_prefix; From 63112e71a3f62cf49a7ee8f13d1250d1f02970ac Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 11 Aug 2026 10:33:08 +0200 Subject: [PATCH 134/259] fast-import: factor out option_*() functions In a following commit we are going to use the parse-options API to start parsing options. Some options will have to be parsed using OPT_CALLBACK as they process their arguments in special ways. When the processing code is already factored out in an option_*() function, like for `--date-format`, we can reuse that function. Unfortunately for other options the processing code has not been factored out yet. Let's do it now and factor out the code that handles the following options: - `--max-pack-size=` - `--big-file-threshold=` - `--signed-commits=` - `--signed-tags=` - `--quiet` into new option_*() functions: - option_max_pack_size() - option_big_file_threshold() - option_signed_commits() - option_signed_tags() - option_quiet() so that we can reuse these functions in following commits when the parse-option API will be used. Note that there are some behavior changes as we now die() with a proper error message when git_parse_ulong() cannot parse the argument from --max-pack-size or from --big-file-threshold. Previously we would end up calling die("unknown option") instead. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano --- builtin/fast-import.c | 69 ++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 9c8edd7c8936e6..a6e3cc00332c84 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -3751,25 +3751,55 @@ static void option_rewrite_submodules(const char *arg, struct string_list *list) free(s); } +static void option_max_pack_size(const char *arg) +{ + unsigned long v; + + if (!git_parse_ulong(arg, &v)) + die(_("--max-pack-size: argument must be a non-negative integer")); + if (v < 8192) { + warning(_("max-pack-size is now in bytes, assuming --max-pack-size=%lum"), v); + v *= 1024 * 1024; + } else if (v < 1024 * 1024) { + warning(_("minimum max-pack-size is 1 MiB")); + v = 1024 * 1024; + } + max_packsize = v; +} + +static void option_big_file_threshold(const char *arg) +{ + unsigned long v; + + if (!git_parse_ulong(arg, &v)) + die(_("--big-file-threshold: argument must be a non-negative integer")); + repo_settings_set_big_file_threshold(the_repository, v); +} + +static void option_signed_commits(const char *arg) +{ + if (parse_sign_mode(arg, &signed_commit_mode, &signed_commit_keyid)) + usagef(_("unknown --signed-commits mode '%s'"), arg); +} + +static void option_signed_tags(const char *arg) +{ + if (parse_sign_mode(arg, &signed_tag_mode, &signed_tag_keyid)) + usagef(_("unknown --signed-tags mode '%s'"), arg); +} + +static void option_quiet(void) +{ + show_stats = 0; + quiet = 1; +} + static int parse_one_option(const char *option) { if (skip_prefix(option, "max-pack-size=", &option)) { - unsigned long v; - if (!git_parse_ulong(option, &v)) - return 0; - if (v < 8192) { - warning(_("max-pack-size is now in bytes, assuming --max-pack-size=%lum"), v); - v *= 1024 * 1024; - } else if (v < 1024 * 1024) { - warning(_("minimum max-pack-size is 1 MiB")); - v = 1024 * 1024; - } - max_packsize = v; + option_max_pack_size(option); } else if (skip_prefix(option, "big-file-threshold=", &option)) { - unsigned long v; - if (!git_parse_ulong(option, &v)) - return 0; - repo_settings_set_big_file_threshold(the_repository, v); + option_big_file_threshold(option); } else if (skip_prefix(option, "depth=", &option)) { option_depth(option); } else if (skip_prefix(option, "active-branches=", &option)) { @@ -3777,14 +3807,11 @@ static int parse_one_option(const char *option) } else if (skip_prefix(option, "export-pack-edges=", &option)) { option_export_pack_edges(option); } else if (skip_prefix(option, "signed-commits=", &option)) { - if (parse_sign_mode(option, &signed_commit_mode, &signed_commit_keyid)) - usagef(_("unknown --signed-commits mode '%s'"), option); + option_signed_commits(option); } else if (skip_prefix(option, "signed-tags=", &option)) { - if (parse_sign_mode(option, &signed_tag_mode, &signed_tag_keyid)) - usagef(_("unknown --signed-tags mode '%s'"), option); + option_signed_tags(option); } else if (!strcmp(option, "quiet")) { - show_stats = 0; - quiet = 1; + option_quiet(); } else if (!strcmp(option, "stats")) { show_stats = 1; } else if (!strcmp(option, "allow-unsafe-features")) { From 417afb2b4d2d2c02121300d64e874cb97e23531c Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 11 Aug 2026 10:33:09 +0200 Subject: [PATCH 135/259] fast-import: introduce 'struct fast_import_state' "builtin/fast-import.c" uses a large number of global variables. This makes it harder than necessary to reason about and improve. Especially adding new features requires adding more global variables, while modernizing and eventually libifying the code becomes more and more difficult. To start reverting the sad trend to more and more globals and to start cleaning things up, let's introduce a 'struct fast_import_state' and pass an instance of it as the first argument to many functions. This is similar to what was done for "builtin/apply.c" by introducing a 'struct apply_state', see 07d7e290ff (apply: move 'struct apply_state' to a header file, 2016-08-11) and related commits. As a first step only the 'global_argc', 'global_argv' and 'global_prefix' variables are moved into the new struct. More variables will be moved into it in the following commits. Some functions receive the new 'state' parameter only to pass it along or for future use, so they are marked with UNUSED for now to satisfy '-Werror=unused-parameter'. This is a mostly mechanical refactoring with no intended behavior change. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano --- builtin/fast-import.c | 294 ++++++++++++++++++++++++------------------ 1 file changed, 169 insertions(+), 125 deletions(-) diff --git a/builtin/fast-import.c b/builtin/fast-import.c index a6e3cc00332c84..0f838d8488158c 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -184,10 +184,6 @@ static int failure; static FILE *pack_edges; static int show_stats = 1; static int quiet; -static int global_argc; -static const char **global_argv; -static const char *global_prefix; - static enum sign_mode signed_tag_mode = SIGN_VERBATIM; static enum sign_mode signed_commit_mode = SIGN_VERBATIM; static const char *signed_commit_keyid; @@ -276,10 +272,29 @@ static kh_oid_map_t *sub_oid_map; /* Where to write output of cat-blob commands */ static int cat_blob_fd = STDOUT_FILENO; -static void parse_argv(void); -static void parse_get_mark(const char *p); -static void parse_cat_blob(const char *p); -static void parse_ls(const char *p, struct branch *b); +/* Command state */ +struct fast_import_state { + int argc; + const char **argv; + const char *prefix; +}; + +static void fast_import_state_init(struct fast_import_state *state, + int argc, const char **argv, + const char *prefix) +{ + memset(state, 0, sizeof(*state)); + state->argc = argc; + state->argv = argv; + state->prefix = prefix; +} + +static void parse_argv(struct fast_import_state *state); +static void parse_get_mark(struct fast_import_state *state, const char *p); +static void parse_cat_blob(struct fast_import_state *state, const char *p); +static void parse_ls(struct fast_import_state *state, + const char *p, + struct branch *b); static void for_each_mark(struct mark_set *m, uintmax_t base, each_mark_fn_t callback, void *p) { @@ -1845,7 +1860,7 @@ static void read_marks(void) } -static int read_next_command(void) +static int read_next_command(struct fast_import_state *state) { static int stdin_eof = 0; @@ -1867,7 +1882,7 @@ static int read_next_command(void) if (!seen_data_command && !starts_with(command_buf.buf, "feature ") && !starts_with(command_buf.buf, "option ")) { - parse_argv(); + parse_argv(state); } rc = rc_free; @@ -1899,22 +1914,22 @@ static void skip_optional_lf(void) ungetc(term_char, stdin); } -static void parse_mark(void) +static void parse_mark(struct fast_import_state *state) { const char *v; if (skip_prefix(command_buf.buf, "mark :", &v)) { next_mark = strtoumax(v, NULL, 10); - read_next_command(); + read_next_command(state); } else next_mark = 0; } -static void parse_original_identifier(void) +static void parse_original_identifier(struct fast_import_state *state) { const char *v; if (skip_prefix(command_buf.buf, "original-oid ", &v)) - read_next_command(); + read_next_command(state); } static int parse_data(struct strbuf *sb, uintmax_t limit, uintmax_t *len_res) @@ -2068,11 +2083,11 @@ static void parse_and_store_blob( } } -static void parse_new_blob(void) +static void parse_new_blob(struct fast_import_state *state) { - read_next_command(); - parse_mark(); - parse_original_identifier(); + read_next_command(state); + parse_mark(state); + parse_original_identifier(state); parse_and_store_blob(&last_blob, NULL, next_mark); } @@ -2368,7 +2383,9 @@ static void parse_path_space(struct strbuf *sb, const char *p, (*endp)++; } -static void file_change_m(const char *p, struct branch *b) +static void file_change_m(struct fast_import_state *state, + const char *p, + struct branch *b) { static struct strbuf path = STRBUF_INIT; struct object_entry *oe; @@ -2435,10 +2452,10 @@ static void file_change_m(const char *p, struct branch *b) if (S_ISDIR(mode)) die(_("directories cannot be specified 'inline': %s"), command_buf.buf); - while (read_next_command() != EOF) { + while (read_next_command(state) != EOF) { const char *v; if (skip_prefix(command_buf.buf, "cat-blob ", &v)) - parse_cat_blob(v); + parse_cat_blob(state, v); else { parse_and_store_blob(&last_blob, &oid, 0); break; @@ -2512,7 +2529,10 @@ static void file_change_cr(const char *p, struct branch *b, int rename) leaf.tree); } -static void note_change_n(const char *p, struct branch *b, unsigned char *old_fanout) +static void note_change_n(struct fast_import_state *state, + const char *p, + struct branch *b, + unsigned char *old_fanout) { struct object_entry *oe; struct branch *s; @@ -2577,7 +2597,7 @@ static void note_change_n(const char *p, struct branch *b, unsigned char *old_fa die(_("invalid ref name or SHA1 expression: %s"), p); if (inline_data) { - read_next_command(); + read_next_command(state); parse_and_store_blob(&last_blob, &oid, 0); } else if (oe) { if (oe->type != OBJ_BLOB) @@ -2644,7 +2664,9 @@ static void parse_from_existing(struct branch *b) } } -static int parse_objectish(struct branch *b, const char *objectish) +static int parse_objectish(struct fast_import_state *state, + struct branch *b, + const char *objectish) { struct branch *s; struct object_id oid; @@ -2687,31 +2709,34 @@ static int parse_objectish(struct branch *b, const char *objectish) b->branch_tree.tree = NULL; } - read_next_command(); + read_next_command(state); return 1; } -static int parse_from(struct branch *b) +static int parse_from(struct fast_import_state *state, struct branch *b) { const char *from; if (!skip_prefix(command_buf.buf, "from ", &from)) return 0; - return parse_objectish(b, from); + return parse_objectish(state, b, from); } -static int parse_objectish_with_prefix(struct branch *b, const char *prefix) +static int parse_objectish_with_prefix(struct fast_import_state *state, + struct branch *b, + const char *prefix) { const char *base; if (!skip_prefix(command_buf.buf, prefix, &base)) return 0; - return parse_objectish(b, base); + return parse_objectish(state, b, base); } -static struct hash_list *parse_merge(unsigned int *count) +static struct hash_list *parse_merge(struct fast_import_state *state, + unsigned int *count) { struct hash_list *list = NULL, **tail = &list, *n; const char *from; @@ -2745,7 +2770,7 @@ static struct hash_list *parse_merge(unsigned int *count) tail = &n->next; (*count)++; - read_next_command(); + read_next_command(state); } return list; } @@ -2756,7 +2781,9 @@ struct signature_data { struct strbuf data; /* The actual signature data */ }; -static void parse_one_signature(struct signature_data *sig, const char *v) +static void parse_one_signature(struct fast_import_state *state, + struct signature_data *sig, + const char *v) { char *args = xstrdup(v); /* Will be freed when sig->hash_algo is freed */ char *space = strchr(args, ' '); @@ -2781,15 +2808,15 @@ static void parse_one_signature(struct signature_data *sig, const char *v) warning(_("'unknown' signature format in gpgsig")); /* Read signature data */ - read_next_command(); + read_next_command(state); parse_data(&sig->data, 0, NULL); } -static void discard_one_signature(void) +static void discard_one_signature(struct fast_import_state *state) { struct strbuf data = STRBUF_INIT; - read_next_command(); + read_next_command(state); parse_data(&data, 0, NULL); strbuf_release(&data); } @@ -2827,13 +2854,14 @@ static void store_signature(struct signature_data *stored_sig, } } -static void import_one_signature(struct signature_data *sig_sha1, +static void import_one_signature(struct fast_import_state *state, + struct signature_data *sig_sha1, struct signature_data *sig_sha256, const char *v) { struct signature_data sig = { NULL, NULL, STRBUF_INIT }; - parse_one_signature(&sig, v); + parse_one_signature(state, &sig, v); if (!strcmp(sig.hash_algo, "sha1")) store_signature(sig_sha1, &sig, "SHA-1"); @@ -2947,7 +2975,7 @@ static void handle_signature_if_invalid(struct strbuf *new_data, strbuf_release(&tmp_buf); } -static void parse_new_commit(const char *arg) +static void parse_new_commit(struct fast_import_state *state, const char *arg) { static struct strbuf msg = STRBUF_INIT; struct signature_data sig_sha1 = { NULL, NULL, STRBUF_INIT }; @@ -2965,16 +2993,16 @@ static void parse_new_commit(const char *arg) if (!b) b = new_branch(arg); - read_next_command(); - parse_mark(); - parse_original_identifier(); + read_next_command(state); + parse_mark(state); + parse_original_identifier(state); if (skip_prefix(command_buf.buf, "author ", &v)) { author = parse_ident(v); - read_next_command(); + read_next_command(state); } if (skip_prefix(command_buf.buf, "committer ", &v)) { committer = parse_ident(v); - read_next_command(); + read_next_command(state); } if (!committer) die(_("expected committer but didn't get one")); @@ -2990,7 +3018,7 @@ static void parse_new_commit(const char *arg) warning(_("stripping a commit signature")); /* fallthru */ case SIGN_STRIP: - discard_one_signature(); + discard_one_signature(state); break; /* Second, modes that parse the signature */ @@ -3001,24 +3029,24 @@ static void parse_new_commit(const char *arg) case SIGN_STRIP_IF_INVALID: case SIGN_SIGN_IF_INVALID: case SIGN_ABORT_IF_INVALID: - import_one_signature(&sig_sha1, &sig_sha256, v); + import_one_signature(state, &sig_sha1, &sig_sha256, v); break; /* Third, BUG */ default: BUG("invalid signed_commit_mode value %d", signed_commit_mode); } - read_next_command(); + read_next_command(state); } if (skip_prefix(command_buf.buf, "encoding ", &v)) { encoding = xstrdup(v); - read_next_command(); + read_next_command(state); } parse_data(&msg, 0, NULL); - read_next_command(); - parse_from(b); - merge_list = parse_merge(&merge_count); + read_next_command(state); + parse_from(state, b); + merge_list = parse_merge(state, &merge_count); /* ensure the branch is active/loaded */ if (!b->branch_tree.tree || !max_active_branches) { @@ -3031,7 +3059,7 @@ static void parse_new_commit(const char *arg) /* file_change* */ while (command_buf.len > 0) { if (skip_prefix(command_buf.buf, "M ", &v)) - file_change_m(v, b); + file_change_m(state, v, b); else if (skip_prefix(command_buf.buf, "D ", &v)) file_change_d(v, b); else if (skip_prefix(command_buf.buf, "R ", &v)) @@ -3039,18 +3067,18 @@ static void parse_new_commit(const char *arg) else if (skip_prefix(command_buf.buf, "C ", &v)) file_change_cr(v, b, 0); else if (skip_prefix(command_buf.buf, "N ", &v)) - note_change_n(v, b, &prev_fanout); + note_change_n(state, v, b, &prev_fanout); else if (!strcmp("deleteall", command_buf.buf)) file_change_deleteall(b); else if (skip_prefix(command_buf.buf, "ls ", &v)) - parse_ls(v, b); + parse_ls(state, v, b); else if (skip_prefix(command_buf.buf, "cat-blob ", &v)) - parse_cat_blob(v); + parse_cat_blob(state, v); else { unread_command_buf = 1; break; } - if (read_next_command() == EOF) + if (read_next_command(state) == EOF) break; } @@ -3188,7 +3216,7 @@ static void handle_tag_signature(struct strbuf *buf, struct strbuf *msg, const c } } -static void parse_new_tag(const char *arg) +static void parse_new_tag(struct fast_import_state *state, const char *arg) { static struct strbuf msg = STRBUF_INIT; const char *from; @@ -3207,8 +3235,8 @@ static void parse_new_tag(const char *arg) else first_tag = t; last_tag = t; - read_next_command(); - parse_mark(); + read_next_command(state); + parse_mark(state); /* from ... */ if (!skip_prefix(command_buf.buf, "from ", &from)) @@ -3236,15 +3264,15 @@ static void parse_new_tag(const char *arg) type = oe->type; } else die(_("invalid ref name or SHA1 expression: %s"), from); - read_next_command(); + read_next_command(state); /* original-oid ... */ - parse_original_identifier(); + parse_original_identifier(state); /* tagger ... */ if (skip_prefix(command_buf.buf, "tagger ", &v)) { tagger = parse_ident(v); - read_next_command(); + read_next_command(state); } else tagger = NULL; @@ -3275,7 +3303,7 @@ static void parse_new_tag(const char *arg) t->pack_id = pack_id; } -static void parse_reset_branch(const char *arg) +static void parse_reset_branch(struct fast_import_state *state, const char *arg) { struct branch *b; const char *tag_name; @@ -3292,8 +3320,8 @@ static void parse_reset_branch(const char *arg) } else b = new_branch(arg); - read_next_command(); - parse_from(b); + read_next_command(state); + parse_from(state, b); if (b->delete && skip_prefix(b->name, "refs/tags/", &tag_name)) { /* * Elsewhere, we call dump_branches() before dump_tags(), @@ -3378,7 +3406,8 @@ static void cat_blob(struct object_entry *oe, struct object_id *oid) free(buf); } -static void parse_get_mark(const char *p) +static void parse_get_mark(struct fast_import_state *state UNUSED, + const char *p) { struct object_entry *oe; char output[GIT_MAX_HEXSZ + 2]; @@ -3395,7 +3424,8 @@ static void parse_get_mark(const char *p) cat_blob_write(output, the_hash_algo->hexsz + 1); } -static void parse_cat_blob(const char *p) +static void parse_cat_blob(struct fast_import_state *state UNUSED, + const char *p) { struct object_entry *oe; struct object_id oid; @@ -3560,7 +3590,9 @@ static void print_ls(int mode, const unsigned char *hash, const char *path) cat_blob_write(line.buf, line.len); } -static void parse_ls(const char *p, struct branch *b) +static void parse_ls(struct fast_import_state *state UNUSED, + const char *p, + struct branch *b) { static struct strbuf path = STRBUF_INIT; struct tree_entry *root = NULL; @@ -3607,13 +3639,13 @@ static void checkpoint(void) dump_marks(); } -static void parse_checkpoint(void) +static void parse_checkpoint(struct fast_import_state *state UNUSED) { checkpoint_requested = 1; skip_optional_lf(); } -static void parse_progress(void) +static void parse_progress(struct fast_import_state *state UNUSED) { fwrite(command_buf.buf, 1, command_buf.len, stdout); fputc('\n', stdout); @@ -3621,37 +3653,40 @@ static void parse_progress(void) skip_optional_lf(); } -static void parse_alias(void) +static void parse_alias(struct fast_import_state *state) { struct object_entry *e; struct branch b; skip_optional_lf(); - read_next_command(); + read_next_command(state); /* mark ... */ - parse_mark(); + parse_mark(state); if (!next_mark) die(_("expected 'mark' command, got %s"), command_buf.buf); /* to ... */ memset(&b, 0, sizeof(b)); - if (!parse_objectish_with_prefix(&b, "to ")) + if (!parse_objectish_with_prefix(state, &b, "to ")) die(_("expected 'to' command, got %s"), command_buf.buf); e = find_object(&b.oid); assert(e); insert_mark(&marks, next_mark, e); } -static char* make_fast_import_path(const char *path) +static char* make_fast_import_path(struct fast_import_state *state, + const char *path) { if (!relative_marks_paths || is_absolute_path(path)) - return prefix_filename(global_prefix, path); + return prefix_filename(state->prefix, path); return repo_git_path(the_repository, "info/fast-import/%s", path); } -static void option_import_marks(const char *marks, - int from_stream, int ignore_missing) +static void option_import_marks(struct fast_import_state *state, + const char *marks, + int from_stream, + int ignore_missing) { if (import_marks_file) { if (from_stream) @@ -3663,7 +3698,7 @@ static void option_import_marks(const char *marks, } free(import_marks_file); - import_marks_file = make_fast_import_path(marks); + import_marks_file = make_fast_import_path(state, marks); import_marks_file_from_stream = from_stream; import_marks_file_ignore_missing = ignore_missing; } @@ -3703,13 +3738,15 @@ static void option_active_branches(const char *branches) max_active_branches = ulong_arg("--active-branches", branches); } -static void option_export_marks(const char *marks) +static void option_export_marks(struct fast_import_state *state, + const char *marks) { free(export_marks_file); - export_marks_file = make_fast_import_path(marks); + export_marks_file = make_fast_import_path(state, marks); } -static void option_cat_blob_fd(const char *fd) +static void option_cat_blob_fd(struct fast_import_state *state UNUSED, + const char *fd) { unsigned long n = ulong_arg("--cat-blob-fd", fd); if (n > (unsigned long) INT_MAX) @@ -3717,16 +3754,19 @@ static void option_cat_blob_fd(const char *fd) cat_blob_fd = (int) n; } -static void option_export_pack_edges(const char *edges) +static void option_export_pack_edges(struct fast_import_state *state, + const char *edges) { - char *fn = prefix_filename(global_prefix, edges); + char *fn = prefix_filename(state->prefix, edges); if (pack_edges) fclose(pack_edges); pack_edges = xfopen(fn, "a"); free(fn); } -static void option_rewrite_submodules(const char *arg, struct string_list *list) +static void option_rewrite_submodules(struct fast_import_state *state, + const char *arg, + struct string_list *list) { struct mark_set *ms; FILE *fp; @@ -3738,7 +3778,7 @@ static void option_rewrite_submodules(const char *arg, struct string_list *list) f++; CALLOC_ARRAY(ms, 1); - f = prefix_filename(global_prefix, f); + f = prefix_filename(state->prefix, f); fp = fopen(f, "r"); if (!fp) die_errno(_("cannot read '%s'"), f); @@ -3794,7 +3834,7 @@ static void option_quiet(void) quiet = 1; } -static int parse_one_option(const char *option) +static int parse_one_option(struct fast_import_state *state, const char *option) { if (skip_prefix(option, "max-pack-size=", &option)) { option_max_pack_size(option); @@ -3805,7 +3845,7 @@ static int parse_one_option(const char *option) } else if (skip_prefix(option, "active-branches=", &option)) { option_active_branches(option); } else if (skip_prefix(option, "export-pack-edges=", &option)) { - option_export_pack_edges(option); + option_export_pack_edges(state, option); } else if (skip_prefix(option, "signed-commits=", &option)) { option_signed_commits(option); } else if (skip_prefix(option, "signed-tags=", &option)) { @@ -3823,34 +3863,38 @@ static int parse_one_option(const char *option) return 1; } -static void check_unsafe_feature(const char *feature, int from_stream) +static void check_unsafe_feature(struct fast_import_state *state UNUSED, + const char *feature, + int from_stream) { if (from_stream && !allow_unsafe_features) die(_("feature '%s' forbidden in input without --allow-unsafe-features"), feature); } -static int parse_one_feature(const char *feature, int from_stream) +static int parse_one_feature(struct fast_import_state *state, + const char *feature, + int from_stream) { const char *arg; if (skip_prefix(feature, "date-format=", &arg)) { option_date_format(arg); } else if (skip_prefix(feature, "import-marks=", &arg)) { - check_unsafe_feature("import-marks", from_stream); - option_import_marks(arg, from_stream, 0); + check_unsafe_feature(state, "import-marks", from_stream); + option_import_marks(state, arg, from_stream, 0); } else if (skip_prefix(feature, "import-marks-if-exists=", &arg)) { - check_unsafe_feature("import-marks-if-exists", from_stream); - option_import_marks(arg, from_stream, 1); + check_unsafe_feature(state, "import-marks-if-exists", from_stream); + option_import_marks(state, arg, from_stream, 1); } else if (skip_prefix(feature, "export-marks=", &arg)) { - check_unsafe_feature(feature, from_stream); - option_export_marks(arg); + check_unsafe_feature(state, feature, from_stream); + option_export_marks(state, arg); } else if (!strcmp(feature, "alias")) { ; /* Don't die - this feature is supported */ } else if (skip_prefix(feature, "rewrite-submodules-to=", &arg)) { - option_rewrite_submodules(arg, &sub_marks_to); + option_rewrite_submodules(state, arg, &sub_marks_to); } else if (skip_prefix(feature, "rewrite-submodules-from=", &arg)) { - option_rewrite_submodules(arg, &sub_marks_from); + option_rewrite_submodules(state, arg, &sub_marks_from); } else if (!strcmp(feature, "get-mark")) { ; /* Don't die - this feature is supported */ } else if (!strcmp(feature, "cat-blob")) { @@ -3872,23 +3916,23 @@ static int parse_one_feature(const char *feature, int from_stream) return 1; } -static void parse_feature(const char *feature) +static void parse_feature(struct fast_import_state *state, const char *feature) { if (seen_data_command) die(_("got feature command '%s' after data command"), feature); - if (parse_one_feature(feature, 1)) + if (parse_one_feature(state, feature, 1)) return; die(_("this version of fast-import does not support feature %s."), feature); } -static void parse_option(const char *option) +static void parse_option(struct fast_import_state *state, const char *option) { if (seen_data_command) die(_("got option command '%s' after data command"), option); - if (parse_one_option(option)) + if (parse_one_option(state, option)) return; die(_("this version of fast-import does not support option: %s"), option); @@ -3924,12 +3968,12 @@ static void git_pack_config(void) static const char fast_import_usage[] = "git fast-import [--date-format=] [--max-pack-size=] [--big-file-threshold=] [--depth=] [--active-branches=] [--export-marks=]"; -static void parse_argv(void) +static void parse_argv(struct fast_import_state *state) { unsigned int i; - for (i = 1; i < global_argc; i++) { - const char *a = global_argv[i]; + for (i = 1; i < state->argc; i++) { + const char *a = state->argv[i]; if (*a != '-' || !strcmp(a, "--")) break; @@ -3937,20 +3981,20 @@ static void parse_argv(void) if (!skip_prefix(a, "--", &a)) die(_("unknown option %s"), a); - if (parse_one_option(a)) + if (parse_one_option(state, a)) continue; - if (parse_one_feature(a, 0)) + if (parse_one_feature(state, a, 0)) continue; if (skip_prefix(a, "cat-blob-fd=", &a)) { - option_cat_blob_fd(a); + option_cat_blob_fd(state, a); continue; } die(_("unknown option --%s"), a); } - if (i != global_argc) + if (i != state->argc) usage(fast_import_usage); seen_data_command = 1; @@ -3964,6 +4008,8 @@ int cmd_fast_import(int argc, const char *prefix, struct repository *repo) { + struct fast_import_state state; + show_usage_if_asked(argc, argv, fast_import_usage); reset_pack_idx_option(&pack_idx_opts); @@ -3992,9 +4038,7 @@ int cmd_fast_import(int argc, allow_unsafe_features = 1; } - global_argc = argc; - global_argv = argv; - global_prefix = prefix; + fast_import_state_init(&state, argc, argv, prefix); rc_free = mem_pool_alloc(&fi_mem_pool, cmd_save * sizeof(*rc_free)); for (unsigned int i = 0; i < (cmd_save - 1); i++) @@ -4004,34 +4048,34 @@ int cmd_fast_import(int argc, start_packfile(); set_die_routine(die_nicely); set_checkpoint_signal(); - while (read_next_command() != EOF) { + while (read_next_command(&state) != EOF) { const char *v; if (!strcmp("blob", command_buf.buf)) - parse_new_blob(); + parse_new_blob(&state); else if (skip_prefix(command_buf.buf, "commit ", &v)) - parse_new_commit(v); + parse_new_commit(&state, v); else if (skip_prefix(command_buf.buf, "tag ", &v)) - parse_new_tag(v); + parse_new_tag(&state, v); else if (skip_prefix(command_buf.buf, "reset ", &v)) - parse_reset_branch(v); + parse_reset_branch(&state, v); else if (skip_prefix(command_buf.buf, "ls ", &v)) - parse_ls(v, NULL); + parse_ls(&state, v, NULL); else if (skip_prefix(command_buf.buf, "cat-blob ", &v)) - parse_cat_blob(v); + parse_cat_blob(&state, v); else if (skip_prefix(command_buf.buf, "get-mark ", &v)) - parse_get_mark(v); + parse_get_mark(&state, v); else if (!strcmp("checkpoint", command_buf.buf)) - parse_checkpoint(); + parse_checkpoint(&state); else if (!strcmp("done", command_buf.buf)) break; else if (!strcmp("alias", command_buf.buf)) - parse_alias(); + parse_alias(&state); else if (starts_with(command_buf.buf, "progress ")) - parse_progress(); + parse_progress(&state); else if (skip_prefix(command_buf.buf, "feature ", &v)) - parse_feature(v); + parse_feature(&state, v); else if (skip_prefix(command_buf.buf, "option git ", &v)) - parse_option(v); + parse_option(&state, v); else if (starts_with(command_buf.buf, "option ")) /* ignore non-git options*/; else @@ -4043,7 +4087,7 @@ int cmd_fast_import(int argc, /* argv hasn't been parsed yet, do so */ if (!seen_data_command) - parse_argv(); + parse_argv(&state); if (require_explicit_termination && feof(stdin)) die(_("stream ends early")); From ee996af567434d3fc5204ab8f53a0ade5803303d Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 11 Aug 2026 10:33:10 +0200 Subject: [PATCH 136/259] fast-import: move command state globals into 'struct fast_import_state' A previous commit introduced 'struct fast_import_state' to hold some command state, and reduce the need for global variables. Let's continue in the same direction and move two more global variables that describe the command state into it: 'seen_data_command' and 'allow_unsafe_features'. All the sites accessing these variables are already in functions that receive the 'state' parameter (or in cmd_fast_import() which owns the struct), so no additional threading is needed. As 'state->allow_unsafe_features' is now dereferenced in check_unsafe_feature(), its 'state' parameter is no longer unused, so the UNUSED marker is removed. The fast_import_state_init() call is moved up before the early command-line scan for '--allow-unsafe-features', so that this option can be recorded directly into the struct without being clobbered by the memset() in fast_import_state_init(). This is a mechanical refactoring with no intended behavior change. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano --- builtin/fast-import.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 0f838d8488158c..52da29c1bde64e 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -257,9 +257,7 @@ static struct recent_command *rc_free; static unsigned int cmd_save = 100; static uintmax_t next_mark; static struct strbuf new_data = STRBUF_INIT; -static int seen_data_command; static int require_explicit_termination; -static int allow_unsafe_features; /* Signal handling */ static volatile sig_atomic_t checkpoint_requested; @@ -277,6 +275,8 @@ struct fast_import_state { int argc; const char **argv; const char *prefix; + int seen_data_command; + int allow_unsafe_features; }; static void fast_import_state_init(struct fast_import_state *state, @@ -1879,7 +1879,7 @@ static int read_next_command(struct fast_import_state *state) if (stdin_eof) return EOF; - if (!seen_data_command + if (!state->seen_data_command && !starts_with(command_buf.buf, "feature ") && !starts_with(command_buf.buf, "option ")) { parse_argv(state); @@ -3863,11 +3863,11 @@ static int parse_one_option(struct fast_import_state *state, const char *option) return 1; } -static void check_unsafe_feature(struct fast_import_state *state UNUSED, +static void check_unsafe_feature(struct fast_import_state *state, const char *feature, int from_stream) { - if (from_stream && !allow_unsafe_features) + if (from_stream && !state->allow_unsafe_features) die(_("feature '%s' forbidden in input without --allow-unsafe-features"), feature); } @@ -3918,7 +3918,7 @@ static int parse_one_feature(struct fast_import_state *state, static void parse_feature(struct fast_import_state *state, const char *feature) { - if (seen_data_command) + if (state->seen_data_command) die(_("got feature command '%s' after data command"), feature); if (parse_one_feature(state, feature, 1)) @@ -3929,7 +3929,7 @@ static void parse_feature(struct fast_import_state *state, const char *feature) static void parse_option(struct fast_import_state *state, const char *option) { - if (seen_data_command) + if (state->seen_data_command) die(_("got option command '%s' after data command"), option); if (parse_one_option(state, option)) @@ -3997,7 +3997,7 @@ static void parse_argv(struct fast_import_state *state) if (i != state->argc) usage(fast_import_usage); - seen_data_command = 1; + state->seen_data_command = 1; if (import_marks_file) read_marks(); build_mark_map(&sub_marks_from, &sub_marks_to); @@ -4012,6 +4012,8 @@ int cmd_fast_import(int argc, show_usage_if_asked(argc, argv, fast_import_usage); + fast_import_state_init(&state, argc, argv, prefix); + reset_pack_idx_option(&pack_idx_opts); git_pack_config(); @@ -4035,11 +4037,9 @@ int cmd_fast_import(int argc, if (*arg != '-' || !strcmp(arg, "--")) break; if (!strcmp(arg, "--allow-unsafe-features")) - allow_unsafe_features = 1; + state.allow_unsafe_features = 1; } - fast_import_state_init(&state, argc, argv, prefix); - rc_free = mem_pool_alloc(&fi_mem_pool, cmd_save * sizeof(*rc_free)); for (unsigned int i = 0; i < (cmd_save - 1); i++) rc_free[i].next = &rc_free[i + 1]; @@ -4086,7 +4086,7 @@ int cmd_fast_import(int argc, } /* argv hasn't been parsed yet, do so */ - if (!seen_data_command) + if (!state.seen_data_command) parse_argv(&state); if (require_explicit_termination && feof(stdin)) From 27ab5815f72848e9a25e11ee93d6b036a338933b Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 11 Aug 2026 10:33:11 +0200 Subject: [PATCH 137/259] fast-import: use struct option for usage string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently `git fast-import -h` shows the following on a single line: usage : git fast-import [--date-format=] [--max-pack-size=] \ [--big-file-threshold=] [--depth=] \ [--active-branches=] \ [--export-marks=] This output has a number of issues like: - It's missing a lot of options. - It's not consistent with the SYNOPSIS section of the doc. - With `--help-all` instead of `-h` additional hidden options should be shown, but that's not the case. - It's not standard style anymore. - Most other Git commands show additional lines for most of the options they support. Also while most commands use the parse-options API to handle their options, "builtin/fast-import.c" still doesn't use it. Let's improve on that by using the parse-options API to display the options when `-h` and `--help-all` are used. While at it, let's make the SYNOPSIS section of "Documentation/git-fast-import.adoc" consistent with the new usage string. This deliberately leaves it to future work to also use the parse-options API to actually parse the options. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano --- Documentation/git-fast-import.adoc | 2 +- builtin/fast-import.c | 83 +++++++++++++++++++++++++++--- t/t0450/adoc-help-mismatches | 1 - 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/Documentation/git-fast-import.adoc b/Documentation/git-fast-import.adoc index d68bc52b7e9cd7..7c5900e048cefb 100644 --- a/Documentation/git-fast-import.adoc +++ b/Documentation/git-fast-import.adoc @@ -9,7 +9,7 @@ git-fast-import - Backend for fast Git data importers SYNOPSIS -------- [verse] -frontend | 'git fast-import' [] +'git fast-import' [] DESCRIPTION ----------- diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 52da29c1bde64e..879c2860439ea3 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -30,6 +30,7 @@ #include "khash.h" #include "date.h" #include "gpg-interface.h" +#include "parse-options.h" #define PACK_ID_BITS 16 #define MAX_PACK_ID ((1<argc = argc; state->argv = argv; state->prefix = prefix; + state->option = option; } static void parse_argv(struct fast_import_state *state); @@ -3965,8 +3968,10 @@ static void git_pack_config(void) repo_config(the_repository, git_default_config, NULL); } -static const char fast_import_usage[] = -"git fast-import [--date-format=] [--max-pack-size=] [--big-file-threshold=] [--depth=] [--active-branches=] [--export-marks=]"; +static const char *const fast_import_usage[] = { + N_("git fast-import []"), + NULL +}; static void parse_argv(struct fast_import_state *state) { @@ -3995,7 +4000,7 @@ static void parse_argv(struct fast_import_state *state) die(_("unknown option --%s"), a); } if (i != state->argc) - usage(fast_import_usage); + usage_with_options(fast_import_usage, state->option); state->seen_data_command = 1; if (import_marks_file) @@ -4010,9 +4015,75 @@ int cmd_fast_import(int argc, { struct fast_import_state state; - show_usage_if_asked(argc, argv, fast_import_usage); + unsigned long pack_size_limit, big_file_threshold; + char *edges, *signed_commits, *signed_tags, *date_format; + char *import_marks_if_exists, *submodules_from, *submodules_to; - fast_import_state_init(&state, argc, argv, prefix); + /* + * NEEDSWORK: For now this is used only to render + * `-h`/`--help-all` usage messages. The actual parsing is + * done by parse_one_option()/parse_one_feature(). + */ + struct option fast_import_options[] = { + OPT_GROUP(N_("Common")), + OPT_STRING_F(0, "date-format", &date_format, N_("fmt"), + N_("format of the commit/tag dates"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "stats", &show_stats, + N_("display some basic statistics (objects, packfiles and memory)"), + PARSE_OPT_NONEG), + OPT_BOOL_F(0, "quiet", &quiet, + N_("disable the output shown by --stats"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "force", &force_update, + N_("force updating modified existing branches"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "done", &require_explicit_termination, + N_("require a terminating 'done' command"), PARSE_OPT_NONEG), + OPT_UNSIGNED(0, "max-pack-size", &pack_size_limit, + N_("maximum size of each output pack file")), + OPT_UNSIGNED(0, "big-file-threshold", &big_file_threshold, + N_("maximum size of a blob that will be deltified")), + OPT_UNSIGNED(0, "depth", &max_depth, + N_("maximum delta depth")), + OPT_UNSIGNED(0, "active-branches", &max_active_branches, + N_("maximum number of branches to maintain active")), + OPT_GROUP(N_("Marks")), + OPT_STRING_F(0, "import-marks", &import_marks_file, N_("file"), + N_("import marks from "), PARSE_OPT_NONEG), + OPT_STRING_F(0, "import-marks-if-exists", &import_marks_if_exists, N_("file"), + N_("import marks from if it exists"), PARSE_OPT_NONEG), + OPT_STRING_F(0, "export-marks", &export_marks_file, N_("file"), + N_("dump marks to "), PARSE_OPT_NONEG), + OPT_BOOL(0, "relative-marks", &relative_marks_paths, + N_("are --(import|export)-marks= paths relative to '.git/info/fast-import'?")), + OPT_GROUP(N_("Submodule rewrite")), + OPT_STRING_F(0, "rewrite-submodules-from", &submodules_from, N_("name:filename"), + N_("rewrite object IDs for submodule from "), + PARSE_OPT_NONEG), + OPT_STRING_F(0, "rewrite-submodules-to", &submodules_to, N_("name:filename"), + N_("rewrite object IDs for submodule to "), + PARSE_OPT_NONEG), + OPT_GROUP(N_("Signing")), + OPT_STRING_F(0, "signed-commits", &signed_commits, N_("mode"), + N_("how to handle signed commits"), + PARSE_OPT_NONEG), + OPT_STRING_F(0, "signed-tags", &signed_tags, N_("mode"), + N_("how to handle signed tags"), + PARSE_OPT_NONEG), + OPT_HIDDEN_GROUP(N_("Advanced")), + OPT_BOOL_F(0, "allow-unsafe-features", &state.allow_unsafe_features, + N_("allow unsafe mark commands from the stream"), + PARSE_OPT_HIDDEN | PARSE_OPT_NONEG), + OPT_STRING_F(0, "export-pack-edges", &edges, N_("file"), + N_("dump edge commits to "), + PARSE_OPT_HIDDEN | PARSE_OPT_NONEG), + OPT_INTEGER_F(0, "cat-blob-fd", &cat_blob_fd, + N_("write some responses to instead of stdout"), + PARSE_OPT_HIDDEN | PARSE_OPT_NONEG), + OPT_END() + }; + + show_usage_with_options_if_asked(argc, argv, fast_import_usage, fast_import_options); + + fast_import_state_init(&state, argc, argv, prefix, fast_import_options); reset_pack_idx_option(&pack_idx_opts); git_pack_config(); diff --git a/t/t0450/adoc-help-mismatches b/t/t0450/adoc-help-mismatches index c4a55ff4e35a4f..baf3b1d80927d4 100644 --- a/t/t0450/adoc-help-mismatches +++ b/t/t0450/adoc-help-mismatches @@ -12,7 +12,6 @@ column credential credential-cache credential-store -fast-import fetch-pack fmt-merge-msg format-patch From 87cf4d72999cdfdb343964e2027ce4a3c6cb8d04 Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 11 Aug 2026 10:33:12 +0200 Subject: [PATCH 138/259] fast-import: use callbacks to parse some options A previous commit started using the parse-option API to generate proper `git fast-import -h` and `git fast-import --help-all` output. Let's prepare for when we can use that API to also parse the options by using OPT_CALLBACK for some options that require special processing of their arguments. A following commit will actually parse the options using these callbacks. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano --- builtin/fast-import.c | 208 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 168 insertions(+), 40 deletions(-) diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 879c2860439ea3..40cc9c4a23fe7f 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -4008,6 +4008,126 @@ static void parse_argv(struct fast_import_state *state) build_mark_map(&sub_marks_from, &sub_marks_to); } +static int option_parse_date_format(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_date_format(arg); + return 0; +} + +static int option_parse_export_pack_edges(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_export_pack_edges(opt->value, arg); + return 0; +} + +static int option_parse_max_pack_size(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_max_pack_size(arg); + return 0; +} + +static int option_parse_big_file_threshold(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_big_file_threshold(arg); + return 0; +} + +static int option_parse_signed_commits(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_signed_commits(arg); + return 0; +} + +static int option_parse_signed_tags(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_signed_tags(arg); + return 0; +} + +static int option_parse_rewrite_submodules_from(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_rewrite_submodules(opt->value, arg, &sub_marks_from); + return 0; +} + +static int option_parse_rewrite_submodules_to(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_rewrite_submodules(opt->value, arg, &sub_marks_to); + return 0; +} + +static int option_parse_cat_blob_fd(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_cat_blob_fd(opt->value, arg); + return 0; +} + +static int option_parse_import_marks(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_import_marks(opt->value, arg, 0, 0); + return 0; +} + +static int option_parse_import_marks_if_exists(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_import_marks(opt->value, arg, 0, 1); + return 0; +} + +static int option_parse_export_marks(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_export_marks(opt->value, arg); + return 0; +} + +static int option_parse_depth(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_depth(arg); + return 0; +} + +static int option_parse_active_branches(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_active_branches(arg); + return 0; +} + +static int option_parse_quiet(const struct option *opt UNUSED, + const char *arg UNUSED, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_quiet(); + return 0; +} + int cmd_fast_import(int argc, const char **argv, const char *prefix, @@ -4015,10 +4135,6 @@ int cmd_fast_import(int argc, { struct fast_import_state state; - unsigned long pack_size_limit, big_file_threshold; - char *edges, *signed_commits, *signed_tags, *date_format; - char *import_marks_if_exists, *submodules_from, *submodules_to; - /* * NEEDSWORK: For now this is used only to render * `-h`/`--help-all` usage messages. The actual parsing is @@ -4026,58 +4142,70 @@ int cmd_fast_import(int argc, */ struct option fast_import_options[] = { OPT_GROUP(N_("Common")), - OPT_STRING_F(0, "date-format", &date_format, N_("fmt"), - N_("format of the commit/tag dates"), PARSE_OPT_NONEG), + OPT_CALLBACK_F(0, "date-format", NULL, N_("fmt"), + N_("format of the commit/tag dates"), + PARSE_OPT_NONEG, option_parse_date_format), OPT_BOOL_F(0, "stats", &show_stats, N_("display some basic statistics (objects, packfiles and memory)"), PARSE_OPT_NONEG), - OPT_BOOL_F(0, "quiet", &quiet, - N_("disable the output shown by --stats"), PARSE_OPT_NONEG), + OPT_CALLBACK_F(0, "quiet", NULL, NULL, + N_("disable the output shown by --stats"), + PARSE_OPT_NOARG | PARSE_OPT_NONEG, + option_parse_quiet), OPT_BOOL_F(0, "force", &force_update, N_("force updating modified existing branches"), PARSE_OPT_NONEG), OPT_BOOL_F(0, "done", &require_explicit_termination, N_("require a terminating 'done' command"), PARSE_OPT_NONEG), - OPT_UNSIGNED(0, "max-pack-size", &pack_size_limit, - N_("maximum size of each output pack file")), - OPT_UNSIGNED(0, "big-file-threshold", &big_file_threshold, - N_("maximum size of a blob that will be deltified")), - OPT_UNSIGNED(0, "depth", &max_depth, - N_("maximum delta depth")), - OPT_UNSIGNED(0, "active-branches", &max_active_branches, - N_("maximum number of branches to maintain active")), + OPT_CALLBACK_F(0, "max-pack-size", NULL, N_("n"), + N_("maximum size of each output pack file"), + PARSE_OPT_NONEG, option_parse_max_pack_size), + OPT_CALLBACK_F(0, "big-file-threshold", NULL, N_("n"), + N_("maximum size of a blob that will be deltified"), + PARSE_OPT_NONEG, option_parse_big_file_threshold), + OPT_CALLBACK_F(0, "depth", NULL, N_("n"), + N_("maximum delta depth"), + PARSE_OPT_NONEG, option_parse_depth), + OPT_CALLBACK_F(0, "active-branches", NULL, N_("n"), + N_("maximum number of branches to maintain active"), + PARSE_OPT_NONEG, option_parse_active_branches), OPT_GROUP(N_("Marks")), - OPT_STRING_F(0, "import-marks", &import_marks_file, N_("file"), - N_("import marks from "), PARSE_OPT_NONEG), - OPT_STRING_F(0, "import-marks-if-exists", &import_marks_if_exists, N_("file"), - N_("import marks from if it exists"), PARSE_OPT_NONEG), - OPT_STRING_F(0, "export-marks", &export_marks_file, N_("file"), - N_("dump marks to "), PARSE_OPT_NONEG), + OPT_CALLBACK_F(0, "import-marks", &state, N_("file"), + N_("import marks from "), + PARSE_OPT_NONEG, option_parse_import_marks), + OPT_CALLBACK_F(0, "import-marks-if-exists", &state, N_("file"), + N_("import marks from if it exists"), + PARSE_OPT_NONEG, option_parse_import_marks_if_exists), + OPT_CALLBACK_F(0, "export-marks", &state, N_("file"), + N_("dump marks to "), + PARSE_OPT_NONEG, option_parse_export_marks), OPT_BOOL(0, "relative-marks", &relative_marks_paths, N_("are --(import|export)-marks= paths relative to '.git/info/fast-import'?")), OPT_GROUP(N_("Submodule rewrite")), - OPT_STRING_F(0, "rewrite-submodules-from", &submodules_from, N_("name:filename"), - N_("rewrite object IDs for submodule from "), - PARSE_OPT_NONEG), - OPT_STRING_F(0, "rewrite-submodules-to", &submodules_to, N_("name:filename"), - N_("rewrite object IDs for submodule to "), - PARSE_OPT_NONEG), + OPT_CALLBACK_F(0, "rewrite-submodules-from", &state, N_("name:filename"), + N_("rewrite object IDs for submodule from "), + PARSE_OPT_NONEG, option_parse_rewrite_submodules_from), + OPT_CALLBACK_F(0, "rewrite-submodules-to", &state, N_("name:filename"), + N_("rewrite object IDs for submodule to "), + PARSE_OPT_NONEG, option_parse_rewrite_submodules_to), OPT_GROUP(N_("Signing")), - OPT_STRING_F(0, "signed-commits", &signed_commits, N_("mode"), - N_("how to handle signed commits"), - PARSE_OPT_NONEG), - OPT_STRING_F(0, "signed-tags", &signed_tags, N_("mode"), - N_("how to handle signed tags"), - PARSE_OPT_NONEG), + OPT_CALLBACK_F(0, "signed-commits", NULL, N_("mode"), + N_("how to handle signed commits"), + PARSE_OPT_NONEG, option_parse_signed_commits), + OPT_CALLBACK_F(0, "signed-tags", NULL, N_("mode"), + N_("how to handle signed tags"), + PARSE_OPT_NONEG, option_parse_signed_tags), OPT_HIDDEN_GROUP(N_("Advanced")), OPT_BOOL_F(0, "allow-unsafe-features", &state.allow_unsafe_features, N_("allow unsafe mark commands from the stream"), PARSE_OPT_HIDDEN | PARSE_OPT_NONEG), - OPT_STRING_F(0, "export-pack-edges", &edges, N_("file"), - N_("dump edge commits to "), - PARSE_OPT_HIDDEN | PARSE_OPT_NONEG), - OPT_INTEGER_F(0, "cat-blob-fd", &cat_blob_fd, - N_("write some responses to instead of stdout"), - PARSE_OPT_HIDDEN | PARSE_OPT_NONEG), + OPT_CALLBACK_F(0, "export-pack-edges", &state, N_("file"), + N_("dump edge commits to "), + PARSE_OPT_HIDDEN | PARSE_OPT_NONEG, + option_parse_export_pack_edges), + OPT_CALLBACK_F(0, "cat-blob-fd", &state, N_("fd"), + N_("write some responses to instead of stdout"), + PARSE_OPT_HIDDEN | PARSE_OPT_NONEG, + option_parse_cat_blob_fd), OPT_END() }; From 863937696ef2bd8faaf4811aaea8cf5301bac78b Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Tue, 11 Aug 2026 10:33:13 +0200 Subject: [PATCH 139/259] fast-import: use parse_options() for command line options Previous commits have started to use the parse-options API to display output from `git fast-import -h` and `git fast-import --help-all` and to prepare for parsing the command line options using this API. Let's now actually use the API to parse command line options. This brings a number of changes that are mostly beneficial: - The `--alias`, `--get-mark`, `--cat-blob`, `--ls` and `--notes` options are no longer accepted on the command line. They were previously accepted as no-ops because parse_argv() fell through to parse_one_feature(). They are not documented in the OPTIONS section and are only meaningful as in-stream feature assertions, so accepting them on the command line was an accident of code sharing dating back to 9c8398f0c9 (fast-import: add option command, 2009-12-04). - Abbreviated options like `--dep=5` now work since parse_options() allows unambiguous prefixes. - As `--cat-blob` is an abbreviation of `--cat-blob-fd`, using the former on the command line will fail with "option `cat-blob-fd' requires a value" unlike the other four options that are not accepted anymore on the command line (see above). - Value-taking options now also accept the space-separated `--opt value` form, like `--depth 5`, in addition to the `--opt=value` form. - A bare or trailing `--` is now accepted and the stream is read normally, while it used to be a usage error. - The error messages for some options might differ a bit. - The code is shorter and more standard. Note that parse_one_feature() is now always called with its `from_stream` argument set to 1, but the code simplifications that can be made are left for a following clean-up commit. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano --- Documentation/git-fast-import.adoc | 7 +++++ builtin/fast-import.c | 43 ++++++++++-------------------- t/t9300-fast-import.sh | 7 +++++ 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/Documentation/git-fast-import.adoc b/Documentation/git-fast-import.adoc index 7c5900e048cefb..fd165e11d2d259 100644 --- a/Documentation/git-fast-import.adoc +++ b/Documentation/git-fast-import.adoc @@ -65,6 +65,13 @@ Only enable this option if you trust the program generating the fast-import stream! This option is enabled automatically for remote-helpers that use the `import` capability, as they are already trusted to run their own code. ++ +Note that this option has to be spelled in full, and has to appear +before any option whose value is separated from it by a space, for +the unsafe `feature` commands in the stream to be allowed. So +`--allow-unsafe` or `--depth 5 --allow-unsafe-features` still refuse +them, while `--allow-unsafe-features --depth 5` and +`--depth=5 --allow-unsafe-features` allow them. `--signed-tags=`:: Specify how to handle signed tags. Behaves in the same way as diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 40cc9c4a23fe7f..dd873ec4336b90 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -3975,31 +3975,11 @@ static const char *const fast_import_usage[] = { static void parse_argv(struct fast_import_state *state) { - unsigned int i; - - for (i = 1; i < state->argc; i++) { - const char *a = state->argv[i]; - - if (*a != '-' || !strcmp(a, "--")) - break; - - if (!skip_prefix(a, "--", &a)) - die(_("unknown option %s"), a); - - if (parse_one_option(state, a)) - continue; - - if (parse_one_feature(state, a, 0)) - continue; - - if (skip_prefix(a, "cat-blob-fd=", &a)) { - option_cat_blob_fd(state, a); - continue; - } + int argc = parse_options(state->argc, state->argv, state->prefix, + state->option, fast_import_usage, + PARSE_OPT_KEEP_ARGV0); - die(_("unknown option --%s"), a); - } - if (i != state->argc) + if (argc > 1) usage_with_options(fast_import_usage, state->option); state->seen_data_command = 1; @@ -4135,11 +4115,6 @@ int cmd_fast_import(int argc, { struct fast_import_state state; - /* - * NEEDSWORK: For now this is used only to render - * `-h`/`--help-all` usage messages. The actual parsing is - * done by parse_one_option()/parse_one_feature(). - */ struct option fast_import_options[] = { OPT_GROUP(N_("Common")), OPT_CALLBACK_F(0, "date-format", NULL, N_("fmt"), @@ -4230,6 +4205,16 @@ int cmd_fast_import(int argc, * "feature" lines at the start of the stream (which allows the command * line to override stream data). But we must do an early parse of any * command-line options that impact how we interpret the feature lines. + * + * NEEDSWORK: This scan only matches the exact "--allow-unsafe-features" + * spelling and stops at the first argument that doesn't start with a + * dash. As parse_options() below also accepts unambiguous abbreviations + * and values separated by a space from their option, the two disagree + * for command lines like "--allow-unsafe" or "--depth 5 + * --allow-unsafe-features": parse_options() accepts the option, but + * this scan doesn't see it, so unsafe features from the stream are + * still refused. This errs on the safe side, but should be fixed by + * teaching this scan about the options that take a value. */ for (int i = 1; i < argc; i++) { const char *arg = argv[i]; diff --git a/t/t9300-fast-import.sh b/t/t9300-fast-import.sh index fe6c2617acb2fe..d9de2ef0d88bc8 100755 --- a/t/t9300-fast-import.sh +++ b/t/t9300-fast-import.sh @@ -2827,6 +2827,13 @@ test_expect_success 'R: unknown commandline options are rejected' '\ test_must_fail git fast-import --non-existing-option < /dev/null ' +test_expect_success 'R: feature-only names are rejected on the command line' ' + for opt in --alias --get-mark --ls --notes + do + test_must_fail git fast-import "$opt" Date: Tue, 11 Aug 2026 10:33:14 +0200 Subject: [PATCH 140/259] fast-import: remove useless from_stream argument Now that a previous commit has removed a call to parse_one_feature() from parse_argv(), the former is always called with its `from_stream` argument set to 1. Let's take advantage of that to simplify and cleanup the code a bit. Signed-off-by: Christian Couder Signed-off-by: Junio C Hamano --- builtin/fast-import.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/builtin/fast-import.c b/builtin/fast-import.c index dd873ec4336b90..4e3c9601505da8 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -3867,30 +3867,28 @@ static int parse_one_option(struct fast_import_state *state, const char *option) } static void check_unsafe_feature(struct fast_import_state *state, - const char *feature, - int from_stream) + const char *feature) { - if (from_stream && !state->allow_unsafe_features) + if (!state->allow_unsafe_features) die(_("feature '%s' forbidden in input without --allow-unsafe-features"), feature); } static int parse_one_feature(struct fast_import_state *state, - const char *feature, - int from_stream) + const char *feature) { const char *arg; if (skip_prefix(feature, "date-format=", &arg)) { option_date_format(arg); } else if (skip_prefix(feature, "import-marks=", &arg)) { - check_unsafe_feature(state, "import-marks", from_stream); - option_import_marks(state, arg, from_stream, 0); + check_unsafe_feature(state, "import-marks"); + option_import_marks(state, arg, 1, 0); } else if (skip_prefix(feature, "import-marks-if-exists=", &arg)) { - check_unsafe_feature(state, "import-marks-if-exists", from_stream); - option_import_marks(state, arg, from_stream, 1); + check_unsafe_feature(state, "import-marks-if-exists"); + option_import_marks(state, arg, 1, 1); } else if (skip_prefix(feature, "export-marks=", &arg)) { - check_unsafe_feature(state, feature, from_stream); + check_unsafe_feature(state, feature); option_export_marks(state, arg); } else if (!strcmp(feature, "alias")) { ; /* Don't die - this feature is supported */ @@ -3924,7 +3922,7 @@ static void parse_feature(struct fast_import_state *state, const char *feature) if (state->seen_data_command) die(_("got feature command '%s' after data command"), feature); - if (parse_one_feature(state, feature, 1)) + if (parse_one_feature(state, feature)) return; die(_("this version of fast-import does not support feature %s."), feature); From cd2ab0e12896ef505f4be8822788912fe95394eb Mon Sep 17 00:00:00 2001 From: Kristofer Karlsson Date: Tue, 11 Aug 2026 09:28:43 +0000 Subject: [PATCH 141/259] Documentation/technical: add paint-down-to-common doc Add a technical document describing the paint_down_to_common() algorithm used for merge-base computation, covering the paint walk, generation number regions, and termination conditions. Signed-off-by: Kristofer Karlsson Signed-off-by: Junio C Hamano --- Documentation/Makefile | 1 + Documentation/technical/meson.build | 1 + .../technical/paint-down-to-common.adoc | 174 ++++++++++++++++++ commit-reach.c | 6 +- 4 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 Documentation/technical/paint-down-to-common.adoc diff --git a/Documentation/Makefile b/Documentation/Makefile index 2699f0b24af192..f8dea4b3953250 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -129,6 +129,7 @@ TECH_DOCS += technical/long-running-process-protocol TECH_DOCS += technical/multi-pack-index TECH_DOCS += technical/packfile-uri TECH_DOCS += technical/pack-heuristics +TECH_DOCS += technical/paint-down-to-common TECH_DOCS += technical/parallel-checkout TECH_DOCS += technical/partial-clone TECH_DOCS += technical/platform-support diff --git a/Documentation/technical/meson.build b/Documentation/technical/meson.build index ec07088c57617f..9ce11d5e484d9c 100644 --- a/Documentation/technical/meson.build +++ b/Documentation/technical/meson.build @@ -18,6 +18,7 @@ articles = [ 'multi-pack-index.adoc', 'packfile-uri.adoc', 'pack-heuristics.adoc', + 'paint-down-to-common.adoc', 'parallel-checkout.adoc', 'partial-clone.adoc', 'platform-support.adoc', diff --git a/Documentation/technical/paint-down-to-common.adoc b/Documentation/technical/paint-down-to-common.adoc new file mode 100644 index 00000000000000..4bd3c2adb56a64 --- /dev/null +++ b/Documentation/technical/paint-down-to-common.adoc @@ -0,0 +1,174 @@ +Merge-Base Computation and paint_down_to_common() +================================================== + +The function `paint_down_to_common()` in `commit-reach.c` computes merge +bases by walking the commit graph backwards from two sets of tips and +finding where their ancestry meets. + +Use cases +--------- + +Computing merge bases is used in two different ways: + + 1. *Finding all merge bases* (`merge-base --all`, `merge-tree`, + `merge`, `rebase`). A merge base is a common ancestor that is + not itself an ancestor of another common ancestor. + + 2. *Ancestry checks* (`in_merge_bases`, used by `merge-base + --is-ancestor`, `branch -d`, `fetch`). These ask: "is commit A + an ancestor of commit B?" If a common ancestor equals one of the + inputs, that input is necessarily the only merge base -- no other + common ancestor can be both as recent and not an ancestor of it. + +Both use cases share the same algorithm and implementation. + +Algorithm +--------- + +Given a commit `one` and a set of commits `twos[]`, the walk paints +commits with two colors: + + - PARENT1: reachable from `one` + - PARENT2: reachable from any commit in `twos[]` + +The walk uses a priority queue ordered by generation number +(highest first), breaking ties by commit date. Each step dequeues +the highest-priority commit and propagates its paint flags to its +parents, enqueuing any parent that gained new flags. When a +commit receives both PARENT1 and PARENT2, it is a merge-base +candidate. A candidate gains the STALE flag so its ancestors +propagate staleness -- any deeper common ancestor is necessarily +redundant. + +[[generation-regions]] +Topologically ordered and unordered generation regions +------------------------------------------------------ + +Commits fall into two regions based on whether their generation +numbers provide a topological ordering guarantee: + +.... + +------------------------------------------+ + | Unordered region | + | generation = INFINITY or V1_MAX | + | queue order: heuristic (commit date) | + +------------------------------------------+ + | + v + +------------------------------------------+ + | Ordered region | + | generation = finite, unsaturated | + | queue order: topological | + +------------------------------------------+ +.... + +In the ordered region, a child's generation is strictly greater +than its parent's. Same-generation commits are necessarily +independent, so the queue always processes children before +their parents. + +In the unordered region, parent-child pairs can share the same +generation number, so topological order is not guaranteed. The +queue uses commit-date as a heuristic, which typically produces +a reasonable traversal order but may process a parent before +its child. + +Commits not in the commit-graph have generation INFINITY; v1 +commit-graphs saturate at V1_MAX. Both place commits in the +unordered region. Any optimization that depends on generation +ordering must account for this saturation boundary. + +With generation ordering, values in the unordered region exceed +those in the ordered region. The walk may therefore transition +from the unordered region into the ordered region, but never in +the reverse direction. Without a commit-graph, every commit has INFINITY +and the walk operates entirely in the unordered region. + +In the ordered region, paint on a dequeued commit is final -- +no future step can add flags to it. In the unordered region, +a dequeued commit may later gain additional paint. Paint flags +are only added, never removed, bounding the number of +re-enqueues per commit. + +Termination +----------- + +The walk uses a `nonstale_queue` wrapper around `prio_queue` that +tracks `max_nonstale`: the lowest-priority non-stale commit enqueued +so far. Once that commit is dequeued, every remaining entry is known +to be STALE and the loop terminates. Specifically, the main loop +ends when one of the following conditions holds: + + 1. The queue is empty. + 2. `max_nonstale` has been dequeued, meaning the queue only contains + STALE entries. + 3. Generation cutoff: the dequeued commit's generation is below + a caller-supplied `min_generation` threshold. + 4. Single result: the caller only needs one merge base, one has + been found, and the walk has entered the ordered region. + +Stale entry condition +~~~~~~~~~~~~~~~~~~~~~ +Once all queued entries are stale, no new merge-base candidates can +be discovered -- that requires at least one non-stale commit from +each side meeting. Continuing the walk could still invalidate +existing candidates by proving one is an ancestor of another, but +`remove_redundant()` handles that as a post-processing step, so it +is safe to exit early. + +Generation cutoff +~~~~~~~~~~~~~~~~~ +Some callers (notably `remove_redundant()`) supply a `min_generation` +threshold equal to the minimum generation of the input commits. +These callers only need to determine reachability among the inputs, +not find deep merge bases, so the walk can safely terminate when it +dequeues a commit below this threshold. + +Single result +~~~~~~~~~~~~~ +When only one merge base is needed and the walk is in the +ordered region with generation ordering, the first candidate +found is necessarily the highest-generation common ancestor. +No remaining commit in the queue can be a descendant of this +candidate (generation ordering guarantees children are visited +first), so it cannot be redundant and the walk can stop +immediately. + +This optimization is NOT safe when the date-ordering fallback is +active, because commit-date order can visit a deeper ancestor +before a shallower one -- see <>. + +[[date-ordering-fallback]] +Date-ordering fallback +---------------------- + +When the commit-graph has generation numbers v1 and no +generation floor is specified, topological ordering +(via generation numbers) is disabled. Topological levels are +correct but unbalanced -- ordering by such generation numbers +can sometimes cause the walk to detour too far before finding +merge bases. Commit-date ordering typically reaches them in +fewer steps -- see this change for more details: + + 091f4cf3 (commit: don't use generation numbers if not needed, + 2018-08-30) + +With generation number v2 (corrected commit dates) we have the best +of both worlds and do not need this fallback. + +For v1, `paint_down_to_common()` falls back to pure commit-date +ordering via `compare_commits_by_commit_date`. Because commit +dates are not monotonic (clock skew, rebases, etc.), the queue +may visit commits out of topological order. + +This disables the optimization that depends on generation ordering: + + - *Single result*: the first merge-base candidate found may not + be the shallowest, because a deeper ancestor with a higher + commit date can be dequeued first. + +Related documentation +--------------------- + + - `Documentation/technical/commit-graph.adoc` -- generation numbers + and the reachability closure property. diff --git a/commit-reach.c b/commit-reach.c index 708798a39b2d8e..bbf8c3eff068f6 100644 --- a/commit-reach.c +++ b/commit-reach.c @@ -96,7 +96,11 @@ static struct commit *nonstale_queue_get_dedup(struct nonstale_queue *queue) return commit; } -/* all input commits in one and twos[] must have been parsed! */ +/* + * See Documentation/technical/paint-down-to-common.adoc + * + * All input commits in one and twos[] must have been parsed! + */ static int paint_down_to_common(struct repository *r, struct commit *one, int n, struct commit **twos, From fe4877bc17b1dd5b25b9444eadee20ec5fa8bf83 Mon Sep 17 00:00:00 2001 From: Kristofer Karlsson Date: Tue, 11 Aug 2026 09:28:44 +0000 Subject: [PATCH 142/259] test-lib-functions: improve diagnostic output for trace2 data assertions test_trace2_data is a bare grep that silently exits on failure. Add a more informative variant that verifies the event appears exactly once and reports what went wrong: key not found, multiple entries, or value mismatch. Diagnostics go to FD 4 like test_grep. Before (value mismatch): $ test_trace2_data status count/changed 999 Signed-off-by: Junio C Hamano --- t/test-lib-functions.sh | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 809c6621241944..8c6d327b03cbe8 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -1996,6 +1996,41 @@ test_trace2_data () { grep -e '"category":"'"$1"'","key":"'"$2"'","value":"'"$3"'"' } +# Check that the given trace2 data event has the expected value and +# appears exactly once. Produces a diagnostic on failure. +# +# test_trace2_data_singular [