diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 205325eb33b064..4fff34487453d9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,18 +5,20 @@ on: [push, pull_request] env: DEVELOPER: 1 -# If more than one workflow run is triggered for the very same commit hash -# (which happens when multiple branches pointing to the same commit), only -# the first one is allowed to run, the second will be kept in the "queued" -# state. This allows a successful completion of the first run to be reused -# in the second run via the `skip-if-redundant` logic in the `config` job. +# For pull requests, only the latest workflow run is allowed to proceed. +# Older runs are canceled when a new revision is pushed. # -# The only caveat is that if a workflow run is triggered for the same commit -# hash that another run is already being held, that latter run will be -# canceled. For more details about the `concurrency` attribute, see: +# For pushes, if more than one workflow run is triggered for the very same +# commit hash (which happens when multiple branches point to the same commit), +# only the first one is allowed to run. This allows a successful completion of +# the first run to be reused in the second run via the `skip-if-redundant` +# logic in the `config` job. +# +# For more details about the `concurrency` attribute, see: # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency concurrency: - group: ${{ github.sha }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: ci-config: diff --git a/.gitignore b/.gitignore index 4da58c6754899e..4173111c01b2b6 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ /git-diagnose /git-diff /git-diff-files +/git-diff-hunks /git-diff-index /git-diff-pairs /git-diff-tree diff --git a/Documentation/BreakingChanges.adoc b/Documentation/BreakingChanges.adoc index 73bb939359c72e..dbc46d14e33dce 100644 --- a/Documentation/BreakingChanges.adoc +++ b/Documentation/BreakingChanges.adoc @@ -171,6 +171,11 @@ JGit, libgit2 and Gitoxide need to support it. matches the default branch name used in new repositories by many of the big Git forges. +* Git will accept hex object IDs only in lowercase. The fact that Git has + historically allowed uppercase characters in hex object IDs has been the + source of a variety of bugs and security problems in software using Git. We + don't expect most users to notice any change. + * Git will require Rust as a mandatory part of the build process. While Git already started to adopt Rust in Git 2.49, all parts written in Rust are optional for the time being. This includes: diff --git a/Documentation/Makefile b/Documentation/Makefile index 2699f0b24af192..cf9ba445b2c44a 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -33,6 +33,7 @@ MAN5_TXT += gitattributes.adoc MAN5_TXT += gitformat-bundle.adoc MAN5_TXT += gitformat-chunk.adoc MAN5_TXT += gitformat-commit-graph.adoc +MAN5_TXT += gitformat-diff-hunks.adoc MAN5_TXT += gitformat-index.adoc MAN5_TXT += gitformat-loose.adoc MAN5_TXT += gitformat-pack.adoc @@ -129,6 +130,7 @@ TECH_DOCS += technical/long-running-process-protocol TECH_DOCS += technical/multi-pack-index TECH_DOCS += technical/packfile-uri TECH_DOCS += technical/pack-heuristics +TECH_DOCS += technical/paint-down-to-common TECH_DOCS += technical/parallel-checkout TECH_DOCS += technical/partial-clone TECH_DOCS += technical/platform-support diff --git a/Documentation/SubmittingPatches b/Documentation/SubmittingPatches index c269e474e3f2bf..c60855f7069492 100644 --- a/Documentation/SubmittingPatches +++ b/Documentation/SubmittingPatches @@ -117,7 +117,7 @@ of review. can still continue to further improve them by adding more patches on top, but by the time a topic gets merged to 'next', it is expected that everybody agrees that the scope and the basic direction of the - topic are appropriate, so such an incremental updates are limited to + topic are appropriate, so such incremental updates are limited to small corrections and polishing. After a topic cooks for some time (like 7 calendar days) in 'next' without needing further tweaks on top, it gets merged to the 'master' branch and waits to become part diff --git a/Documentation/config.adoc b/Documentation/config.adoc index f67dcd2f8e05dd..ceb13ade6e4fa2 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -472,6 +472,8 @@ include::config/credential.adoc[] include::config/diff.adoc[] +include::config/diff-hunks.adoc[] + include::config/difftool.adoc[] include::config/extensions.adoc[] diff --git a/Documentation/config/advice.adoc b/Documentation/config/advice.adoc index 81f80a92745123..3d91c90eda72f9 100644 --- a/Documentation/config/advice.adoc +++ b/Documentation/config/advice.adoc @@ -59,6 +59,10 @@ all advice messages. forceDeleteBranch:: Shown when the user tries to delete a not fully merged branch without the force option set. + historyUpdateRefs:: + Shown when `git history squash` refuses because a local branch + cannot be rewritten as a descendant of the squashed commit, to + tell the user about `--update-refs=head`. ignoredHook:: Shown when a hook is ignored because the hook is not set as executable. diff --git a/Documentation/config/branch.adoc b/Documentation/config/branch.adoc index 5a85fde8de9286..c2acd89c9936cb 100644 --- a/Documentation/config/branch.adoc +++ b/Documentation/config/branch.adoc @@ -103,3 +103,10 @@ for details). `git branch --edit-description`. Branch description is automatically added to the `format-patch` cover letter or `request-pull` summary. + +`branch..deleteMerged`:: + If set to `false`, branch __ is exempt from + `git branch --delete-merged`. Useful for a topic branch you + intend to develop further after an initial round has been + merged upstream. Defaults to true. Explicit deletion via + `git branch -d` is unaffected. diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 340329edc38143..b0a2dc4fc956e9 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -118,6 +118,12 @@ core.trustctime:: crawlers and some backup systems). See linkgit:git-update-index[1]. True by default. +core.useNanosec:: + If true, use nanosecond precision for ctime and mtime + comparisions between the index and the working tree (if Git + was compiled to store it). + See link:technical/racy-git.html[Racy Git]. False by default. + core.splitIndex:: If true, the split-index feature of the index will be used. See linkgit:git-update-index[1]. False by default. @@ -678,12 +684,13 @@ but risks losing recent work in the event of an unclean system shutdown. * `pack` hardens objects added to the repo in packfile form. * `pack-metadata` hardens packfile bitmaps and indexes. * `commit-graph` hardens the commit-graph file. +* `diff-hunks` hardens the diff-hunks store. * `index` hardens the index when it is modified. * `objects` is an aggregate option that is equivalent to `loose-object,pack`. * `reference` hardens references modified in the repo. * `derived-metadata` is an aggregate option that is equivalent to - `pack-metadata,commit-graph`. + `pack-metadata,commit-graph,diff-hunks`. * `committed` is an aggregate option that is currently equivalent to `objects`. This mode sacrifices some performance to ensure that work that is committed to the repository with `git commit` or similar commands @@ -758,6 +765,13 @@ core.commitGraph:: to parse the graph structure of commits. Defaults to true. See linkgit:git-commit-graph[1] for more information. +core.diffHunks:: + If true, then Git will consult the diff-hunks store (if it + exists) to skip recomputing diff hunk coordinates in commands + such as `git log --stat` and linkgit:git-blame[1]. This controls + only reading; writing the store is controlled by `diffHunks.write`. + See linkgit:git-diff-hunks[1] for more information. Defaults to true. + core.useReplaceRefs:: If set to `false`, behave as if the `--no-replace-objects` option was given on the command line. See linkgit:git[1] and diff --git a/Documentation/config/diff-hunks.adoc b/Documentation/config/diff-hunks.adoc new file mode 100644 index 00000000000000..ad76d1c6a9a2e4 --- /dev/null +++ b/Documentation/config/diff-hunks.adoc @@ -0,0 +1,8 @@ +diffHunks.write:: + If true, diff-producing commands (`git diff`, `git log`, + `git show`, or `git diff-tree` with a `--stat`, `--numstat`, or + `--shortstat` format) write the hunks + they compute to the diff-hunks store, filling it as a side effect. + The `GIT_DIFF_HUNKS_WRITE` environment variable overrides this for + a single invocation. Reading the store is controlled separately by + `core.diffHunks`. See linkgit:git-diff-hunks[1]. Defaults to false. diff --git a/Documentation/config/diff.adoc b/Documentation/config/diff.adoc index 1135a62a0ad3de..349bdbe4927b6e 100644 --- a/Documentation/config/diff.adoc +++ b/Documentation/config/diff.adoc @@ -218,6 +218,12 @@ endif::git-diff[] Set this option to `true` to make the diff driver cache the text conversion outputs. See linkgit:gitattributes[5] for details. +`diff..process`:: + The command to run as a long-running process that answers + which line ranges changed between two blobs. See + linkgit:gitattributes[5] for the protocol and when it is + consulted. + `diff.indentHeuristic`:: Set this option to `false` to disable the default heuristics that shift diff hunk boundaries to make patches easier to read. diff --git a/Documentation/config/fetch.adoc b/Documentation/config/fetch.adoc index 00435e9a16d9f9..20bf199b447404 100644 --- a/Documentation/config/fetch.adoc +++ b/Documentation/config/fetch.adoc @@ -10,6 +10,20 @@ reference. Defaults to `on-demand`, or to the value of `submodule.recurse` if set. +`fetch.submoduleErrors`:: + Controls how errors from submodule fetches are handled when + `--recurse-submodules` is in effect. When set to `fail` (the default), + any submodule fetch error causes the overall `git fetch` or `git pull` + to exit with a non-zero status. When set to `warn`, submodule fetch + errors are reported to standard error but do not affect the exit + status of the command. This is useful when working in repositories + where some branches reference submodule commits that are not yet + available on the submodule remote, but those commits are not needed + for the currently checked-out branch. ++ +The value of this option can be overridden by the `--submodule-errors` +option of linkgit:git-fetch[1]. + `fetch.fsckObjects`:: If it is set to true, git-fetch-pack will check all fetched objects. See `transfer.fsckObjects` for what's diff --git a/Documentation/config/http.adoc b/Documentation/config/http.adoc index 792a71b41350d4..40b849bf7fddfc 100644 --- a/Documentation/config/http.adoc +++ b/Documentation/config/http.adoc @@ -196,6 +196,23 @@ http.sslVerify:: over HTTPS. Defaults to true. Can be overridden by the `GIT_SSL_NO_VERIFY` environment variable. +http.sslVerifyStatus:: + Whether to check the revocation status of the server + certificate using the stapled OCSP response supplied during + the TLS handshake ("OCSP stapling"). Defaults to false. ++ +This is fail-closed: if the server staples no response, verification +fails. Set it per remote, e.g. +`http.https://example.com/.sslVerifyStatus`, rather than globally. ++ +What it changes depends on the TLS backend libcurl was built against. +An OpenSSL-linked build ignores a stapled response unless this is set. +A GnuTLS-linked build consults the staple during ordinary certificate +verification, so it already rejects a revoked certificate under +`http.sslVerify` alone, and setting this to `false` does not disable +that. Where a backend cannot check the staple at all, git fails with an +error rather than continuing unchecked. + http.sslCert:: File containing the SSL certificate when fetching or pushing over HTTPS. Can be overridden by the `GIT_SSL_CERT` environment diff --git a/Documentation/config/rebase.adoc b/Documentation/config/rebase.adoc index c6187ab28b2fb7..321ab8b529ca3f 100644 --- a/Documentation/config/rebase.adoc +++ b/Documentation/config/rebase.adoc @@ -62,6 +62,12 @@ instead of: + Defaults to false. +rebase.noEdit:: + When set to true, `git rebase --continue` uses the commit message + without launching $EDITOR, as if `--no-edit` were given. The + `--edit` option to `git rebase --continue` overrides this setting. + Defaults to false. + rebase.rescheduleFailedExec:: Automatically reschedule `exec` commands that failed. This only makes sense in interactive mode (or when an `--exec` option was provided). diff --git a/Documentation/config/uploadpack.adoc b/Documentation/config/uploadpack.adoc index 0e1dda944a5490..e960879c161f30 100644 --- a/Documentation/config/uploadpack.adoc +++ b/Documentation/config/uploadpack.adoc @@ -86,3 +86,45 @@ uploadpack.allowRefInWant:: is intended for the benefit of load-balanced servers which may not have the same view of what OIDs their refs point to due to replication delay. + +uploadpack.lazyFetchTrusted:: + These config entries specify repositories that `upload-pack` is + allowed to lazily fetch missing objects for. By default, + `upload-pack` refuses to lazily fetch (see the description of the + `GIT_NO_LAZY_FETCH` environment variable in + linkgit:git-upload-pack[1]), because doing so would run `git fetch`, + which may execute arbitrary commands specified in the configuration + and hooks of the served repository. Listing a repository here tells + `upload-pack` that it is trusted, so lazy fetching from the promisor + remotes configured in it is allowed. This is equivalent to setting + `GIT_NO_LAZY_FETCH` to `0` for the matching repositories. An + explicitly set `GIT_NO_LAZY_FETCH` takes precedence over this + setting. ++ +Note that this allows lazy fetching from any promisor remote +configured in the served repository, not only from the promisor +remotes that the client accepted using the "promisor-remote" protocol +v2 capability (see linkgit:gitprotocol-v2[5]). The served repository +is trusted as a whole, including its configuration, so the promisor +remotes it configures are trusted too. It is the server operator's +responsibility to make sure that the promisor remotes of a trusted +repository are also trustworthy. ++ +This is a multi-valued setting, i.e. you can add more than one +repository via `git config (--global|--system) --add`. To reset the +list of trusted repositories (e.g. to override any such repositories +specified in the system config), add a `uploadpack.lazyFetchTrusted` +entry with an empty value. ++ +A repository is identified by its worktree, or its git directory for a bare +repository, and the value must be an absolute path. Giving a path with `/*` +appended to it will trust all repositories under the named directory. To trust +all served repositories, set `uploadpack.lazyFetchTrusted` to the string `*`. ++ +The value of this setting is interpolated, i.e. `~/` expands to a +path relative to the home directory and `%(prefix)/` expands to a +path relative to Git's (runtime) prefix. ++ +Note that this configuration variable is only respected when it is specified +in protected configuration (see <>). This prevents untrusted +repositories from tampering with this value. diff --git a/Documentation/diff-algorithm-option.adoc b/Documentation/diff-algorithm-option.adoc index 8e3a0b63d784d8..16e6fc7261d093 100644 --- a/Documentation/diff-algorithm-option.adoc +++ b/Documentation/diff-algorithm-option.adoc @@ -18,3 +18,6 @@ For instance, if you configured the `diff.algorithm` variable to a non-default value and want to use the default one, then you have to use `--diff-algorithm=default` option. ++ +Explicitly choosing a diff algorithm on the command line also +bypasses `diff..process` (see linkgit:gitattributes[5]). diff --git a/Documentation/diff-options.adoc b/Documentation/diff-options.adoc index e605d2867801d0..c68c53e11cf835 100644 --- a/Documentation/diff-options.adoc +++ b/Documentation/diff-options.adoc @@ -833,7 +833,20 @@ endif::git-format-patch[] to use this option with linkgit:git-log[1] and friends. `--no-ext-diff`:: - Disallow external diff drivers. + Disallow external diff drivers and processes, including + `diff..command` and `diff..process` + (see linkgit:gitattributes[5]). + +`--diff-process`:: +`--no-diff-process`:: + Allow (or forbid) consulting a diff process configured with + ++diff.++____++.process++ (see linkgit:gitattributes[5]), + leaving external diff drivers unaffected. `git diff`, `git log`, + `git show`, and `git blame` allow consulting by default; the + plumbing diff commands forbid it unless this option or + `--ext-diff` is given. linkgit:git-format-patch[1] accepts the + option but ignores it: a generated patch is always based on the + builtin diff. `--textconv`:: `--no-textconv`:: diff --git a/Documentation/fetch-options.adoc b/Documentation/fetch-options.adoc index 035f780e583cee..78525f6848056f 100644 --- a/Documentation/fetch-options.adoc +++ b/Documentation/fetch-options.adoc @@ -294,6 +294,14 @@ ifndef::git-pull[] `--no-recurse-submodules`:: Disable recursive fetching of submodules (this has the same effect as using the `--recurse-submodules=no` option). + +`--submodule-errors=(fail|warn)`:: + Control how errors from submodule fetches are handled when + `--recurse-submodules` is in effect. When set to `fail` (the default), + any submodule fetch error causes the overall `git fetch` to exit with a + non-zero status. When set to `warn`, submodule fetch errors are reported + to standard error but do not affect the exit status of the command. Can + also be configured via `fetch.submoduleErrors`. See linkgit:git-config[1]. endif::git-pull[] `--set-upstream`:: diff --git a/Documentation/git-bisect.adoc b/Documentation/git-bisect.adoc index d2115b29905f41..aabddd42ca4d31 100644 --- a/Documentation/git-bisect.adoc +++ b/Documentation/git-bisect.adoc @@ -10,7 +10,7 @@ SYNOPSIS -------- [synopsis] git bisect start [--term-(bad|new)= --term-(good|old)=] - [--no-checkout] [--first-parent] [ [...]] [--] [...] + [--no-checkout] [--first-parent] [--reset-when-found[=]] [ [...]] [--] [...] git bisect (bad|new|) [] git bisect (good|old|) [...] git bisect terms [--term-(good|old) | --term-(bad|new)] @@ -20,7 +20,7 @@ git bisect reset [] git bisect (visualize|view) git bisect replay git bisect log -git bisect run [...] +git bisect run [--reset-when-found[=]] [...] git bisect help DESCRIPTION @@ -385,6 +385,16 @@ ignored. This option is particularly useful in avoiding false positives when a merged branch contained broken or non-buildable commits, but the merge itself was OK. +`--reset-when-found[=]`:: + Once the first bad commit is found, report it and clean up the + bisection state. `` may be `original` to return to the commit + checked out before `git bisect start`, or `found` to leave the first + bad commit checked out. If `` is omitted, it defaults to + `original`. ++ +This option may be given to `git bisect start` or to `git bisect run`. It +cannot be used for a bisection started with `--no-checkout`. + EXAMPLES -------- diff --git a/Documentation/git-branch.adoc b/Documentation/git-branch.adoc index c0afddc424d610..bfdf4593298631 100644 --- a/Documentation/git-branch.adoc +++ b/Documentation/git-branch.adoc @@ -13,6 +13,7 @@ git branch [--color[=] | --no-color] [--show-current] [--column[=] | --no-column] [--sort=] [--merged []] [--no-merged []] [--contains []] [--no-contains []] + [(--forked )...] [--points-at ] [--format=] [(-r|--remotes) | (-a|--all)] [--list] [...] @@ -24,6 +25,7 @@ git branch (-m|-M) [] git branch (-c|-C) [] git branch (-d|-D) [-r] ... git branch --edit-description [] +git branch [--dry-run] (--delete-merged )... [...] DESCRIPTION ----------- @@ -51,7 +53,8 @@ merged into the named commit (i.e. the branches whose tip commits are reachable from the named commit) will be listed. With `--no-merged` only branches not merged into the named commit will be listed. If the __ argument is missing it defaults to `HEAD` (i.e. the tip of the current -branch). +branch). With `--forked`, only branches whose configured upstream matches +the given branch or pattern will be listed. The command's second form creates a new branch head named __ which points to the current `HEAD`, or __ if given. As a @@ -199,6 +202,45 @@ This option is only applicable in non-verbose mode. Print the name of the current branch. In detached `HEAD` state, nothing is printed. +`--delete-merged `:: + Delete local branches whose configured upstream matches + __, but only when their tip is reachable from that + upstream. In other words, the work on the branch has already + landed on the upstream it tracks, so the local copy is no longer + needed. __ may name a ref, a remote (using the branch its + `HEAD` points at), or a shell-style glob. The option can be + repeated to widen the upstream match. + Optional __ arguments limit which local branches + are considered, e.g. `git branch --delete-merged 'origin/*' + 'topic-*'`. ++ +A branch is not deleted when: ++ +-- +* its configured upstream ref no longer exists, +* it is checked out in any worktree, +* pushing it to the remote configured by + `branch..remote` would update its upstream, so it cannot be + distinguished from a branch that just looks fully merged right + after a pull; this is determined by the remote's configured push and + fetch refspecs, +* it is the local upstream of a branch that is not being deleted, or +* `branch..deleteMerged` is set to `false`. +-- ++ +When such a local upstream branch has its own upstream deleted by the +same operation, its upstream configuration is cleared. ++ +A branch whose work has not yet been merged into its upstream is +silently skipped. Delete it with `git branch -D` if you want to +remove it anyway. + +`--dry-run`:: + With `--delete-merged`, print which branches would be + deleted and exit without touching any ref. Useful for + sanity-checking a wide pattern like `'origin/*'` before + committing to the deletion. + `-v`:: `-vv`:: `--verbose`:: @@ -311,6 +353,14 @@ superproject's "origin/main", but tracks the submodule's "origin/main". Only list branches whose tips are not reachable from __ (`HEAD` if not specified). Implies `--list`. +`--forked `:: + Only list branches whose configured upstream matches + __. The argument can be a ref (e.g. `origin/main`, + `master`), a remote name like `origin` for the branch its + `origin/HEAD` points at, or a shell-style glob (e.g. + `'origin/*'`). The option can be repeated to widen the + filter. Implies `--list`. + `--points-at `:: Only list branches of __. diff --git a/Documentation/git-cat-file.adoc b/Documentation/git-cat-file.adoc index ac3b528c6f00f6..514bfc00328caf 100644 --- a/Documentation/git-cat-file.adoc +++ b/Documentation/git-cat-file.adoc @@ -348,15 +348,12 @@ newline. The available atoms are: after that first run of whitespace (i.e., the "rest" of the line) are output in place of the `%(rest)` atom. -The command `remote-object-info` only supports the `%(objectname)` and -`%(objectsize)` placeholders. See `CAVEATS` below for more information. +The command `remote-object-info` only supports the `%(objectname)`, +`%(objectsize)` and `%(objecttype)` placeholders. See `CAVEATS` below for more +information. If no format is specified, the default format is `%(objectname) -%(objecttype) %(objectsize)`, except for `remote-object-info` commands which -use `%(objectname) %(objectsize)` because `%(objecttype)` is not supported yet. - -WARNING: When "%(objecttype)" is supported, the default format WILL be unified, -so DO NOT RELY on the current default format to stay the same!!! +%(objecttype) %(objectsize)`. If `--batch` is specified, or if `--batch-command` is used with the `contents` command, the object information is followed by the object contents (consisting @@ -453,9 +450,9 @@ scripting purposes. CAVEATS ------- -Note that only `%(objectname)` and `%(objectsize)` are currently -supported by the `remote-object-info` command. Using any other placeholder in -the format string will return an empty string in its position. +Note that only `%(objectname)`, `%(objectsize)` and `%(objecttype)` are +currently supported by the `remote-object-info` command. Using any other +placeholder in the format string will return an empty string in its position. Note that the sizes of objects on disk are reported accurately, but care should be taken in drawing conclusions about which refs or objects are diff --git a/Documentation/git-commit.adoc b/Documentation/git-commit.adoc index 8329c1034b9b30..98c50a3be5c81c 100644 --- a/Documentation/git-commit.adoc +++ b/Documentation/git-commit.adoc @@ -102,21 +102,23 @@ include::diff-context-options.adoc[] + The commit created by plain `--fixup=` has a title composed of "fixup!" followed by the title of __, -and is recognized specially by `git rebase --autosquash`. The `-m` -option may be used to supplement the log message of the created -commit, but the additional commentary will be thrown away once the -"fixup!" commit is squashed into __ by +and is recognized specially by `git rebase --autosquash`. The `-m`, +`-F`, `-C`, or `-c` option may be used to supplement the log message +of the created commit, but the additional commentary will be thrown +away once the "fixup!" commit is squashed into __ by `git rebase --autosquash`. + The commit created by `--fixup=amend:` is similar but its title is instead prefixed with "amend!". The log message of __ is copied into the log message of the "amend!" commit and -opened in an editor so it can be refined. When `git rebase ---autosquash` squashes the "amend!" commit into __, the -log message of __ is replaced by the refined log message -from the "amend!" commit. It is an error for the "amend!" commit's -log message to be empty unless `--allow-empty-message` is -specified. +opened in an editor so it can be refined. The replacement message may +also be supplied directly using `-m`, `-F`, or `-C`, bypassing the +need to open an editor, or using `-c` to open the editor pre-populated +with the referenced commit's message. When `git rebase +--autosquash` squashes the "amend!" commit into __, the log +message of __ is replaced by the refined log message from the +"amend!" commit. It is an error for the "amend!" commit's log message +to be empty unless `--allow-empty-message` is specified. + `--fixup=reword:` is shorthand for `--fixup=amend: --only`. It creates an "amend!" commit with only a log message diff --git a/Documentation/git-diff-hunks.adoc b/Documentation/git-diff-hunks.adoc new file mode 100644 index 00000000000000..25cab2ea7db8d8 --- /dev/null +++ b/Documentation/git-diff-hunks.adoc @@ -0,0 +1,146 @@ +git-diff-hunks(1) +================= + +NAME +---- +git-diff-hunks - Inspect and manage the diff-hunks store + +SYNOPSIS +-------- +[synopsis] +git diff-hunks verify +git diff-hunks clear + +DESCRIPTION +----------- + +The diff-hunks store is a cache of diff hunk coordinates, the line +ranges that changed between two blobs, so that commands +which need them, such as linkgit:git-blame[1] and `git log` and `git diff` +with the `--stat`, `--numstat`, and `--shortstat` formats, can skip +running the diff algorithm, and blame can skip loading the blob +content. (The summary formats still test each pair for binariness, +which can load the blobs.) + +The store is a single file, `$GIT_DIR/objects/info/diff-hunks`. Reading is +enabled by default; writing is off by default. A `git diff`, `git log`, +`git show`, or `git diff-tree` that produces one of the stat formats +fills the store as a side effect, but only when writing is enabled for +that run (see "WARMING THE STORE" below), so ordinary reads never +modify the repository. When the store does not have the pair, holds a +different object hash, the file is unreadable, or an object replacement +redirects one of the blobs, the consumer falls back to computing the +diff. A store only speeds up these commands; it never changes their +output. + +`git diff-hunks` itself only inspects and manages the file. See +linkgit:gitformat-diff-hunks[5] for the file format. + +WARMING THE STORE +----------------- + +The store is filled by running ordinary commands with writing enabled. +Turn writing on for a single invocation with the `GIT_DIFF_HUNKS_WRITE` +environment variable, or persistently with the `diffHunks.write` +configuration; the environment variable takes precedence. A repository +owner warms the store by running the diff-producing commands they care +about with writing on, for example: + + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null + +A `--stat` walk records one entry per blob pair; +linkgit:git-blame[1] replays the coordinates and the summary formats +sum the counts, so a single warming walk serves both. +A warming run seeds from the existing store and rewrites the file +with the newly computed pairs merged in, so a later run adds to what +earlier runs recorded rather than discarding it. + +A walk records only the pairs it diffs. `git log --all --stat` diffs +each commit against its first parent, so a blame that follows a +merge's second parent computes those pairs itself: blame coverage is +partial on history with merges. Warming with a walk that also diffs +the other parents, for example `git log --all -m --stat`, raises +blame coverage at the cost of a larger store and a longer warming +run. + +COMMANDS +-------- + +`verify`:: + Check the integrity of the store: the trailing hash checksum, the + chunk table of contents, the sort order of the index, and the + bounds of every entry. Exits with non-zero status if the store is + corrupt. An absent store is valid. + +`clear`:: + Remove the store file. + +CORRECTNESS +----------- + +A stored result is interchangeable with a freshly computed one because an +entry is keyed by the inputs that determine the diff: + +* the object IDs of the old and new blob, so a result is used only for + the exact contents it was computed from; and +* the diff algorithm and ignore flags (`xdl_opts`) the hunks were + computed under. A lookup whose `xdl_opts` differ from a stored entry + misses. This is why, for example, `blame -w` and + `--diff-algorithm=` (including a per-path + `diff..algorithm`) do not reuse entries recorded under the + default settings: they change `xdl_opts`. + +The context length is not part of the key because only trim-stable +pairs are recorded: pairs whose zero-context trimmed diff and untrimmed +diff are identical, so one entry answers blame (zero context) and the +summary formats (any context) alike. The rare pair where +the zero-context trimming optimization picks a different but +equally valid set of hunks is +never recorded and is always computed. + +Some options shape the hunks in ways the key does not express, so a +diff that uses them is excluded from the store in both directions: +break detection (`-B`), `--ignore-matching-lines` (`-I`), and +`--anchored`. `--ignore-blank-lines` is different: it is an ignore +flag and therefore part of the key, but the summary formats exclude +it anyway, because it coalesces hunks differently between the code +path that emits text and the one that replays coordinates, so a +served answer would not match a store-less run. +linkgit:git-blame[1] additionally does not +consult the store for reverse blame, ignored revisions, or paths with a +textconv driver. + +The store carries a trailing hash checksum, but readers do not +re-checksum it on every load. As with the commit-graph and +multi-pack-index, the writer fsyncs the file (honoring `core.fsync`) and +commits it atomically, so a committed store is intact; every offset and +count is still bounds-checked as it is read. The checksum is verified by +`git diff-hunks verify`, not on the read path, so structural corruption +that fails a bounds check is read as an absent entry, while a record +that stays within bounds but whose bytes were altered is served until +`verify` detects the mismatch. + +CONFIGURATION +------------- + +`core.diffHunks`:: + Whether commands read the store. Defaults to true. See + linkgit:git-config[1]. + +`diffHunks.write`:: + Whether diff-producing commands write to the store. Defaults to + false. The `GIT_DIFF_HUNKS_WRITE` environment variable overrides it + for a single invocation. See linkgit:git-config[1]. + +Writing the store honors the `core.fsync` configuration through the +`diff-hunks` component; see linkgit:git-config[1]. + +SEE ALSO +-------- +linkgit:git-blame[1], +linkgit:git-log[1], +linkgit:gitformat-diff-hunks[5] + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/git-fast-import.adoc b/Documentation/git-fast-import.adoc index d68bc52b7e9cd7..fd165e11d2d259 100644 --- a/Documentation/git-fast-import.adoc +++ b/Documentation/git-fast-import.adoc @@ -9,7 +9,7 @@ git-fast-import - Backend for fast Git data importers SYNOPSIS -------- [verse] -frontend | 'git fast-import' [] +'git fast-import' [] DESCRIPTION ----------- @@ -65,6 +65,13 @@ Only enable this option if you trust the program generating the fast-import stream! This option is enabled automatically for remote-helpers that use the `import` capability, as they are already trusted to run their own code. ++ +Note that this option has to be spelled in full, and has to appear +before any option whose value is separated from it by a space, for +the unsafe `feature` commands in the stream to be allowed. So +`--allow-unsafe` or `--depth 5 --allow-unsafe-features` still refuse +them, while `--allow-unsafe-features --depth 5` and +`--depth=5 --allow-unsafe-features` allow them. `--signed-tags=`:: Specify how to handle signed tags. Behaves in the same way as diff --git a/Documentation/git-format-rev.adoc b/Documentation/git-format-rev.adoc index 505a52feccd466..184f5ca2e9ada1 100644 --- a/Documentation/git-format-rev.adoc +++ b/Documentation/git-format-rev.adoc @@ -9,7 +9,7 @@ git-format-rev - EXPERIMENTAL: Pretty format revisions on demand SYNOPSIS -------- [synopsis] -(EXPERIMENTAL!) git format-rev --stdin-mode= --format= [--[no-]notes=] [-z] [--[no-]null-output] [--[no-]null-input] +(EXPERIMENTAL!) git format-rev [] --stdin-mode= --format= DESCRIPTION ----------- @@ -33,8 +33,8 @@ OPTIONS The argument `rev` is also accepted. `text`;; Formats all commit object names found in freeform text. These - must be full object names, i.e. abbreviated hexadecimal object - names will not be interpreted. + must be full object names, i.e. abbreviated hexadecimal (_hex_) + object names will not be interpreted. + Anything that is parsed as an object name but that is not found to be a commit object name is left alone (echoed). @@ -76,6 +76,44 @@ This is useful if the output could contain newlines, for example if the + This is useful if the input revision expressions could contain newlines. +`--color[=]`:: +`--no-color`:: + Respect color formatting. The default color behavior is + `auto`. Bare `--color` is the same as `--color=always`. ++ +Giving `--no-color` is the same as `--color=never`. ++ +__ must be one of: ++ +-- +`always`;; + Always use color, even if the output is something like a file. +`never`;; + Never use color. +`auto`;; + Use color when the output is a terminal but not when the output + is something like a file. +-- + +`--abbrev[=]`:: +`--no-abbrev`:: + Abbreviate the commit hex output. Without __ it will find the + minimum length which can describe the commit uniquely, with some + extra slack. Giving __ specifies the minimum length; a longer + length will be used if needed. ++ +Giving `--no-abbrev` will turn off abbreviation, showing the full commit +hex output. ++ +Note that some pretty formats use `--abbrev`. This behavior can be +controlled with these two options. + +`--date=`:: + Date format for pretty formats. Note that date atoms like `%aI` + are not affected. This option cannot be negated. ++ +include::rev-list-option-date-alternatives.adoc[] + [[io]] INPUT AND OUTPUT FORMAT ----------------------- @@ -93,12 +131,13 @@ acts as a _terminator_, not a _separator_. In other words, the final line or record is also terminated by the terminator character. The mode `--stdin-mode=text` replaces each object name with the -formatted commit, i.e. the format `%s` would transform some commit -object name to `` without any termination. Like this: +formatted commit, i.e. the format `"%s"` would transform some commit +object name to `""` without any termination. Like this: ----- +[synopsis] +-- Did we not fix this in ""? ----- +-- It is safe to interactively read and write from this command since each record is immediately flushed. diff --git a/Documentation/git-history.adoc b/Documentation/git-history.adoc index 28b477cd378150..2e2e31f521bee9 100644 --- a/Documentation/git-history.adoc +++ b/Documentation/git-history.adoc @@ -12,6 +12,7 @@ git history drop [--dry-run] [--update-refs=(branches|head)] [--empty=( git history fixup [--dry-run] [--update-refs=(branches|head)] [--reedit-message] [--empty=(drop|keep|abort)] git history reword [--dry-run] [--update-refs=(branches|head)] git history split [--dry-run] [--update-refs=(branches|head)] [--] [...] +git history squash [--dry-run] [--update-refs=(branches|head)] [--[no-]edit] DESCRIPTION ----------- @@ -43,8 +44,11 @@ at once. LIMITATIONS ----------- -This command does not (yet) work with histories that contain merges. You -should use linkgit:git-rebase[1] with the `--rebase-merges` flag instead. +This command does not (yet) replay merge commits onto the rewritten +history: if a commit that would be replayed is a merge, the operation is +rejected, and you should use linkgit:git-rebase[1] with the +`--rebase-merges` flag instead. The `squash` subcommand can still fold merges +that lie inside the selected range, subject to the restrictions below. Furthermore, the command does not support operations that can result in merge conflicts. This limitation is by design as history rewrites are not intended to @@ -113,6 +117,51 @@ linkgit:gitglossary[7]. It is invalid to select either all or no hunks, as that would lead to one of the commits becoming empty. +`squash `:: + Fold all commits in __ into the oldest commit of that + range. The resulting commit keeps the oldest commit's authorship and + takes the tree of the range's newest commit, so the whole range + collapses into a single commit. Commits above the range are replayed + on top of the result. ++ +The range is given in the usual `..` form, where __ is +the commit just below the oldest commit to squash. For example, `git +history squash HEAD~3..HEAD` folds the three most recent commits into +one, and `git history squash HEAD~5..HEAD~2` squashes an interior range +while leaving the two newest commits in place. Several revisions may be +given, for example `HEAD~3..HEAD ^topic` to additionally exclude what is +already on `topic`. Rev-list options may also be given, but any that would +change how the range is walked are overridden with a warning. ++ +An editor opens pre-filled with the messages of all the folded commits so you +can combine them. With `--no-edit`, the oldest commit's message is preserved +instead, except that an `amend!` commit targeting it replaces its message. ++ +The selected commits must form a connected graph with a single tip and must +not include a root commit. Every parent of a commit after the oldest one must +either be selected or also be a parent of the oldest commit. When the oldest +commit is a merge, all of its parents are preserved in the squashed commit. ++ +A `fixup!`, `squash!`, or `amend!` commit is refused unless the commit it +targets is also in the range, so the fold does not silently absorb a +marker meant for a commit outside it. As an exception, a range made up entirely +of markers for one target is combined into a single commit. With `--no-edit`, +the last `amend!` message is used if there is one; a `squash!` or `amend!` is +otherwise refused if folding it would discard its message. ++ +The editor template mirrors `git rebase -i --autosquash`: each `fixup!`, +`squash!`, or `amend!` is grouped under the commit it targets rather than +shown in commit order. A `fixup!` message is dropped (commented out in full), +a `squash!` keeps its body with only the marker subject commented, and an +`amend!` replaces its target's message, unless a `squash!` folded into that +target first, in which case it keeps its body like a `squash!`. ++ +A local branch descended from a selected commit but not from the range tip +cannot be rewritten as a descendant of the result, so with the default +`--update-refs=branches` the command refuses. Rerun with `--update-refs=head` +to rewrite only the current branch and leave such branches unchanged. Tags +and remote-tracking refs are always left unchanged. + OPTIONS ------- @@ -122,6 +171,12 @@ OPTIONS objects will be written into the repository, so applying these printed ref updates is generally safe. +`--edit`:: +`--no-edit`:: + For `squash`, open an editor to combine the messages of the folded commits. + This is the default; use `--no-edit` to keep the selected message without + opening an editor. + `--reedit-message`:: Open an editor to modify the target commit's message. diff --git a/Documentation/git-interpret-trailers.adoc b/Documentation/git-interpret-trailers.adoc index 77b4f63b05cf5b..903d598dcb0595 100644 --- a/Documentation/git-interpret-trailers.adoc +++ b/Documentation/git-interpret-trailers.adoc @@ -3,7 +3,7 @@ git-interpret-trailers(1) NAME ---- -git-interpret-trailers - Add or parse structured information in commit messages +git-interpret-trailers - Add or parse metadata in commit messages SYNOPSIS -------- @@ -14,9 +14,15 @@ git interpret-trailers [--in-place] [--trim-empty] DESCRIPTION ----------- -Add or parse _trailer_ lines that look similar to RFC 822 e-mail -headers, at the end of the otherwise free-form part of a commit -message. For example, in the following commit message +Add or parse trailer metadata at the end of the otherwise +free-form part of a commit message, or any other kind of text. + +A _trailer_ in its simplest form is a key-value pair with a colon as a +separator. The _key_ consists of ASCII alphanumeric characters and +hyphens (`-`). A _trailer block_ consists of one or more trailers. The +trailer block needs to be preceded by a blank line, where a _blank line_ +is either an empty or a whitespace-only line. For example, in the +following commit message ------------------------------------------------ subject @@ -54,10 +60,18 @@ are applied to each input and the way any existing trailer in the input is changed. They also make it possible to automatically add some trailers. -By default, a `=` or `:` argument given -using `--trailer` will be appended after the existing trailers only if -the last trailer has a different (__, __) pair (or if there -is no existing trailer). The __ and __ parts will be trimmed +Let's consider new trailers added with `--trailer`. +By default, the new trailer will appear at the end of the trailer block. +Also by default, this new trailer will only be added +if the last trailer is different to it. +A trailer block will be created with only that trailer if a trailer +block does not already exist. Recall that a trailer block needs to be +preceded by a blank line, so a blank line will be inserted before the +new trailer block in that case. + +This is how the new trailer is added: a `=` or +`:` argument given using `--trailer` will be appended after +the existing trailers. The __ and __ parts will be trimmed to remove starting and trailing whitespace, and the resulting trimmed __ and __ will appear in the output like this: @@ -68,6 +82,16 @@ key: value This means that the trimmed __ and __ will be separated by "`:`{nbsp}" (one colon followed by one space). +Existing trailers are extracted from the input by looking for the +trailer block. A trailer block is a group of one or more lines that (i) +is all trailers, or (ii) contains at least one Git-generated or +user-configured trailer and consists of at +least 25% trailers. +The trailer block is by definition at the end of the commit message. +The message in turn is either (i) at the end of the input, or (ii) the +last non-whitespace lines before a line that starts with `---` (followed +by a space or the end of the line). + For convenience, a __ can be configured to make using `--trailer` shorter to type on the command line. This can be configured using the `trailer..key` configuration variable. The __ must be a prefix @@ -81,20 +105,6 @@ trailer.sign.key "Signed-off-by: " in your configuration, you only need to specify `--trailer="sign: foo"` on the command line instead of `--trailer="Signed-off-by: foo"`. -By default the new trailer will appear at the end of all the existing -trailers. If there is no existing trailer, the new trailer will appear -at the end of the input. A blank line will be added before the new -trailer if there isn't one already. - -Existing trailers are extracted from the input by looking for -a group of one or more lines that (i) is all trailers, or (ii) contains at -least one Git-generated or user-configured trailer and consists of at -least 25% trailers. -The group must be preceded by one or more empty (or whitespace-only) lines. -The group must either be at the end of the input or be the last -non-whitespace lines before a line that starts with `---` (followed by a -space or the end of the line). - When reading trailers, there can be no whitespace before or inside the __, but any number of regular space and tab characters are allowed between the __ and the separator. There can be whitespaces before, @@ -107,8 +117,22 @@ key: This is a very long value, with spaces and newlines in it. ------------------------------------------------ -Note that trailers do not follow (nor are they intended to follow) many of the -rules for RFC 822 headers. For example they do not follow the encoding rule. +OTHER RULES +----------- + +What was covered in the previous section are the rules that are relevant +for regular use. The following points are included for completeness. + +-- +* This command ignores comment lines (see `core.commentString` in + linkgit:git-config[1]). This is for use with the `prepare-commit-msg` + and `commit-msg` hooks. + +* Candidate trailer lines that have `:` as the separator, that have no + whitespace before the value part, and that start with `//` are not + recognized as trailers. This is to avoid accidentally interpreting + URLs as trailers (e.g. lines that start with `https://`). +-- OPTIONS ------- @@ -402,6 +426,29 @@ mv "\$1.new" "\$1" $ chmod +x .git/hooks/commit-msg ------------ +* Here we try to use three different trailer keys. But it fails because + two of them are not recognized as trailer keys. ++ +---- +$ cat msg.txt +subject + +Skapad-på: some-branch +Hash-in-v6.11: 45c12d3269fe48f22834320c782ffe86c3560f2c +Reviewed-by: Alice +$ git interpret-trailers --only-trailers itself rather than the reference it points to via a symbolic ref. -KNOWN LIMITATIONS ------------------ - -The ref format migration has several known limitations in its current form: - -* It is not possible to migrate repositories that have worktrees. - -* There is no way to block concurrent writes to the repository during an - ongoing migration. Concurrent writes can lead to an inconsistent migrated - state. Users are expected to block writes on a higher level. If your - repository is registered for scheduled maintenance, it is recommended to - unregister it first with git-maintenance(1). - -These limitations may eventually be lifted. - GIT --- Part of the linkgit:git[1] suite diff --git a/Documentation/git-repack.adoc b/Documentation/git-repack.adoc index 72c42015e23f94..a1f9e64f668750 100644 --- a/Documentation/git-repack.adoc +++ b/Documentation/git-repack.adoc @@ -12,6 +12,7 @@ SYNOPSIS 'git repack' [-a] [-A] [-d] [-f] [-F] [-l] [-n] [-q] [-b] [-m] [--window=] [--depth=] [--threads=] [--keep-pack=] [--write-midx[=]] [--name-hash-version=] [--path-walk] + [--filter=] [--drop-filtered [--dry-run]] DESCRIPTION ----------- @@ -182,6 +183,42 @@ depth is 4095. `objects` and `objects/info/alternates` sections of linkgit:gitrepository-layout[5]. +--drop-filtered:: + Delete the local objects that match the `--filter` specification + instead of keeping them in a separate packfile, reclaiming the + disk space they occupy. This is intended for partial clones, + where the filtered objects are promisor objects that remain + recoverable from the promisor remote and are lazily re-fetched + on demand when they are next needed. ++ +Only large blobs are supported for now, so `--filter=blob:limit=` +is currently the only accepted filter. Because dropped objects must be +recoverable, this option requires a promisor remote to be configured +and refuses to run otherwise. ++ +This option requires `-a`, and implies `-d`: the objects are dropped by +rebuilding the promisor pack without them and then removing the now +redundant old packs, so the redundant packs must be deleted for the +space to actually be reclaimed. It is incompatible with `--filter-to` +and with bitmap writing (`-b`/`--write-bitmap-index`), since filtering +breaks the single-pack closure that bitmaps require. A bitmap setting +coming from configuration is silently disabled for the duration of the +command. ++ +As a convenience, since dropped objects remain recoverable by lazy fetch, +`--drop-filtered` refuses to run while another operation +(merge, rebase, am, cherry-pick, revert, or bisect) is in progress, to +avoid a surprising network fetch mid-operation, and refuses to drop any +blob that the current index references, since such a blob would only be +lazily re-fetched by the next command that inspects the working tree. +These checks are skipped in bare repositories, which have neither a +working tree nor an index. + +--dry-run:: + Only meaningful with `--drop-filtered`. List the objects that + would be dropped, one object ID per line, without rebuilding any + pack or deleting anything. + -b:: --write-bitmap-index:: Write a reachability bitmap index as part of the repack. This diff --git a/Documentation/git-replay.adoc b/Documentation/git-replay.adoc index ea4d14baddb6a9..026a2b87dc4692 100644 --- a/Documentation/git-replay.adoc +++ b/Documentation/git-replay.adoc @@ -10,7 +10,7 @@ SYNOPSIS -------- [verse] (EXPERIMENTAL!) 'git replay' ([--contained] --onto= | --advance= | --revert=) - [--ref=] [--ref-action=] + [--ref=] [--ref-action=] [--linearize] DESCRIPTION ----------- @@ -91,6 +91,23 @@ Expanded description list compared to 'replay.refAction'. + The default mode can be configured via the `replay.refAction` configuration variable. +--linearize:: + In this mode, each replayed commit is stacked on top of the + previously replayed one, so all replayed commits are flattened into + a single linear history. ++ +When a merge commit is encountered, the behavior of git-rebase(1)'s +option `--no-rebase-merges` is imitated. All commits in the range +reachable from the merge commit are replayed into a linear history, and +the merge commit itself is dropped. A ref that pointed to a merge commit +is updated to the merge's last replayed ancestor. ++ +Only a single branch can be linearized at a time: `--linearize` cannot +be combined with multiple positive revisions or with `--contained`, +because that would concatenate otherwise unrelated histories into one +line. To linearize several branches, replay them in separate `git +replay` invocations. + :: Range of commits to replay; see "Specifying Ranges" in linkgit:git-rev-parse[1]. In `--advance=` or diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index ed7d80c690c720..fb5aceae8f2ce7 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -113,12 +113,69 @@ values that they return: The path to the Git repository's common directory relative to the current working directory. +`path.git-prefix`:: + The path from the root of the working tree to the current working + directory. Returns the empty string when the current working directory + is the root of the working tree. + `path.gitdir.absolute`:: The canonical absolute path to the Git repository directory (the `.git` directory). `path.gitdir.relative`:: The path to the Git repository directory relative to the current working directory. +`path.grafts.absolute`:: + The canonical absolute path to the repository's graft file. + Respects the `GIT_GRAFT_FILE` environment override. The path is + returned regardless of whether the file currently exists on disk. + +`path.grafts.relative`:: + The path to the repository's graft file relative to the current + working directory. Respects the `GIT_GRAFT_FILE` environment + override. The path is returned regardless of whether the file + currently exists on disk. + +`path.hooks.absolute`:: + The canonical absolute path to the repository's hooks directory. + Respects the `core.hooksPath` configuration. If `core.hooksPath` is + set to `/dev/null`, that value is returned unchanged. + +`path.hooks.relative`:: + The path to the repository's hooks directory relative to the current + working directory. Respects the `core.hooksPath` configuration. + +`path.index.absolute`:: + The canonical absolute path to the repository's current index file. + Respects the `GIT_INDEX_FILE` environment override. Returns the + configured index path even if the repository is bare or the file does + not exist. + +`path.index.relative`:: + The path to the repository's current index file relative to the current + working directory. Respects the `GIT_INDEX_FILE` environment override. + Returns the configured index path even if the repository is bare or the + file does not exist. + +`path.superproject-root.absolute`:: + The canonical absolute path to the working tree root of the superproject + if the current repository is an initialized submodule. Outputs an empty + string if not in a submodule. + +`path.superproject-root.relative`:: + The path to the working tree root of the superproject relative to the + current working directory if the current repository is an initialized + submodule. Outputs an empty string if not in a submodule. + +`path.toplevel.absolute`:: + The canonical absolute path to the top-level directory of the + repository's working tree. Outputs an empty string if the repository + is bare. + +`path.toplevel.relative`:: + The path to the top-level directory of the repository's working + tree relative to the current working directory. Outputs an empty + string if the repository is bare. + `references.format`:: The reference storage format. The valid values are: + diff --git a/Documentation/git-upload-pack.adoc b/Documentation/git-upload-pack.adoc index 9167a321d08e51..90c2ba119463c5 100644 --- a/Documentation/git-upload-pack.adoc +++ b/Documentation/git-upload-pack.adoc @@ -71,6 +71,11 @@ This is implemented by having `upload-pack` internally set the (because you are fetching from a partial clone, and you are sure you trust it), you can explicitly set `GIT_NO_LAZY_FETCH` to `0`. ++ +Instead of setting `GIT_NO_LAZY_FETCH` to `0` in the environment, a +server operator can allow lazy fetching on a per-repository basis by +listing trusted repositories in the `uploadpack.lazyFetchTrusted` +configuration variable. See linkgit:git-config[1]. SECURITY -------- diff --git a/Documentation/git.adoc b/Documentation/git.adoc index 8a5cdd3b3d22c5..2e763d1f9357c3 100644 --- a/Documentation/git.adoc +++ b/Documentation/git.adoc @@ -949,7 +949,9 @@ for full details. `GIT_NO_LAZY_FETCH`:: Setting this Boolean environment variable to true tells Git not to lazily fetch missing objects from the promisor remote - on demand. + on demand. On the server side, the `uploadpack.lazyFetchTrusted` + configuration variable can control this per-repository. See + linkgit:git-upload-pack[1]. `GIT_REFLOG_ACTION`:: When a ref is updated, reflog entries are created to keep diff --git a/Documentation/gitattributes.adoc b/Documentation/gitattributes.adoc index 9fea75f96f928f..9a50d6338b8e23 100644 --- a/Documentation/gitattributes.adoc +++ b/Documentation/gitattributes.adoc @@ -784,6 +784,17 @@ with the above configuration, i.e. `j-c-diff`, with 7 parameters, just like `GIT_EXTERNAL_DIFF` program is called. See linkgit:git[1] for details. +An external diff driver replaces the patch Git would otherwise +produce for the path: Git runs the command and shows its output in +place of its own. Output features that post-process Git's diff do +not apply to the driver's output; word diff, function context (`-W`), +`--color-moved`, and coloring all act on Git's builtin diff, not the +driver's output. +The driver is consulted only when Git generates a textual patch. The +summary formats (`--stat`, `--numstat`, `--shortstat`, and +`--dirstat`), `git blame`, and `git log -L` do not run it and +continue to use Git's builtin diff. + If the program is able to ignore certain changes (similar to `git diff --ignore-space-change`), then also set the option `trustExitCode` to true. It is then expected to return exit code 1 if @@ -821,6 +832,166 @@ NOTE: If `diff..command` is defined for path with the (see above), and adding `diff..algorithm` has no effect, as the algorithm is not passed to the external diff driver. +Answering diffs from a long-running process +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Unlike `diff..command`, which replaces the textual patch, the +process configured in `diff..process` feeds hunks back into +Git's own machinery: +it answers "which line ranges changed between these two blobs", and +Git's output is produced from that answer. A process is started +lazily, once per configured command string and per repository, and +consulted over a pkt-line protocol (following the long-running filter +process protocol; see the "Long Running Filter Process" section above +for the filter analogue). + +The process is asked by object names alone: a request carries the +pathname and the `old-oid`/`new-oid` of the blob pair, and no content. +The pathname is the repository-relative old-side (preimage) path, not +the shortened display path a `--relative` diff shows, so a driver +scoped to a directory matches whatever directory the command runs from. +This suits a process that keeps a persistent cache keyed on the pair, and +a process that fetches the blobs itself (for example via +`git cat-file --batch`) to compute its own notion of the changed +lines. Pairs with a side that is not a stored blob (a working-tree +file, textconv output) are not sent; Git computes those itself. + +The exchange opens with a handshake: Git announces its role and +version, the process replies in kind, and then Git lists the +capabilities it supports and the process replies with the ones it +implements. A process that announces a capability Git did not list +aborts the command. + +----------------------- +packet: git> git-diff-client +packet: git> version=1 +packet: git> 0000 +packet: git< git-diff-server +packet: git< version=1 +packet: git< 0000 +packet: git> capability=hunks-by-oid +packet: git> 0000 +packet: git< capability=hunks-by-oid +packet: git< 0000 +----------------------- + +After the handshake, each request and response looks like: + +----------------------- +packet: git> command=hunks-by-oid +packet: git> pathname=path/file.c +packet: git> old-oid= +packet: git> new-oid= +packet: git> 0000 +packet: git< hunk +packet: git< 0000 +packet: git< status=success +packet: git< 0000 +----------------------- + +Start values are 1-based and counts are non-negative; a count of 0 +describes a pure insertion or deletion at the 1-based line the change +sits before (a start of 0 is accepted for an empty file side). Hunks +must be listed in order, must not overlap, and must keep the unchanged +runs between them the same length on both sides; Git validates this +and, with a warning, falls back to its builtin diff on a response +that violates these rules. + +A `status=success` response with zero hunks asserts that the blobs are +equivalent, including their trailing newlines. A process that cannot +answer a pair from its object names (or cannot rule out a +trailing-newline-only difference) responds `status=need-content`, and +Git produces that pair's diff itself. An asserted equivalence makes +the pair vanish from the summary formats, but the pair still counts +as changed for `--exit-code`, the same way a whitespace-only pair +does under `-w`. + +The status names the disposition of the whole request. Git +understands three: `success` (the hunk lines are the answer), +`need-content` (Git produces this pair's diff itself), and `abort`, which +withdraws the capability the request used: Git stops sending +`hunks-by-oid` requests to that process for the rest of the command, +while the process stays alive for request forms negotiated under +other capabilities. Any other status is a protocol error: Git warns, +stops the process, and uses the builtin diff for the remainder of the +command. + +Every response has the same shape whatever its status: zero or more +hunk lines, a flush packet, and a status packet terminated with a +flush packet. A response that carries no hunks, `need-content` +included, still begins with the empty hunk section's flush packet; a +bare status packet is a protocol error: + +----------------------- +packet: git< 0000 +packet: git< status=need-content +packet: git< 0000 +----------------------- + +The process must read the entire request before it responds; Git +writes the whole request before it reads the response. + +The protocol extends without breaking deployed processes: a process +must ignore request keys it does not recognize, and Git ignores +trailing space-separated tokens after the last field of a hunk line, +so a later protocol version can append request keys and hunk fields. +New request forms arrive as capabilities, which a process may decline +to announce; announcing a capability Git did not request aborts the +command, as it does under the long-running filter process protocol. + +There is no shutdown handshake: Git's side of the pipes closes when +the command exits, and the process should exit when it reads EOF. No +flush point is guaranteed, so a process that maintains persistent +state (such as a cache) should persist as it answers rather than at +exit. Git applies no timeout to a response; a process that hangs +hangs the command, as with the long-running filter processes. + +`git blame` and the `--stat`, `--numstat`, and `--shortstat` formats +consult the process; the textual patch and `git log -L` range +tracking are produced by the builtin machinery, so a process whose +answers deliberately differ from the builtin diff shows that +difference only in blame and those formats. `--dirstat=lines` routes +through the diffstat path and consults; the other `--dirstat` modes do +not. A merge's `--stat` (including under `--cc`) is computed against +the first parent, so it consults like any other stat; the combined +patch itself compares one merge result against all of its parents at +once, which the pairwise request above does not express, so that patch +uses the builtin diff, and extending the protocol to combined diffs is +left for future work. A content-carrying extension of this +protocol would bring patch output and `git log -L` range tracking to +the same answer. + +Consulting is allowed per command, as with textconv: `git diff`, +`git log` (`git whatchanged` included) and `git show`, and +`git blame` consult a configured process; the plumbing diff commands +do not unless `--ext-diff` or `--diff-process` is given explicitly, +and the interactive-patch commands (`git add -p` and friends), which +build the hunks they present from plumbing output, always stage from +the builtin diff. `git range-diff` generates the patches it +compares with `--no-ext-diff`. `--diff-process` and +`--no-diff-process` allow or forbid only the consulting; +`--no-ext-diff` disables all external diff mechanisms, this one +included. Options the process is never told about never select it: +with the whitespace-ignoring options, `--ignore-matching-lines`, and +`--anchored`, the pair is answered as when no process is configured. +`--diff-algorithm` (or a configured `diff.algorithm`) forces a builtin +algorithm and bypasses the process the same way. A per-path +++diff.++____++.algorithm++ does so for `git diff` and the stat +formats, which build their diff parameters from it; `git blame` builds +its parameters from its own diff options, so a per-driver algorithm +does not by itself keep blame from consulting the process. +`git format-patch` never +consults the process, so generated patches are always based on the +builtin diff and apply for recipients without the process. On a +path whose driver has a process, the process is consulted before the +diff-hunks store (see linkgit:git-diff-hunks[1]): a pair the process +answers is never served from the store and never recorded into it. +A pair the process does not answer, for example with +`status=need-content`, gets the builtin diff, so the store may serve +it and a warming run may record it: the store holds builtin results, +and for such a pair the builtin result is what would be computed +anyway. + Defining a custom hunk-header ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Documentation/gitformat-diff-hunks.adoc b/Documentation/gitformat-diff-hunks.adoc new file mode 100644 index 00000000000000..f75ab73dd94a75 --- /dev/null +++ b/Documentation/gitformat-diff-hunks.adoc @@ -0,0 +1,129 @@ +gitformat-diff-hunks(5) +======================= + +NAME +---- +gitformat-diff-hunks - Precomputed diff hunk store format + +SYNOPSIS +-------- +[verse] +$GIT_DIR/objects/info/diff-hunks + +DESCRIPTION +----------- + +The diff-hunks store memoizes diff hunk coordinates so that commands +that need them, such as `git log --stat` and linkgit:git-blame[1], can +skip running the diff algorithm (and, for blame, loading the blob +content; the summary formats still test each pair for binariness, +which can load the blobs). See +linkgit:git-diff-hunks[1] for how the store is filled and managed and the +configuration that controls it. + +The store is a single file, `$GIT_DIR/objects/info/diff-hunks`, written +in one pass and replaced atomically, so a reader sees either the old +file or the complete new one. + +Entries are keyed by the object IDs of the blob pair that was diffed +and by the diff algorithm and ignore flags (`xdl_opts`) the pair was +diffed under. A blob pair fully determines the diff input, so an entry +is valid regardless of which commits, branches, or index states the +pair was encountered in, and identical diffs performed in different +contexts share one entry. A reader whose `xdl_opts` differ from an +entry does not match it and falls back to computing the diff. + +FILE FORMAT +----------- + +All multi-byte integers are stored in network byte order. The file is an +8-byte header, the chunk table of contents and chunk data described in +linkgit:gitformat-chunk[5], and a trailing checksum. + +HEADER +~~~~~~ + +- 4-byte signature: `DHPF` (diff-hunks precomputed format) +- 1-byte version number: currently 1 +- 1-byte hash version: 1 for SHA-1, 2 for SHA-256. A store whose hash + function differs from the repository's is ignored. +- 1-byte number of chunks +- 1-byte reserved + +CHUNK LOOKUP +~~~~~~~~~~~~ + +A table of contents in the format of linkgit:gitformat-chunk[5], listing +the offset of each chunk. Both chunks below are required; a file missing +either is treated as corrupt. + +CHUNK DATA +~~~~~~~~~~ + +DHIX (index):: + A sorted sequence of fixed-size entries. Each entry is the old + blob object ID, the new blob object ID, a 4-byte `xdl_opts` + value, and a 4-byte offset into the DHDT chunk. Entries are + sorted by old object ID, then new object ID, then `xdl_opts`, + so lookups can use binary search on the full key. + +DHDT (hunk data):: + For each index entry, at its offset: a 4-byte hunk count followed + by that many 16-byte hunk records. A hunk record is four 4-byte + values: old start, old count, new start, new count. + Starts are 0-based line numbers in the old and new blob; counts + are numbers of lines. The hunk count is at least 1: a record with + no hunks would claim the blob pair equivalent, which the store + never records, so readers treat such a record as invalid. + Identical hunk blocks are stored once: + distinct index entries whose recorded hunks are byte-for-byte + equal point at the same offset. + +TRAILER +~~~~~~~ + +A checksum of all preceding bytes, computed with the repository hash +function. + +CORRECTNESS +----------- + +Serving hunks from a valid store produces the same output as recomputing +the diff. The diff of a blob pair is not unique: a zero context length +triggers xdiff's common-tail trimming, which can pick a different but +equally valid set of hunks than an untrimmed diff does. A pair is +therefore recorded only when its trimmed and untrimmed diffs are +identical, which is the common case. Such an entry answers any consumer +at any context: git-blame replays its coordinates directly (it diffs at +zero context), and diffstat sums its per-hunk line counts, which the +context length does not change. The rare pair whose two diffs differ is +never recorded, so every consumer computes it. + +A store that cannot be used is ignored, and the consumer falls back to +computing the diff. Every offset and count read from the file is +bounds-checked, so a store that is missing, truncated, of an unknown +version, or of a different object hash does not change the diff output +and does not produce a diagnostic; `git diff-hunks verify` is what +reports corruption. + +The store is not re-checksummed on the read path. The writer fsyncs the +file (honoring `core.fsync`) and commits it atomically, so a +committed store is intact, the same trust model the commit-graph and +multi-pack-index use. The trailing checksum is recomputed by +`git diff-hunks verify` to detect corruption. + +The checksum detects corruption but does not prove who wrote the file. A +reader trusts the coordinates in a store that passes its checks, so +anything able to write a checksum-valid file at the store path can +influence output, the same as it could by writing objects directly. + +LIMITATIONS +----------- + +- Hunk counts, offsets, and line coordinates are 32-bit, capping the + hunk data at 4 GiB and a single entry at roughly 268 million hunks. + A result whose coordinates cannot be represented is not recorded. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/gitprotocol-v2.adoc b/Documentation/gitprotocol-v2.adoc index 7bf62014c3917d..dd52fd8110dcf1 100644 --- a/Documentation/gitprotocol-v2.adoc +++ b/Documentation/gitprotocol-v2.adoc @@ -558,14 +558,17 @@ object-info `object-info` is the command to retrieve information about one or more objects. Its main purpose is to allow a client to make decisions based on this -information without having to fully fetch objects. Object size is the only -information that is currently supported. +information without having to fully fetch objects. Currently only object size +and type are supported. An `object-info` request takes the following arguments: size Requests size information to be returned for each listed object id. + type + Requests type information to be returned for each listed object id. + oid Indicates to the server an object which the client wants to obtain information for. They must be full OIDs. @@ -580,11 +583,18 @@ space. info = *PKT-LINE(attr LF) *PKT-LINE(obj-info LF) - attr = "size" + attr = "size" | "type" obj-size = 1*DIGIT - obj-info = obj-id [SP [obj-size]] + obj-type = "blob" | "tree" | "commit" | "tag" + + obj-val = obj-size | obj-type + + obj-info = obj-id [SP [obj-val *(SP obj-val)]] + +The values in `obj-info` appear in the same order as the corresponding `attr` +lines, with exactly one value per requested attribute. If the server does not recognize the OID, the response will be ` SP` regardless of the number of attributes requested. diff --git a/Documentation/line-range-options.adoc b/Documentation/line-range-options.adoc index 72f639b5e79ea4..d619ffa6336513 100644 --- a/Documentation/line-range-options.adoc +++ b/Documentation/line-range-options.adoc @@ -9,13 +9,20 @@ __ and __ (or __) must exist in the starting revision. You can specify this option more than once. Implies `--patch`. Patch output can be suppressed using `--no-patch`. - Non-patch diff formats `--raw`, `--name-only`, `--name-status`, - and `--summary` are supported. Diff stat formats - (`--stat`, `--numstat`, `--shortstat`, `--dirstat`) are not - currently implemented. + The following non-patch diff formats are supported: `--raw`, + `--name-only`, `--name-status`, `--summary`, `--check`, + `--stat`, `--numstat`, and `--shortstat`. + The stat formats count only lines within the tracked range. + `--dirstat` is not supported + with `-L`: it summarizes change as each directory's share of + the total churn, not as counts for the tracked lines. Use + `--numstat` for exact per-file counts within the range. + Patch formatting options such as `--word-diff`, `--color-moved`, `--no-prefix`, and whitespace options (`-w`, `-b`) are supported, -as are pickaxe options (`-S`, `-G`) and `--diff-filter`. +as are pickaxe options (`-S`, `-G`) and `--diff-filter`. `-G` is +scoped to the tracked range; `-S` is still evaluated over the whole +file, so an `-S` query may select a commit for a change outside the +range. + include::line-range-format.adoc[] diff --git a/Documentation/meson.build b/Documentation/meson.build index f4854f802d455f..99cb1cf34afce6 100644 --- a/Documentation/meson.build +++ b/Documentation/meson.build @@ -41,6 +41,7 @@ manpages = { 'git-describe.adoc' : 1, 'git-diagnose.adoc' : 1, 'git-diff-files.adoc' : 1, + 'git-diff-hunks.adoc' : 1, 'git-diff-index.adoc' : 1, 'git-diff-pairs.adoc' : 1, 'git-difftool.adoc' : 1, @@ -175,6 +176,7 @@ manpages = { 'gitformat-bundle.adoc' : 5, 'gitformat-chunk.adoc' : 5, 'gitformat-commit-graph.adoc' : 5, + 'gitformat-diff-hunks.adoc' : 5, 'gitformat-index.adoc' : 5, 'gitformat-loose.adoc' : 5, 'gitformat-pack.adoc' : 5, @@ -379,13 +381,18 @@ foreach manpage, category : manpages output: fs.stem(manpage) + '.xml', ) + man_base_url = 'file://' + htmldir + '/' + if get_option('man_base_url') != '' + man_base_url = get_option('man_base_url') + endif + doc_targets += custom_target( command: [ xmlto, '-m', '@INPUT0@', '-m', '@INPUT1@', '--stringparam', - 'man.base.url.for.relative.links=' + get_option('prefix') / get_option('mandir'), + 'man.base.url.for.relative.links=' + man_base_url, 'man', manpage_xml_target, '-o', diff --git a/Documentation/rev-list-option-date-alternatives.adoc b/Documentation/rev-list-option-date-alternatives.adoc new file mode 100644 index 00000000000000..141570b10598f6 --- /dev/null +++ b/Documentation/rev-list-option-date-alternatives.adoc @@ -0,0 +1,55 @@ +-- +`--date=relative` shows dates relative to the current time, +e.g. ``2 hours ago''. The `-local` option has no effect for +`--date=relative`. + +`--date=local` is an alias for `--date=default-local`. + +`--date=iso` (or `--date=iso8601`) shows timestamps in a ISO 8601-like format. +The differences to the strict ISO 8601 format are: + + - a space instead of the `T` date/time delimiter + - a space between time and time zone + - no colon between hours and minutes of the time zone + +`--date=iso-strict` (or `--date=iso8601-strict`) shows timestamps in strict +ISO 8601 format. + +`--date=rfc` (or `--date=rfc2822`) shows timestamps in RFC 2822 +format, often found in email messages. + +`--date=short` shows only the date, but not the time, in `YYYY-MM-DD` format. + +`--date=raw` shows the date as seconds since the epoch (1970-01-01 +00:00:00 UTC), followed by a space, and then the timezone as an offset +from UTC (a `+` or `-` with four digits; the first two are hours, and +the second two are minutes). I.e., as if the timestamp were formatted +with `strftime("%s %z")`). +Note that the `-local` option does not affect the seconds-since-epoch +value (which is always measured in UTC), but does switch the accompanying +timezone value. + +`--date=human` shows the timezone if the timezone does not match the +current time-zone, and doesn't print the whole date if that matches +(ie skip printing year for dates that are "this year", but also skip +the whole date itself if it's in the last few days and we can just say +what weekday it was). For older dates the hour and minute is also +omitted. + +`--date=unix` shows the date as a Unix epoch timestamp (seconds since +1970). As with `--raw`, this is always in UTC and therefore `-local` +has no effect. + +`--date=format:` feeds the __ to your system `strftime`, +except for `%s`, `%z`, and `%Z`, which are handled internally. +Use `--date=format:%c` to show the date in your system locale's +preferred format. See the `strftime`(3) manual for a complete list of +format placeholders. When using `-local`, the correct syntax is +`--date=format-local:`. + +`--date=default` is the default format, and is based on ctime(3) +output. It shows a single line with three-letter day of the week, +three-letter month, day-of-month, hour-minute-seconds in "HH:MM:SS" +format, followed by 4-digit year, plus timezone information, unless +the local time zone is used, e.g. `Thu Jan 1 00:00:00 1970 +0000`. +-- diff --git a/Documentation/rev-list-options.adoc b/Documentation/rev-list-options.adoc index fd831f0ec64744..6e6093f474732a 100644 --- a/Documentation/rev-list-options.adoc +++ b/Documentation/rev-list-options.adoc @@ -1132,61 +1132,7 @@ include::pretty-options.adoc[] author's). If `-local` is appended to the format (e.g., `iso-local`), the user's local time zone is used instead. + --- -`--date=relative` shows dates relative to the current time, -e.g. ``2 hours ago''. The `-local` option has no effect for -`--date=relative`. - -`--date=local` is an alias for `--date=default-local`. - -`--date=iso` (or `--date=iso8601`) shows timestamps in a ISO 8601-like format. -The differences to the strict ISO 8601 format are: - - - a space instead of the `T` date/time delimiter - - a space between time and time zone - - no colon between hours and minutes of the time zone - -`--date=iso-strict` (or `--date=iso8601-strict`) shows timestamps in strict -ISO 8601 format. - -`--date=rfc` (or `--date=rfc2822`) shows timestamps in RFC 2822 -format, often found in email messages. - -`--date=short` shows only the date, but not the time, in `YYYY-MM-DD` format. - -`--date=raw` shows the date as seconds since the epoch (1970-01-01 -00:00:00 UTC), followed by a space, and then the timezone as an offset -from UTC (a `+` or `-` with four digits; the first two are hours, and -the second two are minutes). I.e., as if the timestamp were formatted -with `strftime("%s %z")`). -Note that the `-local` option does not affect the seconds-since-epoch -value (which is always measured in UTC), but does switch the accompanying -timezone value. - -`--date=human` shows the timezone if the timezone does not match the -current time-zone, and doesn't print the whole date if that matches -(ie skip printing year for dates that are "this year", but also skip -the whole date itself if it's in the last few days and we can just say -what weekday it was). For older dates the hour and minute is also -omitted. - -`--date=unix` shows the date as a Unix epoch timestamp (seconds since -1970). As with `--raw`, this is always in UTC and therefore `-local` -has no effect. - -`--date=format:` feeds the __ to your system `strftime`, -except for `%s`, `%z`, and `%Z`, which are handled internally. -Use `--date=format:%c` to show the date in your system locale's -preferred format. See the `strftime`(3) manual for a complete list of -format placeholders. When using `-local`, the correct syntax is -`--date=format-local:`. - -`--date=default` is the default format, and is based on ctime(3) -output. It shows a single line with three-letter day of the week, -three-letter month, day-of-month, hour-minute-seconds in "HH:MM:SS" -format, followed by 4-digit year, plus timezone information, unless -the local time zone is used, e.g. `Thu Jan 1 00:00:00 1970 +0000`. --- +include::rev-list-option-date-alternatives.adoc[] ifdef::git-rev-list[] `--header`:: diff --git a/Documentation/technical/api-parse-options.adoc b/Documentation/technical/api-parse-options.adoc index 880eb94642587a..95b7924e84e2c8 100644 --- a/Documentation/technical/api-parse-options.adoc +++ b/Documentation/technical/api-parse-options.adoc @@ -150,8 +150,77 @@ Data Structure The main data structure is an array of the `option` struct, say `static struct option builtin_add_options[]`. + +Option flags +~~~~~~~~~~~~ + +Each option can carry flags in the `flags` field of its `option` +struct. These are per-option flags and are distinct from the +`parse_options()` flags described above; they are usually set through +the `OPT_*_F()` macro variants (see below) rather than by hand. They +are the bitwise-or of: + +`PARSE_OPT_OPTARG`:: + The option's argument is optional, i.e. both `--option` and + `--option=` are accepted. + +`PARSE_OPT_NOARG`:: + The option takes no argument at all. Using `--option=` + is rejected. + +`PARSE_OPT_NONEG`:: + Disable the automatically generated negated `--no-option` + form. + +`PARSE_OPT_HIDDEN`:: + Hide the option: it is omitted from the usage shown by + `git -h`, but is still shown by `git --help-all`. + The option is parsed as usual either way. This is meant for + deprecated, advanced or otherwise uncommon options. + +`PARSE_OPT_LASTARG_DEFAULT`:: + The no-argument form is only accepted when the option is the + last token on the command line; used earlier, it still + requires an argument. Should not be combined with + `PARSE_OPT_OPTARG`. + +`PARSE_OPT_NODASH`:: + The option is a single character without a leading dash, such + as the `+` used by some commands. + +`PARSE_OPT_LITERAL_ARGHELP`:: + Use the argument help string (`argh`) verbatim in the usage + output instead of surrounding it with `<>` or `[]`. Useful when + `argh` already contains a hand-formatted description. + +`PARSE_OPT_FROM_ALIAS`:: + Internal flag, set on options that were expanded from a + configured alias. It should not be set by callers. + +`PARSE_OPT_NOCOMPLETE`:: + Do not offer this option for completion. + +`PARSE_OPT_COMP_ARG`:: + The option's argument, rather than the option itself, is what + should be completed. + +`PARSE_OPT_CMDMODE`:: + The option is one of several mutually exclusive "command mode" + options that share the same variable. Using more than one of + them at once is rejected. + +Macros +~~~~~~ + There are some macros to easily define options: +Many of the macros below have an `_F` variant (for example `OPT_BOOL_F`, +`OPT_STRING_F`, `OPT_INTEGER_F`, `OPT_SET_INT_F`, `OPT_BIT_F` and +`OPT_CALLBACK_F`) that takes an additional trailing `flags` argument. +That argument is the bitwise-or of the per-option flags described in the +"Option flags" section above; the non-`_F` macros are simply defined +with `flags` set to `0`. + `OPT__ABBREV(&int_var)`:: Add `--abbrev[=]`. @@ -175,10 +244,21 @@ There are some macros to easily define options: describes the group or an empty string. Start the description with an upper-case letter. +`OPT_HIDDEN_GROUP(description)`:: + Like `OPT_GROUP()`, but the group header carries + `PARSE_OPT_HIDDEN`, so it is only shown by `--help-all` and not + by `-h`. Use it to label a group that contains only hidden + options, which would otherwise show an empty header under `-h`. + `OPT_BOOL(short, long, &int_var, description)`:: Introduce a boolean option. `int_var` is set to one with `--option` and set to zero with `--no-option`. +`OPT_HIDDEN_BOOL(short, long, &int_var, description)`:: + Like `OPT_BOOL()`, but the option carries `PARSE_OPT_HIDDEN`, + so it is hidden from `-h` while still being shown by + `--help-all`. + `OPT_COUNTUP(short, long, &int_var, description)`:: Introduce a count-up option. Each use of `--option` increments `int_var`, starting from zero diff --git a/Documentation/technical/meson.build b/Documentation/technical/meson.build index ec07088c57617f..9ce11d5e484d9c 100644 --- a/Documentation/technical/meson.build +++ b/Documentation/technical/meson.build @@ -18,6 +18,7 @@ articles = [ 'multi-pack-index.adoc', 'packfile-uri.adoc', 'pack-heuristics.adoc', + 'paint-down-to-common.adoc', 'parallel-checkout.adoc', 'partial-clone.adoc', 'platform-support.adoc', diff --git a/Documentation/technical/paint-down-to-common.adoc b/Documentation/technical/paint-down-to-common.adoc new file mode 100644 index 00000000000000..c33c7ea7c1d38e --- /dev/null +++ b/Documentation/technical/paint-down-to-common.adoc @@ -0,0 +1,155 @@ +Merge-Base Computation and paint_down_to_common() +================================================== + +The function `paint_down_to_common()` in `commit-reach.c` computes merge +bases by walking the commit graph backwards from two sets of tips and +finding where their ancestry meets. + +Use cases +--------- + +Computing merge bases is used in two different ways: + + 1. *Finding all merge bases* (`merge-base --all`, `merge-tree`, + `merge`, `rebase`). A merge base is a common ancestor that is + not itself an ancestor of another common ancestor. + + 2. *Ancestry checks* (`in_merge_bases`, used by `merge-base + --is-ancestor`, `branch -d`, `fetch`). These ask: "is commit A + an ancestor of commit B?" If a common ancestor equals one of the + inputs, that input is necessarily the only merge base -- no other + common ancestor can be both as recent and not an ancestor of it. + +Both use cases share the same algorithm and implementation. + +Algorithm +--------- + +Given a commit `one` and a set of commits `twos[]`, the walk paints +commits with two colors: + + - PARENT1: reachable from `one` + - PARENT2: reachable from any commit in `twos[]` + +The walk uses a priority queue ordered by generation number +(highest first), breaking ties by commit date. Each step dequeues +the highest-priority commit and propagates its paint flags to its +parents, enqueuing any parent that gained new flags. When a +commit receives both PARENT1 and PARENT2, it is a merge-base +candidate. A candidate gains the STALE flag so its ancestors +propagate staleness -- any deeper common ancestor is necessarily +redundant. + +[[generation-regions]] +Topologically ordered and unordered generation regions +------------------------------------------------------ + +Commits fall into two regions based on whether their generation +numbers provide a topological ordering guarantee: + +.... + +------------------------------------------+ + | Unordered region | + | generation = INFINITY or V1_MAX | + | queue order: heuristic (commit date) | + +------------------------------------------+ + | + v + +------------------------------------------+ + | Ordered region | + | generation = finite, unsaturated | + | queue order: topological | + +------------------------------------------+ +.... + +In the ordered region, a child's generation is strictly greater +than its parent's. Same-generation commits are necessarily +independent, so the queue always processes children before +their parents. + +In the unordered region, parent-child pairs can share the same +generation number, so topological order is not guaranteed. The +queue uses commit-date as a heuristic, which typically produces +a reasonable traversal order but may process a parent before +its child. + +Commits not in the commit-graph have generation INFINITY; v1 +commit-graphs saturate at V1_MAX. Both place commits in the +unordered region. Any optimization that depends on generation +ordering must account for this saturation boundary. The early +exit gates compare against a topological ceiling -- +`GENERATION_NUMBER_V1_MAX` for v1 graphs and +`GENERATION_NUMBER_INFINITY` for v2 graphs -- so that saturated +commits are treated as unordered. + +With generation ordering, values in the unordered region exceed +those in the ordered region. The walk may therefore transition +from the unordered region into the ordered region, but never in +the reverse direction. Without a commit-graph, every commit has INFINITY +and the walk operates entirely in the unordered region. + +In the ordered region, paint on a dequeued commit is final -- +no future step can add flags to it. In the unordered region, +a dequeued commit may later gain additional paint. Paint flags +are only added, never removed, bounding the number of +re-enqueues per commit. + +Termination +----------- + +The walk tracks the number of commits of each type in the queue +(PARENT1-only, PARENT2-only, pending merge-base). The main loop +ends when one of the following conditions holds: + + 1. The queue is empty. + 2. The queue contains only stale entries. + 3. Generation cutoff: the dequeued commit's generation is below + a caller-supplied `min_generation` threshold. + 4. Single result: the caller only needs one merge base, one has + been found, and the walk has entered the ordered region. + 5. Side exhaustion: no pure PARENT1 or pure PARENT2 commits + remain in the queue, no pending merge-base candidates exist, + and the walk has entered the ordered region. + +Stale entry condition +~~~~~~~~~~~~~~~~~~~~~ +Once all queued entries are stale, no new merge-base candidates can +be discovered -- that requires at least one non-stale commit from +each side meeting. Continuing the walk could still invalidate +existing candidates by proving one is an ancestor of another, but +`remove_redundant()` handles that as a post-processing step, so it +is safe to exit early. + +Side-exhaustion condition +~~~~~~~~~~~~~~~~~~~~~~~~~ +A new merge-base requires commits from both sides to meet. When one +side's exclusive counter reaches zero and there are no pending +merge-base candidates, no future traversal step can produce a new +candidate. This optimization only activates in the ordered region, +where paint flags are final at visit time; in the unordered region, +a side that appears exhausted could reappear through late paint +propagation. + +Generation cutoff +~~~~~~~~~~~~~~~~~ +Some callers (notably `remove_redundant()`) supply a `min_generation` +threshold equal to the minimum generation of the input commits. +These callers only need to determine reachability among the inputs, +not find deep merge bases, so the walk can safely terminate when it +dequeues a commit below this threshold. + +Single result +~~~~~~~~~~~~~ +When only one merge base is needed and the walk is in the +ordered region with generation ordering, the first candidate +found is necessarily the highest-generation common ancestor. +No remaining commit in the queue can be a descendant of this +candidate (generation ordering guarantees children are visited +first), so it cannot be redundant and the walk can stop +immediately. + +Related documentation +--------------------- + + - `Documentation/technical/commit-graph.adoc` -- generation numbers + and the reachability closure property. diff --git a/Documentation/technical/partial-clone.adoc b/Documentation/technical/partial-clone.adoc index e513e391ea82ca..18718a384053a5 100644 --- a/Documentation/technical/partial-clone.adoc +++ b/Documentation/technical/partial-clone.adoc @@ -159,7 +159,7 @@ and prefetch those objects in bulk. - `repack` in GC has been updated to not touch promisor packfiles at all, and to only repack other objects. -- The global variable "fetch_if_missing" is used to control whether an +- The per-repository flag "fetch_if_missing" is used to control whether an object lookup will attempt to dynamically fetch a missing object or report an error. + diff --git a/Documentation/technical/racy-git.adoc b/Documentation/technical/racy-git.adoc index 59bea66c0fc6ef..499231585b00c1 100644 --- a/Documentation/technical/racy-git.adoc +++ b/Documentation/technical/racy-git.adoc @@ -39,8 +39,8 @@ files) from `st_mode` member, `st_mtime` and `st_ctime` timestamps, `st_uid`, `st_gid`, `st_ino`, and `st_size` members. With a `USE_STDEV` compile-time option, `st_dev` is also compared, but this is not enabled by default because this member -is not stable on network filesystems. With `USE_NSEC` -compile-time option, `st_mtim.tv_nsec` and `st_ctim.tv_nsec` +is not stable on network filesystems. With 'core.useNanosec' +config setting, `st_mtim.tv_nsec` and `st_ctim.tv_nsec` members are also compared. On Linux, this is not enabled by default because in-core timestamps can have finer granularity than on-disk timestamps, resulting in meaningless changes when an @@ -49,9 +49,10 @@ of git://git.kernel.org/pub/scm/linux/kernel/git/tglx/history.git ([PATCH] Sync in core time granularity with filesystems, 2005-01-04). This patch is included in kernel 2.6.11 and newer, but only fixes the issue for file systems with exactly 1 ns or 1 s -resolution. Other file systems are still broken in current Linux -kernels (e.g. CEPH, CIFS, NTFS, UDF), see -https://lore.kernel.org/lkml/5577240D.7020309@gmail.com/ +resolution. As of kernel 4.3, other file systems (CEPH, CIFS, NTFS, UFS, FUSE) +were fixed; see https://public-inbox.org/git/5605D88A.20104%40gmail.com/. FAT +has been fixed since 2015. The usual suspects (ext2, ext4, XFS) are known to +work, too. Racy Git -------- diff --git a/Makefile b/Makefile index fac3e8879c8377..b7617bdb03fa98 100644 --- a/Makefile +++ b/Makefile @@ -110,6 +110,9 @@ include shared.mak # Define USE_HOMEBREW_LIBICONV to link against libiconv installed by # Homebrew, if present. # +# Define DARWIN_REGEXEC if regexec() in your platform regex library +# leaks when fed an invalid UTF-8 sequence. +# # Define NO_APPLE_COMMON_CRYPTO if you are building on Darwin/Mac OS X # and do not want to use Apple's CommonCrypto library. This allows you # to provide your own OpenSSL library, for example from MacPorts. @@ -197,18 +200,11 @@ include shared.mak # Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback, # as the compiler can crash (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299) # -# Define USE_NSEC below if you want git to care about sub-second file mtimes -# and ctimes. Note that you need recent glibc (at least 2.2.4) for this. On -# Linux, kernel 2.6.11 or newer is required for reliable sub-second file times -# on file systems with exactly 1 ns or 1 s resolution. If you intend to use Git -# on other file systems (e.g. CEPH, CIFS, NTFS, UDF), don't enable USE_NSEC. See -# Documentation/technical/racy-git.adoc for details. -# # Define USE_ST_TIMESPEC if your "struct stat" uses "st_ctimespec" instead of # "st_ctim" # # Define NO_NSEC if your "struct stat" does not have "st_ctim.tv_nsec" -# available. This automatically turns USE_NSEC off. +# available. # # Define USE_STDEV below if you want git to care about the underlying device # change being considered an inode change from the update-index perspective. @@ -819,6 +815,7 @@ TEST_BUILTINS_OBJS += test-csprng.o TEST_BUILTINS_OBJS += test-date.o TEST_BUILTINS_OBJS += test-delete-gpgsig.o TEST_BUILTINS_OBJS += test-delta.o +TEST_BUILTINS_OBJS += test-diff-process-backend.o TEST_BUILTINS_OBJS += test-dir-iterator.o TEST_BUILTINS_OBJS += test-drop-caches.o TEST_BUILTINS_OBJS += test-dump-cache-tree.o @@ -1151,6 +1148,8 @@ LIB_OBJS += diff-delta.o LIB_OBJS += diff-merges.o LIB_OBJS += diff-lib.o LIB_OBJS += diff-no-index.o +LIB_OBJS += diff-process.o +LIB_OBJS += diff-provider.o LIB_OBJS += diff.o LIB_OBJS += diffcore-break.o LIB_OBJS += diffcore-delta.o @@ -1158,6 +1157,7 @@ LIB_OBJS += diffcore-order.o LIB_OBJS += diffcore-pickaxe.o LIB_OBJS += diffcore-rename.o LIB_OBJS += diffcore-rotate.o +LIB_OBJS += diff-hunks.o LIB_OBJS += dir-iterator.o LIB_OBJS += dir.o LIB_OBJS += editor.o @@ -1421,6 +1421,7 @@ BUILTIN_OBJS += builtin/credential.o BUILTIN_OBJS += builtin/describe.o BUILTIN_OBJS += builtin/diagnose.o BUILTIN_OBJS += builtin/diff-files.o +BUILTIN_OBJS += builtin/diff-hunks.o BUILTIN_OBJS += builtin/diff-index.o BUILTIN_OBJS += builtin/diff-pairs.o BUILTIN_OBJS += builtin/diff-tree.o @@ -1935,9 +1936,6 @@ endif ifdef NO_ST_BLOCKS_IN_STRUCT_STAT BASIC_CFLAGS += -DNO_ST_BLOCKS_IN_STRUCT_STAT endif -ifdef USE_NSEC - BASIC_CFLAGS += -DUSE_NSEC -endif ifdef USE_ST_TIMESPEC BASIC_CFLAGS += -DUSE_ST_TIMESPEC endif @@ -2045,6 +2043,10 @@ ifdef NO_PREAD COMPAT_CFLAGS += -DNO_PREAD COMPAT_OBJS += compat/pread.o endif +ifdef NO_WRITEV + COMPAT_CFLAGS += -DNO_WRITEV + COMPAT_OBJS += compat/writev.o +endif ifdef NO_FAST_WORKING_DIRECTORY BASIC_CFLAGS += -DNO_FAST_WORKING_DIRECTORY endif @@ -2278,6 +2280,10 @@ ifdef USE_ENHANCED_BASIC_REGULAR_EXPRESSIONS COMPAT_CFLAGS += -DUSE_ENHANCED_BASIC_REGULAR_EXPRESSIONS COMPAT_OBJS += compat/regcomp_enhanced.o endif +ifdef DARWIN_REGEXEC + COMPAT_OBJS += compat/darwin/regexec.o + BASIC_CFLAGS += -DDARWIN_REGEXEC +endif endif ifdef NATIVE_CRLF BASIC_CFLAGS += -DNATIVE_CRLF diff --git a/advice.c b/advice.c index 63bf8b0c5f0481..401d0473917cd9 100644 --- a/advice.c +++ b/advice.c @@ -58,6 +58,7 @@ static struct { [ADVICE_FETCH_SHOW_FORCED_UPDATES] = { "fetchShowForcedUpdates" }, [ADVICE_FORCE_DELETE_BRANCH] = { "forceDeleteBranch" }, [ADVICE_GRAFT_FILE_DEPRECATED] = { "graftFileDeprecated" }, + [ADVICE_HISTORY_UPDATE_REFS] = { "historyUpdateRefs" }, [ADVICE_IGNORED_HOOK] = { "ignoredHook" }, [ADVICE_IMPLICIT_IDENTITY] = { "implicitIdentity" }, [ADVICE_MERGE_CONFLICT] = { "mergeConflict" }, diff --git a/advice.h b/advice.h index 66f6cd6a772d8c..3f0b4f0485c7fe 100644 --- a/advice.h +++ b/advice.h @@ -25,6 +25,7 @@ enum advice_type { ADVICE_FETCH_SHOW_FORCED_UPDATES, ADVICE_FORCE_DELETE_BRANCH, ADVICE_GRAFT_FILE_DEPRECATED, + ADVICE_HISTORY_UPDATE_REFS, ADVICE_IGNORED_HOOK, ADVICE_IMPLICIT_IDENTITY, ADVICE_MERGE_CONFLICT, diff --git a/apply.c b/apply.c index f00b7ba4d3a7e6..1af394de1f9b36 100644 --- a/apply.c +++ b/apply.c @@ -1374,6 +1374,9 @@ int parse_git_diff_header(struct strbuf *root, * the default name from the header. */ patch->def_name = git_header_name(p_value, line, len); + if (patch->def_name && !*patch->def_name) + FREE_AND_NULL(patch->def_name); + if (patch->def_name && root->len) { char *s = xstrfmt("%s%s", root->buf, patch->def_name); free(patch->def_name); @@ -1644,15 +1647,27 @@ static int find_header(struct apply_state *state, * or mode change, so we handle that specially */ if (!memcmp("diff --git ", line, 11)) { - int git_hdr_len = parse_git_diff_header(&state->root, - state->patch_input_file, - &state->linenr, - state->p_value, line, len, - size, patch); - if (git_hdr_len < 0) + struct patch git_patch = { 0 }; + int git_linenr = state->linenr; + int git_hdr_len; + + git_patch.inaccurate_eof = patch->inaccurate_eof; + git_patch.recount = patch->recount; + git_hdr_len = parse_git_diff_header(&state->root, + state->patch_input_file, + &git_linenr, + state->p_value, line, len, + size, &git_patch); + if (git_hdr_len < 0) { + release_patch(&git_patch); return -128; - if (git_hdr_len <= len) + } + if (git_hdr_len <= len) { + release_patch(&git_patch); continue; + } + *patch = git_patch; + state->linenr = git_linenr; *hdrsize = git_hdr_len; return offset; } diff --git a/archive-tar.c b/archive-tar.c index 0fc70d13a8807e..a1c66024d4dff4 100644 --- a/archive-tar.c +++ b/archive-tar.c @@ -129,20 +129,20 @@ static void write_trailer(void) */ static int stream_blocked(struct repository *r, const struct object_id *oid) { - struct odb_read_stream *st; + struct odb_stream *st; char buf[BLOCKSIZE]; ssize_t readlen; - st = odb_read_stream_open(r->objects, oid, NULL); + st = odb_stream_from_object(r->objects, oid, NULL); if (!st) return error(_("cannot stream blob %s"), oid_to_hex(oid)); for (;;) { - readlen = odb_read_stream_read(st, buf, sizeof(buf)); + readlen = odb_stream_read(st, buf, sizeof(buf)); if (readlen <= 0) break; do_write_blocked(buf, readlen); } - odb_read_stream_close(st); + odb_stream_close(st); if (!readlen) finish_record(); return readlen; diff --git a/archive-zip.c b/archive-zip.c index 97ea8d60d6187b..6f73ca6d588e83 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -206,7 +206,7 @@ static void *zlib_deflate_raw(void *data, unsigned long size, unsigned long *compressed_size) { git_zstream stream; - unsigned long maxsize; + size_t maxsize; void *buffer; int result; @@ -309,7 +309,7 @@ static int write_zip_entry(struct archiver_args *args, enum zip_method method; unsigned char *out; void *deflated = NULL; - struct odb_read_stream *stream = NULL; + struct odb_stream *stream = NULL; unsigned long flags = 0; int is_binary = -1; const char *path_without_prefix = path + args->baselen; @@ -347,7 +347,7 @@ static int write_zip_entry(struct archiver_args *args, method = ZIP_METHOD_DEFLATE; if (!buffer) { - stream = odb_read_stream_open(args->repo->objects, oid, NULL); + stream = odb_stream_from_object(args->repo->objects, oid, NULL); if (!stream) return error(_("cannot stream blob %s"), oid_to_hex(oid)); @@ -428,7 +428,7 @@ static int write_zip_entry(struct archiver_args *args, ssize_t readlen; for (;;) { - readlen = odb_read_stream_read(stream, buf, sizeof(buf)); + readlen = odb_stream_read(stream, buf, sizeof(buf)); if (readlen <= 0) break; crc = crc32(crc, buf, readlen); @@ -438,7 +438,7 @@ static int write_zip_entry(struct archiver_args *args, buf, readlen); write_or_die(1, buf, readlen); } - odb_read_stream_close(stream); + odb_stream_close(stream); if (readlen) return readlen; @@ -461,7 +461,7 @@ static int write_zip_entry(struct archiver_args *args, zstream.avail_out = sizeof(compressed); for (;;) { - readlen = odb_read_stream_read(stream, buf, sizeof(buf)); + readlen = odb_stream_read(stream, buf, sizeof(buf)); if (readlen <= 0) break; crc = crc32(crc, buf, readlen); @@ -485,7 +485,7 @@ static int write_zip_entry(struct archiver_args *args, } } - odb_read_stream_close(stream); + odb_stream_close(stream); if (readlen) return readlen; diff --git a/bisect.c b/bisect.c index 94c7028d2a746a..9cbb3dc677cc47 100644 --- a/bisect.c +++ b/bisect.c @@ -488,6 +488,7 @@ static GIT_PATH_FUNC(git_path_bisect_start, "BISECT_START") static GIT_PATH_FUNC(git_path_bisect_log, "BISECT_LOG") static GIT_PATH_FUNC(git_path_bisect_terms, "BISECT_TERMS") static GIT_PATH_FUNC(git_path_bisect_first_parent, "BISECT_FIRST_PARENT") +static GIT_PATH_FUNC(git_path_bisect_reset_when_found, "BISECT_RESET_WHEN_FOUND") static void read_bisect_paths(struct strvec *array) { @@ -1019,10 +1020,12 @@ void read_bisect_terms(char **read_bad, char **read_good) die_errno(_("could not read file '%s'"), filename); } } else { - strbuf_getline_lf(&str, fp); + if (strbuf_getline_lf(&str, fp) == EOF) + die(_("could not read bad term from file '%s'"), filename); free(*read_bad); *read_bad = strbuf_detach(&str, NULL); - strbuf_getline_lf(&str, fp); + if (strbuf_getline_lf(&str, fp) == EOF) + die(_("could not read good term from file '%s'"), filename); free(*read_good); *read_good = strbuf_detach(&str, NULL); } @@ -1211,6 +1214,7 @@ int bisect_clean_state(void) unlink_or_warn(git_path_bisect_run()); unlink_or_warn(git_path_bisect_terms()); unlink_or_warn(git_path_bisect_first_parent()); + unlink_or_warn(git_path_bisect_reset_when_found()); /* * Cleanup BISECT_START last to support the --no-checkout option * introduced in the commit 4796e823a. diff --git a/blame.c b/blame.c index 126e2324162353..3189bcdabaa886 100644 --- a/blame.c +++ b/blame.c @@ -23,6 +23,8 @@ #include "commit-slab.h" #include "bloom.h" #include "commit-graph.h" +#include "diff-provider.h" +#include "userdiff.h" define_commit_slab(blame_suspects, struct blame_origin *); static struct blame_suspects blame_suspects; @@ -1936,6 +1938,46 @@ static int blame_chunk_cb(long start_a, long count_a, return 0; } +/* + * A hunk provider's key names the (old blob, new blob) pair and may only + * serve a diff whose result is determined by that pair and the xdiff + * settings. Textconv rewrites the buffers being diffed away from the + * blob contents the key names, so any origin whose path has a textconv + * driver must withhold the pair's identity. + */ +static int blame_textconv_active(struct blame_scoreboard *sb, + const char *path) +{ + struct userdiff_driver *drv; + + if (!sb->revs->diffopt.flags.allow_textconv) + return 0; + drv = userdiff_find_by_path(sb->repo->index, path); + return drv && drv->textconv; +} + +struct blame_diff_fill_data { + struct blame_scoreboard *sb; + struct blame_origin *parent, *target; + int ignore_diffs; +}; + +/* + * Content load for diff_provider_emit_hunks(): runs when the diff is + * computed. + */ +static int blame_diff_fill(void *data, mmfile_t *old_file, mmfile_t *new_file) +{ + struct blame_diff_fill_data *f = data; + + fill_origin_blob(&f->sb->revs->diffopt, f->parent, old_file, + &f->sb->num_read_blob, f->ignore_diffs); + fill_origin_blob(&f->sb->revs->diffopt, f->target, new_file, + &f->sb->num_read_blob, f->ignore_diffs); + f->sb->num_get_patch++; + return 0; +} + /* * We are looking at the origin 'target' and aiming to pass blame * for the lines it is suspected to its parent. Run diff to find @@ -1945,9 +1987,12 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, struct blame_origin *target, struct blame_origin *parent, int ignore_diffs) { - mmfile_t file_p, file_o; struct blame_chunk_cb_data d; struct blame_entry *newdest = NULL; + struct blame_diff_fill_data fill_data = { sb, parent, target, ignore_diffs }; + xpparam_t xpp = { .flags = sb->xdl_opts }; + struct diff_provider_request req = { .repo = sb->repo, .xpp = &xpp }; + int provider_usable; if (!target->suspects) return; /* nothing remains for this target */ @@ -1958,13 +2003,35 @@ static void pass_blame_to_parent(struct blame_scoreboard *sb, d.ignore_diffs = ignore_diffs; d.dstq = &newdest; d.srcq = &target->suspects; - fill_origin_blob(&sb->revs->diffopt, parent, &file_p, - &sb->num_read_blob, ignore_diffs); - fill_origin_blob(&sb->revs->diffopt, target, &file_o, - &sb->num_read_blob, ignore_diffs); - sb->num_get_patch++; + /* + * Offer the pair's identity only where blame's diff is the plain + * blob-pair diff the recording key describes; reverse blame, + * ignored revisions, and textconv paths withhold it and always + * compute. The working-tree/--contents pseudo-commit (marked by + * its null commit id) holds a blob that is not a stored object, + * so its pairs withhold identity too: no id may be sent that + * names bytes a process cannot look up. + */ + provider_usable = !sb->reverse && !ignore_diffs && + !is_null_oid(&target->commit->object.oid) && + !blame_textconv_active(sb, target->path) && + !blame_textconv_active(sb, parent->path); - if (diff_hunks(&file_p, &file_o, blame_chunk_cb, &d, sb->xdl_opts)) + /* + * Look up the driver by the parent (old) path, as builtin_diff() + * does with name_a, so a renamed file resolves to the same driver + * across diff and blame. A process that reports a pair + * equivalent emits no hunks, so blame passes the whole commit + * through and looks past it. + */ + if (provider_usable) { + req.old_oid = &parent->blob_oid; + req.new_oid = &target->blob_oid; + } + req.path = parent->path; + req.diffopt = &sb->revs->diffopt; + if (diff_provider_emit_hunks(&req, blame_diff_fill, &fill_data, + blame_chunk_cb, &d) == DIFF_PROVIDER_ERROR) die("unable to generate diff (%s -> %s)", oid_to_hex(&parent->commit->object.oid), oid_to_hex(&target->commit->object.oid)); diff --git a/bloom.c b/bloom.c index caf22f9831bb93..b96534e6e37fa9 100644 --- a/bloom.c +++ b/bloom.c @@ -607,6 +607,18 @@ int bloom_filter_contains_vec(const struct bloom_filter *filter, return ret; } +int bloom_filter_contains_any_vec(const struct bloom_filter *filter, + const struct bloom_keyvec *vec, + const struct bloom_filter_settings *settings) +{ + int ret = 0; + + for (size_t nr = 0; !ret && nr < vec->count; nr++) + ret = bloom_filter_contains(filter, &vec->key[nr], settings); + + return ret; +} + uint32_t test_bloom_murmur3_seeded(uint32_t seed, const char *data, size_t len, int version) { diff --git a/bloom.h b/bloom.h index 92ab2100d390e7..f508db23addf51 100644 --- a/bloom.h +++ b/bloom.h @@ -164,6 +164,17 @@ int bloom_filter_contains_vec(const struct bloom_filter *filter, const struct bloom_keyvec *v, const struct bloom_filter_settings *settings); +/* + * bloom_filter_contains_any_vec - Check if any key in a key vector is in the + * Bloom filter. + * + * Returns 1 if **any** key in the vector is present in the filter, 0 if none + * of them are. + */ +int bloom_filter_contains_any_vec(const struct bloom_filter *filter, + const struct bloom_keyvec *v, + const struct bloom_filter_settings *settings); + uint32_t test_bloom_murmur3_seeded(uint32_t seed, const char *data, size_t len, int version); diff --git a/branch.c b/branch.c index 22f4f46b96ed3a..ec714ed0a7fa4d 100644 --- a/branch.c +++ b/branch.c @@ -466,9 +466,24 @@ static void prepare_checked_out_branches(void) &update_refs)) { struct string_list_item *item; for_each_string_list_item(item, &update_refs) { + char *resolved_ref; + int flags = 0; + register_checked_out_branch("", item->string, wt->path, BRANCH_CHECKOUT_KIND_UPDATE_REF); + + resolved_ref = refs_resolve_refdup( + get_main_ref_store(the_repository), + item->string, RESOLVE_REF_READING, + NULL, &flags); + if (resolved_ref && (flags & REF_ISSYMREF)) { + char *old = strmap_put( + ¤t_checked_out_branches, + resolved_ref, xstrdup(wt->path)); + free(old); + } + free(resolved_ref); } string_list_clear(&update_refs, 1); } diff --git a/builtin.h b/builtin.h index 4e47a4ebd30ba3..7e64da9f433836 100644 --- a/builtin.h +++ b/builtin.h @@ -175,6 +175,7 @@ int cmd_credential_store(int argc, const char **argv, const char *prefix, struct int cmd_describe(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diagnose(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff_files(int argc, const char **argv, const char *prefix, struct repository *repo); +int cmd_diff_hunks(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff_index(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_diff_pairs(int argc, const char **argv, const char *prefix, struct repository *repo); diff --git a/builtin/add.c b/builtin/add.c index eab8f03cad31d6..f565d4d0bc46d2 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -13,7 +13,6 @@ #include "dir.h" #include "gettext.h" #include "pathspec.h" -#include "run-command.h" #include "object-file.h" #include "odb.h" #include "odb/transaction.h" @@ -23,10 +22,10 @@ #include "diff.h" #include "read-cache.h" #include "revision.h" -#include "strvec.h" #include "submodule.h" #include "add-interactive.h" #include "merge-ll.h" +#include "apply.h" static const char * const builtin_add_usage[] = { N_("git add [] [--] ..."), @@ -189,7 +188,8 @@ static int edit_patch(struct repository *repo, const char *prefix) { char *file = repo_git_path(repo, "ADD_EDIT.patch"); - struct child_process child = CHILD_PROCESS_INIT; + struct apply_state state; + const char *apply_argv[2]; struct rev_info rev; int out; struct stat st; @@ -219,11 +219,16 @@ static int edit_patch(struct repository *repo, if (!st.st_size) die(_("empty patch. aborted")); - child.git_cmd = 1; - strvec_pushl(&child.args, "apply", "--recount", "--cached", file, - NULL); - if (run_command(&child)) + apply_argv[0] = file; + apply_argv[1] = NULL; + if (init_apply_state(&state, repo, NULL)) + die(_("could not initialize apply state")); + state.cached = 1; + if (check_apply_state(&state, 0)) + die(_("could not check apply state")); + if (apply_all_patches(&state, 1, apply_argv, APPLY_OPT_RECOUNT)) die(_("could not apply '%s'"), file); + clear_apply_state(&state); unlink(file); free(file); @@ -466,7 +471,7 @@ int cmd_add(int argc, char *seen = NULL; char *ps_matched = NULL; struct lock_file lock_file = LOCK_INIT; - struct odb_transaction *transaction; + struct odb_transaction *transaction = NULL; repo_config(repo, add_config, NULL); @@ -680,7 +685,7 @@ int cmd_add(int argc, if (chmod_arg && pathspec.nr) exit_status |= chmod_pathspec(repo, &pathspec, chmod_arg[0], show_only); - odb_transaction_commit(transaction); + odb_transaction_commit_and_finalize_or_die(transaction); finish: if (write_locked_index(repo->index, &lock_file, diff --git a/builtin/bisect.c b/builtin/bisect.c index 3264e2da542698..1cfb8a794b36ec 100644 --- a/builtin/bisect.c +++ b/builtin/bisect.c @@ -24,11 +24,12 @@ static GIT_PATH_FUNC(git_path_bisect_start, "BISECT_START") static GIT_PATH_FUNC(git_path_bisect_log, "BISECT_LOG") static GIT_PATH_FUNC(git_path_bisect_names, "BISECT_NAMES") static GIT_PATH_FUNC(git_path_bisect_first_parent, "BISECT_FIRST_PARENT") +static GIT_PATH_FUNC(git_path_bisect_reset_when_found, "BISECT_RESET_WHEN_FOUND") static GIT_PATH_FUNC(git_path_bisect_run, "BISECT_RUN") #define BUILTIN_GIT_BISECT_START_USAGE \ N_("git bisect start [--term-(bad|new)= --term-(good|old)=]\n" \ - " [--no-checkout] [--first-parent] [ [...]] [--] [...]") + " [--no-checkout] [--first-parent] [--reset-when-found[=]] [ [...]] [--] [...]") #define BUILTIN_GIT_BISECT_BAD_USAGE \ N_("git bisect (bad|new|) []") #define BUILTIN_GIT_BISECT_GOOD_USAGE \ @@ -48,7 +49,7 @@ static GIT_PATH_FUNC(git_path_bisect_run, "BISECT_RUN") #define BUILTIN_GIT_BISECT_LOG_USAGE \ "git bisect log" #define BUILTIN_GIT_BISECT_RUN_USAGE \ - N_("git bisect run [...]") + N_("git bisect run [--reset-when-found[=]] [...]") #define BUILTIN_GIT_BISECT_HELP_USAGE \ "git bisect help" @@ -68,6 +69,12 @@ static const char * const git_bisect_usage[] = { NULL }; +enum reset_when_found_mode { + RESET_WHEN_FOUND_NONE, + RESET_WHEN_FOUND_TO_ORIGINAL, + RESET_WHEN_FOUND_TO_FOUND, +}; + struct add_bisect_ref_data { struct rev_info *revs; unsigned int object_flags; @@ -234,7 +241,7 @@ static int write_terms(const char *bad, const char *good) return res; } -static int bisect_reset(const char *commit) +static int bisect_reset(const char *commit, bool quiet) { struct strbuf branch = STRBUF_INIT; @@ -255,8 +262,10 @@ static int bisect_reset(const char *commit) struct child_process cmd = CHILD_PROCESS_INIT; cmd.git_cmd = 1; - strvec_pushl(&cmd.args, "checkout", "--ignore-other-worktrees", - branch.buf, "--", NULL); + strvec_pushl(&cmd.args, "checkout", "--ignore-other-worktrees", NULL); + if (quiet) + strvec_push(&cmd.args, "--quiet"); + strvec_pushl(&cmd.args, branch.buf, "--", NULL); if (run_command(&cmd)) { error(_("could not check out original" " HEAD '%s'. Try 'git bisect" @@ -267,7 +276,79 @@ static int bisect_reset(const char *commit) } strbuf_release(&branch); - return bisect_clean_state(); + return 0; +} + +static int parse_reset_when_found(const char *value, + enum reset_when_found_mode *mode) +{ + if (!strcmp(value, "original")) + *mode = RESET_WHEN_FOUND_TO_ORIGINAL; + else if (!strcmp(value, "found")) + *mode = RESET_WHEN_FOUND_TO_FOUND; + else + return error(_("invalid value for '--reset-when-found': '%s'"), + value); + + return 0; +} + +static const char *reset_when_found_mode_name(enum reset_when_found_mode mode) +{ + switch (mode) { + case RESET_WHEN_FOUND_TO_ORIGINAL: + return "original"; + case RESET_WHEN_FOUND_TO_FOUND: + return "found"; + case RESET_WHEN_FOUND_NONE: + BUG("no name for unset reset-when-found mode"); + } + BUG("unknown reset-when-found mode %d", mode); +} + +static int read_reset_when_found(enum reset_when_found_mode *mode) +{ + struct strbuf value = STRBUF_INIT; + int res = 0; + + *mode = RESET_WHEN_FOUND_NONE; + if (is_empty_or_missing_file(git_path_bisect_reset_when_found())) + return 0; + + if (strbuf_read_file(&value, git_path_bisect_reset_when_found(), 0) < 0) { + res = error_errno(_("could not read '%s'"), + git_path_bisect_reset_when_found()); + goto out; + } + strbuf_trim(&value); + if (parse_reset_when_found(value.buf, mode)) + res = -1; + +out: + strbuf_release(&value); + return res; +} + +static int bisect_reset_when_found(enum reset_when_found_mode mode) +{ + struct bisect_terms terms = { 0 }; + char *commit = NULL; + int res; + + if (mode == RESET_WHEN_FOUND_TO_FOUND) { + read_bisect_terms(&terms.term_bad, &terms.term_good); + commit = xstrfmt("refs/bisect/%s", terms.term_bad); + } else if (mode == RESET_WHEN_FOUND_NONE) { + BUG("automatic reset requested without a reset mode"); + } + + res = bisect_reset(commit, true); + if (!res) + res = bisect_clean_state(); + + free(commit); + free_terms(&terms); + return res; } static void log_commit(FILE *fp, @@ -485,7 +566,7 @@ static int bisect_next_check(const struct bisect_terms *terms, return decide_next(terms, current_term, !state.nr_good, !state.nr_bad); } -static int get_terms(struct bisect_terms *terms) +static int get_terms(struct bisect_terms *terms, int file_missing_is_ok) { struct strbuf str = STRBUF_INIT; FILE *fp = NULL; @@ -493,14 +574,21 @@ static int get_terms(struct bisect_terms *terms) fp = fopen(git_path_bisect_terms(), "r"); if (!fp) { - res = -1; + res = file_missing_is_ok ? 0 : -1; goto finish; } free_terms(terms); - strbuf_getline_lf(&str, fp); + if (strbuf_getline_lf(&str, fp) == EOF) { + res = -1; + goto finish; + } terms->term_bad = strbuf_detach(&str, NULL); - strbuf_getline_lf(&str, fp); + if (strbuf_getline_lf(&str, fp) == EOF) { + res = -1; + FREE_AND_NULL(terms->term_bad); + goto finish; + } terms->term_good = strbuf_detach(&str, NULL); finish: @@ -512,7 +600,7 @@ static int get_terms(struct bisect_terms *terms) static int bisect_terms(struct bisect_terms *terms, const char *option) { - if (get_terms(terms)) + if (get_terms(terms, 0)) return error(_("no terms defined")); if (!option) { @@ -680,7 +768,8 @@ static int bisect_successful(struct bisect_terms *terms) return res; } -static enum bisect_error bisect_next(struct bisect_terms *terms, const char *prefix) +static enum bisect_error bisect_next(struct bisect_terms *terms, + const char *prefix) { enum bisect_error res; @@ -703,7 +792,8 @@ static enum bisect_error bisect_next(struct bisect_terms *terms, const char *pre return res; } -static enum bisect_error bisect_auto_next(struct bisect_terms *terms, const char *prefix) +static enum bisect_error bisect_auto_next(struct bisect_terms *terms, + const char *prefix) { if (bisect_next_check(terms, NULL)) { bisect_print_status(terms); @@ -727,6 +817,7 @@ static enum bisect_error bisect_start(struct bisect_terms *terms, int argc, struct strbuf bisect_names = STRBUF_INIT; struct object_id head_oid; struct object_id oid; + enum reset_when_found_mode reset_when_found = RESET_WHEN_FOUND_NONE; const char *head; if (is_bare_repository(the_repository)) @@ -750,6 +841,13 @@ static enum bisect_error bisect_start(struct bisect_terms *terms, int argc, no_checkout = 1; } else if (!strcmp(arg, "--first-parent")) { first_parent_only = 1; + } else if (!strcmp(arg, "--reset-when-found")) { + reset_when_found = RESET_WHEN_FOUND_TO_ORIGINAL; + } else if (skip_prefix(arg, "--reset-when-found=", &arg)) { + if (parse_reset_when_found(arg, &reset_when_found)) { + res = BISECT_FAILED; + goto finish; + } } else if (!strcmp(arg, "--term-good") || !strcmp(arg, "--term-old")) { i++; @@ -787,6 +885,11 @@ static enum bisect_error bisect_start(struct bisect_terms *terms, int argc, break; } } + if (reset_when_found != RESET_WHEN_FOUND_NONE && no_checkout) { + res = error(_("options '%s' and '%s' cannot be used together"), + "--reset-when-found", "--no-checkout"); + goto finish; + } pathspec_pos = i; /* @@ -866,6 +969,10 @@ static enum bisect_error bisect_start(struct bisect_terms *terms, int argc, if (first_parent_only) write_file(git_path_bisect_first_parent(), "\n"); + if (reset_when_found != RESET_WHEN_FOUND_NONE) + write_file(git_path_bisect_reset_when_found(), "%s\n", + reset_when_found_mode_name(reset_when_found)); + if (no_checkout) { if (repo_get_oid(the_repository, start_head.buf, &oid) < 0) { res = error(_("invalid ref: '%s'"), start_head.buf); @@ -1057,7 +1164,8 @@ static int process_replay_line(struct bisect_terms *terms, struct strbuf *line) rev = word_end + strspn(word_end, " \t"); *word_end = '\0'; /* NUL-terminate the word */ - get_terms(terms); + if (get_terms(terms, 1)) + return error(_("no terms defined")); if (check_and_set_terms(terms, p)) return -1; @@ -1096,7 +1204,7 @@ static enum bisect_error bisect_replay(struct bisect_terms *terms, const char *f if (is_empty_or_missing_file(filename)) return error(_("cannot read file '%s' for replaying"), filename); - if (bisect_reset(NULL)) + if (bisect_clean_state()) return BISECT_FAILED; fp = fopen(filename, "r"); @@ -1244,13 +1352,36 @@ static int bisect_run(struct bisect_terms *terms, int argc, const char **argv) { int res = BISECT_OK; struct strbuf command = STRBUF_INIT; + const char *reset_when_found_arg; const char *new_state; int temporary_stdout_fd, saved_stdout; int is_first_run = 1; + enum reset_when_found_mode reset_when_found = RESET_WHEN_FOUND_NONE; if (bisect_next_check(terms, NULL)) return BISECT_FAILED; + if (argc && !strcmp(argv[0], "--reset-when-found")) { + reset_when_found = RESET_WHEN_FOUND_TO_ORIGINAL; + } else if (argc && skip_prefix(argv[0], "--reset-when-found=", + &reset_when_found_arg)) { + if (parse_reset_when_found(reset_when_found_arg, + &reset_when_found)) + return BISECT_FAILED; + } + + if (reset_when_found != RESET_WHEN_FOUND_NONE && + refs_ref_exists(get_main_ref_store(the_repository), "BISECT_HEAD")) + return error(_("options '%s' and '%s' cannot be used together"), + "--reset-when-found", "--no-checkout"); + + if (reset_when_found != RESET_WHEN_FOUND_NONE) { + write_file(git_path_bisect_reset_when_found(), "%s\n", + reset_when_found_mode_name(reset_when_found)); + argc--; + argv++; + } + if (!argc) { error(_("bisect run failed: no command provided.")); return BISECT_FAILED; @@ -1307,7 +1438,14 @@ static int bisect_run(struct bisect_terms *terms, int argc, const char **argv) fflush(stdout); saved_stdout = dup(1); - dup2(temporary_stdout_fd, 1); + if (saved_stdout < 0 || + dup2(temporary_stdout_fd, 1) < 0) { + res = error_errno(_("could not duplicate stdout")); + if (saved_stdout >= 0) + close(saved_stdout); + close(temporary_stdout_fd); + break; + } res = bisect_state(terms, 1, &new_state); @@ -1325,7 +1463,6 @@ static int bisect_run(struct bisect_terms *terms, int argc, const char **argv) res = BISECT_OK; } else if (res == BISECT_INTERNAL_SUCCESS_1ST_BAD_FOUND) { printf(_("bisect found first '%s' commit\n"), terms->term_bad); - res = BISECT_OK; } else if (res) { error(_("bisect run failed: 'git bisect %s'" " exited with error code %d"), new_state, res); @@ -1342,10 +1479,15 @@ static int bisect_run(struct bisect_terms *terms, int argc, const char **argv) static int cmd_bisect__reset(int argc, const char **argv, const char *prefix UNUSED, struct repository *repo UNUSED) { + int res; + if (argc > 1) return error(_("'%s' requires either no argument or a commit"), "git bisect reset"); - return bisect_reset(argc ? argv[0] : NULL); + res = bisect_reset(argc ? argv[0] : NULL, false); + if (res) + return res; + return bisect_clean_state(); } static int cmd_bisect__terms(int argc, const char **argv, const char *prefix UNUSED, @@ -1383,7 +1525,8 @@ static int cmd_bisect__next(int argc, const char **argv UNUSED, const char *pref if (argc) return error(_("'%s' requires 0 arguments"), "git bisect next"); - get_terms(&terms); + if (get_terms(&terms, 1)) + return error(_("no terms defined")); res = bisect_next(&terms, prefix); free_terms(&terms); return res; @@ -1417,7 +1560,8 @@ static int cmd_bisect__skip(int argc, const char **argv, const char *prefix UNUS struct bisect_terms terms = { 0 }; set_terms(&terms, "bad", "good"); - get_terms(&terms); + if (get_terms(&terms, 1)) + return error(_("no terms defined")); res = bisect_skip(&terms, argc, argv); free_terms(&terms); return res; @@ -1429,7 +1573,8 @@ static int cmd_bisect__visualize(int argc, const char **argv, const char *prefix int res; struct bisect_terms terms = { 0 }; - get_terms(&terms); + if (get_terms(&terms, 1)) + return error(_("no terms defined")); res = bisect_visualize(&terms, argc, argv); free_terms(&terms); return res; @@ -1443,7 +1588,8 @@ static int cmd_bisect__run(int argc, const char **argv, const char *prefix UNUSE if (!argc) return error(_("'%s' failed: no command provided."), "git bisect run"); - get_terms(&terms); + if (get_terms(&terms, 1)) + return error(_("no terms defined")); res = bisect_run(&terms, argc, argv); free_terms(&terms); return res; @@ -1482,12 +1628,14 @@ int cmd_bisect(int argc, usage_with_options(git_bisect_usage, options); set_terms(&terms, "bad", "good"); - get_terms(&terms); + if (get_terms(&terms, 1)) + return error(_("no terms defined")); if (check_and_set_terms(&terms, argv[0]) || !one_of(argv[0], terms.term_good, terms.term_bad, NULL)) usage_msg_optf(_("unknown command: '%s'"), git_bisect_usage, options, argv[0]); - res = bisect_state(&terms, argc, argv); + else + res = bisect_state(&terms, argc, argv); free_terms(&terms); } else { argc--; @@ -1495,5 +1643,15 @@ int cmd_bisect(int argc, res = fn(argc, argv, prefix, repo); } + if (res == BISECT_INTERNAL_SUCCESS_1ST_BAD_FOUND) { + enum reset_when_found_mode mode; + + if (read_reset_when_found(&mode)) + res = BISECT_FAILED; + else if (mode != RESET_WHEN_FOUND_NONE && + bisect_reset_when_found(mode)) + res = BISECT_FAILED; + } + return is_bisect_success(res) ? 0 : -res; } diff --git a/builtin/blame.c b/builtin/blame.c index 48d5251c6df700..13e3ff9e36a50e 100644 --- a/builtin/blame.c +++ b/builtin/blame.c @@ -15,6 +15,7 @@ #include "hex.h" #include "commit.h" #include "diff.h" +#include "diff-hunks.h" #include "revision.h" #include "quote.h" #include "string-list.h" @@ -1024,6 +1025,7 @@ int cmd_blame(int argc, repo_init_revisions(the_repository, &revs, NULL); revs.date_mode = blame_date_mode; revs.diffopt.flags.allow_textconv = 1; + revs.diffopt.flags.allow_diff_process = 1; revs.diffopt.flags.follow_renames = 1; save_commit_buffer = 0; @@ -1060,7 +1062,7 @@ int cmd_blame(int argc, parse_done: revision_opts_finish(&revs); no_whole_file_rename = !revs.diffopt.flags.follow_renames; - xdl_opts |= revs.diffopt.xdl_opts & XDF_INDENT_HEURISTIC; + xdl_opts |= revs.diffopt.xdl_opts & DIFF_HUNKS_DEFAULT_XDL_OPTS; revs.diffopt.flags.follow_renames = 0; argc = parse_options_end(&ctx); @@ -1315,9 +1317,14 @@ int cmd_blame(int argc, output(&sb, output_option); if (show_stats) { + unsigned long hunk_hits, hunk_misses; + + diff_hunks_read_stats(sb.repo, &hunk_hits, &hunk_misses); printf("num read blob: %d\n", sb.num_read_blob); printf("num get patch: %d\n", sb.num_get_patch); printf("num commits: %d\n", sb.num_commits); + printf("num precomputed hits: %lu\n", hunk_hits); + printf("num precomputed misses: %lu\n", hunk_misses); } cleanup: diff --git a/builtin/branch.c b/builtin/branch.c index 357209748d4fe4..a613148fc72786 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -21,6 +21,7 @@ #include "branch.h" #include "path.h" #include "string-list.h" +#include "strmap.h" #include "column.h" #include "utf8.h" #include "ref-filter.h" @@ -30,7 +31,7 @@ #include "commit-reach.h" static const char * const builtin_branch_usage[] = { - N_("git branch [] [-r | -a] [--merged] [--no-merged]"), + N_("git branch [] [-r | -a] [--merged] [--no-merged] [(--forked )...]"), N_("git branch [] [-f] [--recurse-submodules] []"), N_("git branch [] [-l] [...]"), N_("git branch [] [-r] (-d | -D) ..."), @@ -38,6 +39,8 @@ static const char * const builtin_branch_usage[] = { N_("git branch [] (-c | -C) [] "), N_("git branch [] [-r | -a] [--points-at]"), N_("git branch [] [-r | -a] [--format]"), + N_("git branch [] (--delete-merged )... " + "[...]"), NULL }; @@ -168,10 +171,13 @@ static int branch_merged(int kind, const char *name, * upstream, if any, otherwise with HEAD", we should just * return the result of the repo_in_merge_bases() above without * any of the following code, but during the transition period, - * a gentle reminder is in order. + * a gentle reminder is in order. Callers that opt out of the + * HEAD fallback by passing head_rev=NULL are not interested in + * the reminder either: they have already established that the + * branch has an upstream, so HEAD is irrelevant to the decision. */ - if (head_rev != reference_rev) { - int expect = head_rev ? repo_in_merge_bases(the_repository, rev, head_rev) : 0; + if (head_rev && head_rev != reference_rev) { + int expect = repo_in_merge_bases(the_repository, rev, head_rev); if (expect < 0) exit(128); if (expect == merged) @@ -189,20 +195,32 @@ static int branch_merged(int kind, const char *name, return merged; } +enum delete_branch_flags { + DELETE_BRANCH_FORCE = (1 << 0), + DELETE_BRANCH_QUIET = (1 << 1), + DELETE_BRANCH_SKIP_UNMERGED = (1 << 2), + DELETE_BRANCH_NO_HEAD_FALLBACK = (1 << 3), + DELETE_BRANCH_DRY_RUN = (1 << 4), +}; + static int check_branch_commit(const char *branchname, const char *refname, const struct object_id *oid, struct commit *head_rev, - int kinds, int force) + int kinds, unsigned int flags) { struct commit *rev = lookup_commit_reference(the_repository, oid); - if (!force && !rev) { + if (!(flags & DELETE_BRANCH_FORCE) && !rev) { error(_("couldn't look up commit object for '%s'"), refname); return -1; } - if (!force && !branch_merged(kinds, branchname, rev, head_rev)) { - error(_("the branch '%s' is not fully merged"), branchname); - advise_if_enabled(ADVICE_FORCE_DELETE_BRANCH, - _("If you are sure you want to delete it, " - "run 'git branch -D %s'"), branchname); + if (!(flags & DELETE_BRANCH_FORCE) && + !branch_merged(kinds, branchname, rev, head_rev)) { + if (!(flags & DELETE_BRANCH_SKIP_UNMERGED)) { + error(_("the branch '%s' is not fully merged"), + branchname); + advise_if_enabled(ADVICE_FORCE_DELETE_BRANCH, + _("If you are sure you want to delete it, " + "run 'git branch -D %s'"), branchname); + } return -1; } return 0; @@ -217,8 +235,8 @@ static void delete_branch_config(const char *branchname) strbuf_release(&buf); } -static int delete_branches(int argc, const char **argv, int force, int kinds, - int quiet) +static int delete_branches(int argc, const char **argv, int kinds, + unsigned int flags) { struct commit *head_rev = NULL; struct object_id oid; @@ -241,7 +259,7 @@ static int delete_branches(int argc, const char **argv, int force, int kinds, remote_branch = 1; allowed_interpret = INTERPRET_BRANCH_REMOTE; - force = 1; + flags |= DELETE_BRANCH_FORCE; break; case FILTER_REFS_BRANCHES: fmt = "refs/heads/%s"; @@ -252,12 +270,13 @@ static int delete_branches(int argc, const char **argv, int force, int kinds, } branch_name_pos = strcspn(fmt, "%"); - if (!force) + if (!(flags & DELETE_BRANCH_FORCE) && + !(flags & DELETE_BRANCH_NO_HEAD_FALLBACK)) head_rev = lookup_commit_reference(the_repository, &head_oid); for (i = 0; i < argc; i++, strbuf_reset(&bname)) { char *target = NULL; - int flags = 0; + int ref_flags = 0; copy_branchname(the_repository, &bname, argv[i], allowed_interpret); @@ -287,7 +306,7 @@ static int delete_branches(int argc, const char **argv, int force, int kinds, RESOLVE_REF_READING | RESOLVE_REF_NO_RECURSE | RESOLVE_REF_ALLOW_BAD_NAME, - &oid, &flags); + &oid, &ref_flags); if (!target) { if (remote_branch) { error(_("remote-tracking branch '%s' not found"), bname.buf); @@ -299,7 +318,7 @@ static int delete_branches(int argc, const char **argv, int force, int kinds, | RESOLVE_REF_NO_RECURSE | RESOLVE_REF_ALLOW_BAD_NAME, &oid, - &flags); + &ref_flags); FREE_AND_NULL(virtual_name); if (virtual_target) @@ -314,31 +333,39 @@ static int delete_branches(int argc, const char **argv, int force, int kinds, continue; } - if (!(flags & (REF_ISSYMREF|REF_ISBROKEN)) && + if (!(ref_flags & (REF_ISSYMREF|REF_ISBROKEN)) && check_branch_commit(bname.buf, name, &oid, head_rev, kinds, - force)) { - ret = 1; + flags)) { + if (!(flags & DELETE_BRANCH_SKIP_UNMERGED)) + ret = 1; goto next; } item = string_list_append(&refs_to_delete, name); - item->util = xstrdup((flags & REF_ISBROKEN) ? "broken" - : (flags & REF_ISSYMREF) ? target + item->util = xstrdup((ref_flags & REF_ISBROKEN) ? "broken" + : (ref_flags & REF_ISSYMREF) ? target : repo_find_unique_abbrev(the_repository, &oid, DEFAULT_ABBREV)); next: free(target); } - if (refs_delete_refs(get_main_ref_store(the_repository), NULL, &refs_to_delete, REF_NO_DEREF)) + if (!(flags & DELETE_BRANCH_DRY_RUN) && + refs_delete_refs(get_main_ref_store(the_repository), NULL, &refs_to_delete, REF_NO_DEREF)) ret = 1; for_each_string_list_item(item, &refs_to_delete) { char *describe_ref = item->util; char *name = item->string; - if (!refs_ref_exists(get_main_ref_store(the_repository), name)) { + if (flags & DELETE_BRANCH_DRY_RUN) { + if (!(flags & DELETE_BRANCH_QUIET)) + printf(remote_branch + ? _("Would delete remote-tracking branch %s (was %s).\n") + : _("Would delete branch %s (was %s).\n"), + name + branch_name_pos, describe_ref); + } else if (!refs_ref_exists(get_main_ref_store(the_repository), name)) { char *refname = name + branch_name_pos; - if (!quiet) + if (!(flags & DELETE_BRANCH_QUIET)) printf(remote_branch ? _("Deleted remote-tracking branch %s (was %s).\n") : _("Deleted branch %s (was %s).\n"), @@ -681,6 +708,208 @@ static void copy_or_rename_branch(const char *oldname, const char *newname, int free_worktrees(worktrees); } +static int parse_opt_forked(const struct option *opt, const char *arg, int unset) +{ + struct ref_filter *filter = opt->value; + + BUG_ON_OPT_NEG(unset); + if (ref_filter_forked_add(filter, arg) < 0) + die(_("'%s' is not a valid branch or pattern"), arg); + return 0; +} + +struct stacked_branch_data { + struct strset *deletable_branch_names; + struct strset *protected_branch_names; +}; + +static int collect_stacked_branch_base(const struct reference *ref, + void *cb_data) +{ + struct stacked_branch_data *data = cb_data; + const char *branch_name; + struct branch *branch; + const char *upstream_refname; + const char *upstream_branch_name; + + if (!skip_prefix(ref->name, "refs/heads/", &branch_name)) + BUG("expected local branch ref, got '%s'", ref->name); + if (strset_contains(data->deletable_branch_names, branch_name)) + return 0; + + branch = branch_get(branch_name); + upstream_refname = branch_get_upstream(branch, NULL); + if (!upstream_refname || + !skip_prefix(upstream_refname, "refs/heads/", + &upstream_branch_name) || + !strset_contains(data->deletable_branch_names, + upstream_branch_name)) + return 0; + + strset_add(data->protected_branch_names, upstream_branch_name); + return 0; +} + +static void protect_stacked_branch_bases(struct ref_store *refs, + struct strset *deletable_branch_names, + struct strset *protected_branch_names) +{ + struct stacked_branch_data data = { + .deletable_branch_names = deletable_branch_names, + .protected_branch_names = protected_branch_names, + }; + struct refs_for_each_ref_options opts = { + .prefix = "refs/heads/", + }; + struct hashmap_iter iter; + struct strmap_entry *entry; + + refs_for_each_ref_ext(refs, collect_stacked_branch_base, &data, &opts); + + strset_for_each_entry(protected_branch_names, &iter, entry) + strset_remove(deletable_branch_names, entry->key); +} + +static void clear_deleted_upstreams(struct strset *protected_branch_names, + struct strset *deletable_branch_names) +{ + struct strbuf key = STRBUF_INIT; + struct hashmap_iter iter; + struct strmap_entry *entry; + + strset_for_each_entry(protected_branch_names, &iter, entry) { + struct branch *branch = branch_get(entry->key); + const char *upstream_refname = branch_get_upstream(branch, NULL); + const char *upstream_branch_name; + + if (!upstream_refname || + !skip_prefix(upstream_refname, "refs/heads/", + &upstream_branch_name) || + !strset_contains(deletable_branch_names, + upstream_branch_name)) + continue; + + strbuf_addf(&key, "branch.%s.merge", branch->name); + repo_config_set_gently(the_repository, key.buf, NULL); + strbuf_reset(&key); + strbuf_addf(&key, "branch.%s.remote", branch->name); + repo_config_set_gently(the_repository, key.buf, NULL); + strbuf_reset(&key); + } + + strbuf_release(&key); +} + +static int branch_pushes_to_upstream(struct branch *branch, + const char *upstream) +{ + struct remote *remote = remote_get(remote_for_branch(branch, NULL)); + char *push_refname = NULL; + char *tracking = NULL; + int ret = 0; + + if (!remote) + return 0; + if (remote->push.nr) + push_refname = apply_refspecs(&remote->push, branch->refname); + else + push_refname = xstrdup(branch->refname); + if (push_refname) + tracking = apply_refspecs(&remote->fetch, push_refname); + if (tracking && !strcmp(tracking, upstream)) + ret = 1; + + free(push_refname); + free(tracking); + return ret; +} + +static int delete_merged_branches(const struct strvec *upstreams, + const char **argv, unsigned int flags) +{ + struct ref_store *refs = get_main_ref_store(the_repository); + struct ref_filter filter = REF_FILTER_INIT; + struct ref_array candidates = { 0 }; + struct strset deletable_branch_names = STRSET_INIT; + struct strset protected_branch_names = STRSET_INIT; + struct strvec branches_to_delete = STRVEC_INIT; + struct strbuf key = STRBUF_INIT; + struct hashmap_iter iter; + struct strmap_entry *entry; + int ret = 0; + + for (size_t i = 0; i < upstreams->nr; i++) + if (ref_filter_forked_add(&filter, upstreams->v[i]) < 0) + die(_("'%s' is not a valid branch or pattern"), + upstreams->v[i]); + + filter.kind = FILTER_REFS_BRANCHES; + filter.name_patterns = argv; + filter_refs(&candidates, &filter, filter.kind); + + for (int i = 0; i < candidates.nr; i++) { + const char *branch_refname = candidates.items[i]->refname; + const char *branch_name; + struct branch *branch; + const char *upstream_refname; + int opt_out; + + if (!skip_prefix(branch_refname, "refs/heads/", &branch_name)) + BUG("filter returned non-branch ref '%s'", branch_refname); + if (branch_checked_out(branch_refname)) + continue; + + branch = branch_get(branch_name); + upstream_refname = branch_get_upstream(branch, NULL); + if (!upstream_refname || !refs_ref_exists(refs, upstream_refname)) + continue; + if (branch_pushes_to_upstream(branch, upstream_refname)) + continue; + if (check_branch_commit(branch_name, branch_name, + &candidates.items[i]->objectname, NULL, + FILTER_REFS_BRANCHES, DELETE_BRANCH_SKIP_UNMERGED)) + continue; + + strbuf_reset(&key); + strbuf_addf(&key, "branch.%s.deletemerged", branch_name); + if (!repo_config_get_bool(the_repository, key.buf, &opt_out) && + !opt_out) { + if (!(flags & DELETE_BRANCH_QUIET)) + fprintf(stderr, + _("Skipping '%s' (branch.%s.deleteMerged is false)\n"), + branch_name, branch_name); + continue; + } + + strset_add(&deletable_branch_names, branch_name); + } + + protect_stacked_branch_bases(refs, &deletable_branch_names, + &protected_branch_names); + + strset_for_each_entry(&deletable_branch_names, &iter, entry) + strvec_push(&branches_to_delete, entry->key); + + if (branches_to_delete.nr) + ret = delete_branches(branches_to_delete.nr, branches_to_delete.v, + FILTER_REFS_BRANCHES, + DELETE_BRANCH_SKIP_UNMERGED | + DELETE_BRANCH_NO_HEAD_FALLBACK | + flags); + + if (!ret && !(flags & DELETE_BRANCH_DRY_RUN)) + clear_deleted_upstreams(&protected_branch_names, + &deletable_branch_names); + + strbuf_release(&key); + strvec_clear(&branches_to_delete); + strset_clear(&protected_branch_names); + strset_clear(&deletable_branch_names); + ref_array_clear(&candidates); + ref_filter_clear(&filter); + return ret; +} + static GIT_PATH_FUNC(edit_description, "EDIT_DESCRIPTION") static int edit_branch_description(const char *branch_name) @@ -745,6 +974,8 @@ int cmd_branch(int argc, /* possible actions */ int delete = 0, rename = 0, copy = 0, list = 0, unset_upstream = 0, show_current = 0, edit_description = 0; + struct strvec delete_merged = STRVEC_INIT; + int dry_run = 0; const char *new_upstream = NULL; int noncreate_actions = 0; /* possible options */ @@ -798,9 +1029,17 @@ int cmd_branch(int argc, OPT_BOOL(0, "create-reflog", &reflog, N_("create the branch's reflog")), OPT_BOOL(0, "edit-description", &edit_description, N_("edit the description for the branch")), + OPT_CALLBACK_F(0, "delete-merged", &delete_merged, N_("pattern"), + N_("delete merged branches whose upstream matches (repeatable)"), + PARSE_OPT_NONEG, parse_opt_strvec), + OPT_BOOL(0, "dry-run", &dry_run, + N_("with --delete-merged, only print which branches would be deleted")), OPT__FORCE(&force, N_("force creation, move/rename, deletion"), PARSE_OPT_NOCOMPLETE), OPT_MERGED(&filter, N_("print only branches that are merged")), OPT_NO_MERGED(&filter, N_("print only branches that are not merged")), + OPT_CALLBACK_F(0, "forked", &filter, N_("branch"), + N_("print only branches whose upstream matches (repeatable)"), + PARSE_OPT_NONEG, parse_opt_forked), OPT_COLUMN(0, "column", &colopts, N_("list branches in columns")), OPT_REF_SORT(&sorting_options), OPT_CALLBACK(0, "points-at", &filter.points_at, N_("object"), @@ -842,19 +1081,24 @@ int cmd_branch(int argc, 0); if (!delete && !rename && !copy && !edit_description && !new_upstream && - !show_current && !unset_upstream && argc == 0) + !show_current && !unset_upstream && !delete_merged.nr && + argc == 0) list = 1; if (filter.with_commit || filter.no_commit || - filter.reachable_from || filter.unreachable_from || filter.points_at.nr) + filter.reachable_from || filter.unreachable_from || + filter.points_at.nr || filter.forked.nr) list = 1; noncreate_actions = !!delete + !!rename + !!copy + !!new_upstream + !!show_current + !!list + !!edit_description + - !!unset_upstream; + !!unset_upstream + !!delete_merged.nr; if (noncreate_actions > 1) usage_with_options(builtin_branch_usage, options); + if (dry_run && !delete_merged.nr) + die(_("--dry-run requires --delete-merged")); + if (recurse_submodules_explicit) { if (!submodule_propagate_branches) die(_("branch with --recurse-submodules can only be used if submodule.propagateBranches is enabled")); @@ -889,7 +1133,14 @@ int cmd_branch(int argc, if (delete) { if (!argc) die(_("branch name required")); - ret = delete_branches(argc, argv, delete > 1, filter.kind, quiet); + ret = delete_branches(argc, argv, filter.kind, + (delete > 1 ? DELETE_BRANCH_FORCE : 0) | + (quiet ? DELETE_BRANCH_QUIET : 0)); + goto out; + } else if (delete_merged.nr) { + ret = delete_merged_branches(&delete_merged, argv, + (quiet ? DELETE_BRANCH_QUIET : 0) | + (dry_run ? DELETE_BRANCH_DRY_RUN : 0)); goto out; } else if (show_current) { print_current_branch_name(); @@ -1062,6 +1313,7 @@ int cmd_branch(int argc, ret = 0; out: + strvec_clear(&delete_merged); string_list_clear(&sorting_options, 0); return ret; } diff --git a/builtin/bundle.c b/builtin/bundle.c index 1e170e92780ed4..de86e092a6321f 100644 --- a/builtin/bundle.c +++ b/builtin/bundle.c @@ -69,31 +69,23 @@ static int parse_options_cmd_bundle(int argc, static int cmd_bundle_create(int argc, const char **argv, const char *prefix, struct repository *repo UNUSED) { - struct strvec pack_opts = STRVEC_INIT; + int progress = isatty(STDERR_FILENO); int version = -1; - int ret; struct option options[] = { - OPT_PASSTHRU_ARGV('q', "quiet", &pack_opts, NULL, - N_("do not show progress meter"), - PARSE_OPT_NOARG), - OPT_PASSTHRU_ARGV(0, "progress", &pack_opts, NULL, - N_("show progress meter"), - PARSE_OPT_NOARG), - OPT_PASSTHRU_ARGV(0, "all-progress", &pack_opts, NULL, - N_("historical; same as --progress"), - PARSE_OPT_NOARG | PARSE_OPT_HIDDEN), - OPT_PASSTHRU_ARGV(0, "all-progress-implied", &pack_opts, NULL, - N_("historical; does nothing"), - PARSE_OPT_NOARG | PARSE_OPT_HIDDEN), + OPT_NEGBIT('q', "quiet", &progress, + N_("do not show progress meter"), 1), + OPT_BIT(0, "progress", &progress, + N_("show progress meter"), 1), + OPT_BIT_F(0, "all-progress", &progress, + N_("historical; same as --progress"), 1, + PARSE_OPT_HIDDEN), + OPT_NOOP_NOARG(0, "all-progress-implied"), OPT_INTEGER(0, "version", &version, N_("specify bundle format version")), OPT_END() }; char *bundle_file; - - if (isatty(STDERR_FILENO)) - strvec_push(&pack_opts, "--progress"); - strvec_push(&pack_opts, "--all-progress-implied"); + int ret; argc = parse_options_cmd_bundle(argc, argv, prefix, builtin_bundle_create_usage, options, &bundle_file); @@ -101,8 +93,7 @@ static int cmd_bundle_create(int argc, const char **argv, const char *prefix, if (!startup_info->have_repository) die(_("Need a repository to create a bundle.")); - ret = !!create_bundle(the_repository, bundle_file, argc, argv, &pack_opts, version); - strvec_clear(&pack_opts); + ret = !!create_bundle(the_repository, bundle_file, argc, argv, version, progress); free(bundle_file); return ret; } diff --git a/builtin/cat-file.c b/builtin/cat-file.c index e85e2cfc1b0484..8870a210ec2e94 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -31,6 +31,7 @@ #include "alias.h" #include "remote.h" #include "transport.h" +#include "fetch-object-info.h" /* * Maximum length for a remote URL. While no universal standard exists, @@ -681,9 +682,8 @@ static void batch_one_object(const char *obj_name, static int get_remote_info(int argc, const char **argv, - struct object_info **remote_object_info, - struct oid_array *object_info_oids, - struct string_list *object_info_options) + struct fetch_object_info_results *results, + struct oid_array *object_info_oids) { int retval = 0; struct remote *remote = NULL; @@ -724,12 +724,8 @@ static int get_remote_info(int argc, goto cleanup; } - CALLOC_ARRAY(*remote_object_info, object_info_oids->nr); - gtransport->smart_options->object_info_oids = object_info_oids; - - gtransport->smart_options->object_info_options = object_info_options; - gtransport->smart_options->object_info_data = *remote_object_info; - retval = transport_fetch_object_info(gtransport); + retval = transport_fetch_object_info(gtransport, object_info_oids, + results); cleanup: transport_disconnect(gtransport); return retval; @@ -819,21 +815,6 @@ static void parse_cmd_mailmap(struct batch_options *opt UNUSED, load_mailmap(); } -struct protocol_placeholder_entry { - const char *option; - const char *atom; -}; - -static const struct protocol_placeholder_entry remote_atom_map[] = { - {"size", "objectsize"}, - {"type", "objecttype"}, - /* - * Add new protocol options here. Even if the server doesn't support - * them the allow_list will drop them if the server doesn't advertise - * them. - */ -}; - static void parse_cmd_remote_object_info(struct batch_options *opt, const char *line, struct strbuf *output, struct expand_data *data) @@ -841,18 +822,11 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, int count; const char **argv; char *line_to_split; - struct object_info *remote_object_info = NULL; + struct fetch_object_info_results results = FETCH_OBJECT_INFO_RESULTS_INIT; struct oid_array object_info_oids = OID_ARRAY_INIT; - struct string_list object_info_options = STRING_LIST_INIT_NODUP; - const char *saved_format = opt->format; if (strlen(line) >= MAX_REMOTE_OBJ_INFO_LINE) die(_("remote-object-info command too long")); - /* - * TODO: Use the default format once %(objecttype) is supported. - */ - if (!opt->format) - opt->format = "%(objectname) %(objectsize)"; line_to_split = xstrdup(line); count = split_cmdline(line_to_split, &argv); @@ -864,26 +838,25 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, MAX_ALLOWED_OBJ_LIMIT); if (data->info.sizep) - string_list_append(&object_info_options, "size"); + results.wants_size = 1; if (data->info.typep) - string_list_append(&object_info_options, "type"); + results.wants_type = 1; - if (get_remote_info(count, argv, &remote_object_info, - &object_info_oids, &object_info_options)) + if (get_remote_info(count, argv, &results, &object_info_oids)) die(_("failed to get object info from the remote: %s"), argv[0]); string_list_clear(&data->remote_allowed_atoms, 0); string_list_append(&data->remote_allowed_atoms, "objectname"); - for (size_t i = 0; i < ARRAY_SIZE(remote_atom_map); i++) - if (unsorted_string_list_has_string(&object_info_options, remote_atom_map[i].option)) - string_list_append(&data->remote_allowed_atoms, - remote_atom_map[i].atom); + if (results.sizes) + string_list_append(&data->remote_allowed_atoms, "objectsize"); + if (results.types) + string_list_append(&data->remote_allowed_atoms, "objecttype"); data->skip_object_info = 1; - for (size_t i = 0; i < object_info_oids.nr; i++) { + for (size_t i = 0; i < results.nr; i++) { data->oid = object_info_oids.oid[i]; - if (remote_object_info[i].unrecognized) { + if (results.unrecognized[i]) { report_object_status(opt, oid_to_hex(&data->oid), &data->oid, "missing"); continue; @@ -893,13 +866,11 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, * When reaching here, it means remote-object-info can retrieve * information from server without downloading them. */ - if (remote_object_info[i].sizep) { - data->size = *remote_object_info[i].sizep; - } + if (results.sizes) + data->size = results.sizes[i]; - if (remote_object_info[i].typep) { - data->type = *remote_object_info[i].typep; - } + if (results.types) + data->type = results.types[i]; opt->batch_mode = BATCH_MODE_INFO; data->is_remote = 1; @@ -907,14 +878,10 @@ static void parse_cmd_remote_object_info(struct batch_options *opt, data->is_remote = 0; } data->skip_object_info = 0; - opt->format = saved_format; - for (size_t i = 0; i < object_info_oids.nr; i++) - free_object_info_contents(&remote_object_info[i]); - string_list_clear(&object_info_options, 0); + free_fetch_object_info_results(&results); free(line_to_split); free(argv); - free(remote_object_info); oid_array_clear(&object_info_oids); } diff --git a/builtin/checkout.c b/builtin/checkout.c index 55e3a89a852712..09b3c87e2a1280 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -838,10 +838,11 @@ static void init_topts(struct unpack_trees_options *topts, static int merge_working_tree(const struct checkout_opts *opts, struct branch_info *old_branch_info, struct branch_info *new_branch_info, - bool quiet, + bool allow_autostash, int *writeout_error) { int ret; + bool can_autostash = false; struct lock_file lock_file = LOCK_INIT; struct tree *new_tree; @@ -888,9 +889,13 @@ static int merge_working_tree(const struct checkout_opts *opts, return 1; } + if (allow_autostash) + can_autostash = has_unstaged_changes(the_repository, 1) || + has_uncommitted_changes(the_repository, 1); + /* 2-way merge to the new branch */ init_topts(&topts, opts->show_progress, - opts->overwrite_ignore, quiet); + opts->overwrite_ignore, can_autostash); init_checkout_metadata(&topts.meta, new_branch_info->refname, new_branch_info->commit ? &new_branch_info->commit->object.oid : @@ -917,7 +922,8 @@ static int merge_working_tree(const struct checkout_opts *opts, clear_unpack_trees_porcelain(&topts); if (ret == -1) { rollback_lock_file(&lock_file); - return MERGE_WORKING_TREE_UNPACK_FAILED; + return can_autostash ? + MERGE_WORKING_TREE_UNPACK_FAILED : 1; } } @@ -1166,6 +1172,7 @@ static int switch_branches(const struct checkout_opts *opts, int flag, writeout_error = 0; int do_merge = 1; int created_autostash = 0; + bool autostash_conflicted = false; struct strbuf old_commit_shortname = STRBUF_INIT; struct strbuf autostash_msg = STRBUF_INIT; const char *stash_label_base = NULL; @@ -1242,7 +1249,8 @@ static int switch_branches(const struct checkout_opts *opts, new_branch_info->name, "local", stash_label_base, - autostash_msg.buf); + autostash_msg.buf, + &autostash_conflicted); } if (ret) { branch_info_release(&old_branch_info); @@ -1255,6 +1263,8 @@ static int switch_branches(const struct checkout_opts *opts, if (!opts->quiet && !old_branch_info.path && old_branch_info.commit && new_branch_info->commit != old_branch_info.commit) orphaned_commit_warning(old_branch_info.commit, new_branch_info->commit); + if (autostash_conflicted && !opts->quiet) + fputc('\n', stderr); update_refs_for_switch(opts, &old_branch_info, new_branch_info); if (created_autostash) { diff --git a/builtin/commit.c b/builtin/commit.c index 28f61745034506..abf6ed3606ad4c 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -804,18 +804,18 @@ static int prepare_to_commit(const char *index_file, const char *prefix, if (have_option_m && !fixup_message) { strbuf_addbuf(&sb, &message); hook_arg1 = "message"; - } else if (logfile && !strcmp(logfile, "-")) { + } else if (logfile && !fixup_message && !strcmp(logfile, "-")) { if (isatty(0)) fprintf(stderr, _("(reading log message from standard input)\n")); if (strbuf_read(&sb, 0, 0) < 0) die_errno(_("could not read log from standard input")); hook_arg1 = "message"; - } else if (logfile) { + } else if (logfile && !fixup_message) { if (strbuf_read_file(&sb, logfile, 0) < 0) die_errno(_("could not read log file '%s'"), logfile); hook_arg1 = "message"; - } else if (use_message) { + } else if (use_message && !fixup_message) { const char *buffer; buffer = strstr(use_message_buffer, "\n\n"); if (buffer) @@ -837,20 +837,26 @@ static int prepare_to_commit(const char *index_file, const char *prefix, hook_arg1 = "message"; /* - * Only `-m` commit message option is checked here, as - * it supports `--fixup` to append the commit message. - * - * The other commit message options `-c`/`-C`/`-F` are - * incompatible with all the forms of `--fixup` and - * have already errored out while parsing the `git commit` - * options. + * `-m`, `-F`, `-C`, and `-c` provide the message body. + * If none was given and this is an amend, use the target + * commit's body instead. */ - if (have_option_m && !strcmp(fixup_prefix, "fixup")) + if (have_option_m) { strbuf_addbuf(&sb, &message); - - if (!strcmp(fixup_prefix, "amend")) { - if (have_option_m) - die(_("options '%s' and '%s:%s' cannot be used together"), "-m", "--fixup", fixup_message); + } else if (logfile && !strcmp(logfile, "-")) { + if (isatty(0)) + fprintf(stderr, _("(reading log message from standard input)\n")); + if (strbuf_read(&sb, 0, 0) < 0) + die_errno(_("could not read log from standard input")); + } else if (logfile) { + if (strbuf_read_file(&sb, logfile, 0) < 0) + die_errno(_("could not read log file '%s'"), logfile); + } else if (use_message) { + struct commit *c = lookup_commit_reference_by_name(use_message); + if (!c) + die(_("could not lookup commit '%s'"), use_message); + prepare_amend_commit(c, &sb, &ctx); + } else if (!strcmp(fixup_prefix, "amend")) { prepare_amend_commit(commit, &sb, &ctx); } } else if (!stat(git_path_merge_msg(the_repository), &statbuf)) { @@ -1338,10 +1344,9 @@ static int parse_and_validate_options(int argc, const char *argv[], } if (fixup_message && squash_message) die(_("options '%s' and '%s' cannot be used together"), "--squash", "--fixup"); - die_for_incompatible_opt4(!!use_message, "-C", + die_for_incompatible_opt3(!!use_message, "-C", !!edit_message, "-c", - !!logfile, "-F", - !!fixup_message, "--fixup"); + !!logfile, "-F"); die_for_incompatible_opt4(have_option_m, "-m", !!edit_message, "-c", !!use_message, "-C", @@ -1980,7 +1985,7 @@ int cmd_commit(int argc, } apply_autostash_ref(the_repository, "MERGE_AUTOSTASH", - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); cleanup: free_commit_extra_headers(extra); diff --git a/builtin/config.c b/builtin/config.c index 0882899c3fbd2a..2554322317163c 100644 --- a/builtin/config.c +++ b/builtin/config.c @@ -1313,7 +1313,10 @@ static int show_editor(struct config_location_options *opts) else if (errno != EEXIST) die_errno(_("cannot create configuration file %s"), config_file); } - launch_editor(config_file, NULL, NULL); + if (launch_editor(config_file, NULL, NULL)) { + free(config_file); + return -1; + } free(config_file); return 0; diff --git a/builtin/diff-hunks.c b/builtin/diff-hunks.c new file mode 100644 index 00000000000000..3aea2dad56aa4d --- /dev/null +++ b/builtin/diff-hunks.c @@ -0,0 +1,53 @@ +#include "builtin.h" +#include "config.h" +#include "diff-hunks.h" +#include "gettext.h" +#include "parse-options.h" +#include "repository.h" + +static const char * const diff_hunks_usage[] = { + N_("git diff-hunks verify"), + N_("git diff-hunks clear"), + NULL +}; + +static int cmd_diff_hunks_verify(int argc, const char **argv, + const char *prefix UNUSED, + struct repository *r) +{ + struct option options[] = { OPT_END() }; + + argc = parse_options(argc, argv, NULL, options, diff_hunks_usage, 0); + if (argc) + usage_with_options(diff_hunks_usage, options); + return diff_hunks_verify(r) ? 1 : 0; +} + +static int cmd_diff_hunks_clear(int argc, const char **argv, + const char *prefix UNUSED, + struct repository *r) +{ + struct option options[] = { OPT_END() }; + + argc = parse_options(argc, argv, NULL, options, diff_hunks_usage, 0); + if (argc) + usage_with_options(diff_hunks_usage, options); + return diff_hunks_clear(r) ? 1 : 0; +} + +int cmd_diff_hunks(int argc, const char **argv, const char *prefix, + struct repository *repo) +{ + parse_opt_subcommand_fn *fn = NULL; + struct option options[] = { + OPT_SUBCOMMAND("verify", &fn, cmd_diff_hunks_verify), + OPT_SUBCOMMAND("clear", &fn, cmd_diff_hunks_clear), + OPT_END() + }; + + repo_config(repo, git_default_config, NULL); + + argc = parse_options(argc, argv, prefix, options, diff_hunks_usage, 0); + + return fn(argc, argv, prefix, repo); +} diff --git a/builtin/diff-tree.c b/builtin/diff-tree.c index 8b8f8b54e40664..296c6a137e23c6 100644 --- a/builtin/diff-tree.c +++ b/builtin/diff-tree.c @@ -170,6 +170,8 @@ int cmd_diff_tree(int argc, opt->diffopt.rotate_to_strict = 1; + diff_hunks_attach(&opt->diffopt); + /* * NOTE! We expect "a..b" to expand to "^a b" but it is * perfectly valid for revision range parser to yield "b ^a", @@ -234,5 +236,6 @@ int cmd_diff_tree(int argc, diff_free(&opt->diffopt); } + diff_hunks_detach(&opt->diffopt); return diff_result_code(opt); } diff --git a/builtin/diff.c b/builtin/diff.c index 18b1083e984a35..a39ffe69a48a5c 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -510,6 +510,7 @@ int cmd_diff(int argc, init_diffstat_widths(&rev.diffopt); rev.diffopt.flags.allow_external = 1; rev.diffopt.flags.allow_textconv = 1; + rev.diffopt.flags.allow_diff_process = 1; /* If this is a no-index diff, just run it and exit there. */ if (no_index) @@ -568,6 +569,15 @@ int cmd_diff(int argc, } } + /* + * The hunk store is keyed by blob pair, so any diff whose + * file pairs carry known blob object IDs (tree-to-tree, + * index-to-tree) can consult the same entries that + * "git log --stat" and "git blame" use; pairs without known + * blobs bypass it at lookup time. + */ + diff_hunks_attach(&rev.diffopt); + symdiff_prepare(&rev, &sdiff); for (i = 0; i < rev.pending.nr; i++) { struct object_array_entry *entry = &rev.pending.objects[i]; @@ -648,6 +658,7 @@ int cmd_diff(int argc, result = diff_result_code(&rev); if (1 < rev.diffopt.skip_stat_unmatch) refresh_index_quietly(); + diff_hunks_detach(&rev.diffopt); release_revisions(&rev); object_array_clear(&ent); symdiff_release(&sdiff); diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 6692f7cd812d0e..fbd919982c956a 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -30,6 +30,7 @@ #include "khash.h" #include "date.h" #include "gpg-interface.h" +#include "parse-options.h" #define PACK_ID_BITS 16 #define MAX_PACK_ID ((1<argc = argc; + state->argv = argv; + state->prefix = prefix; + state->option = option; +} + +static void parse_argv(struct fast_import_state *state); +static void parse_get_mark(struct fast_import_state *state, const char *p); +static void parse_cat_blob(struct fast_import_state *state, const char *p); +static void parse_ls(struct fast_import_state *state, + const char *p, + struct branch *b); static void for_each_mark(struct mark_set *m, uintmax_t base, each_mark_fn_t callback, void *p) { @@ -962,7 +980,7 @@ static int store_object( struct object_entry *e; unsigned char hdr[96]; struct object_id oid; - unsigned long hdrlen, deltalen; + unsigned long hdrlen, deltalen = 0; struct git_hash_ctx c; git_zstream s; struct repo_config_values *cfg = repo_config_values(the_repository); @@ -998,11 +1016,13 @@ static int store_object( if (last && last->data.len && last->data.buf && last->depth < max_depth && dat->len > the_hash_algo->rawsz) { + size_t deltalen_st; delta_count_attempts_by_type[type]++; delta = diff_delta(last->data.buf, last->data.len, dat->buf, dat->len, - &deltalen, dat->len - the_hash_algo->rawsz); + &deltalen_st, dat->len - the_hash_algo->rawsz); + deltalen = cast_size_t_to_ulong(deltalen_st); } else delta = NULL; @@ -1845,7 +1865,7 @@ static void read_marks(void) } -static int read_next_command(void) +static int read_next_command(struct fast_import_state *state) { static int stdin_eof = 0; @@ -1864,10 +1884,10 @@ static int read_next_command(void) if (stdin_eof) return EOF; - if (!seen_data_command + if (!state->seen_data_command && !starts_with(command_buf.buf, "feature ") && !starts_with(command_buf.buf, "option ")) { - parse_argv(); + parse_argv(state); } rc = rc_free; @@ -1899,22 +1919,22 @@ static void skip_optional_lf(void) ungetc(term_char, stdin); } -static void parse_mark(void) +static void parse_mark(struct fast_import_state *state) { const char *v; if (skip_prefix(command_buf.buf, "mark :", &v)) { next_mark = strtoumax(v, NULL, 10); - read_next_command(); + read_next_command(state); } else next_mark = 0; } -static void parse_original_identifier(void) +static void parse_original_identifier(struct fast_import_state *state) { const char *v; if (skip_prefix(command_buf.buf, "original-oid ", &v)) - read_next_command(); + read_next_command(state); } static int parse_data(struct strbuf *sb, uintmax_t limit, uintmax_t *len_res) @@ -2068,11 +2088,11 @@ static void parse_and_store_blob( } } -static void parse_new_blob(void) +static void parse_new_blob(struct fast_import_state *state) { - read_next_command(); - parse_mark(); - parse_original_identifier(); + read_next_command(state); + parse_mark(state); + parse_original_identifier(state); parse_and_store_blob(&last_blob, NULL, next_mark); } @@ -2368,7 +2388,9 @@ static void parse_path_space(struct strbuf *sb, const char *p, (*endp)++; } -static void file_change_m(const char *p, struct branch *b) +static void file_change_m(struct fast_import_state *state, + const char *p, + struct branch *b) { static struct strbuf path = STRBUF_INIT; struct object_entry *oe; @@ -2435,10 +2457,10 @@ static void file_change_m(const char *p, struct branch *b) if (S_ISDIR(mode)) die(_("directories cannot be specified 'inline': %s"), command_buf.buf); - while (read_next_command() != EOF) { + while (read_next_command(state) != EOF) { const char *v; if (skip_prefix(command_buf.buf, "cat-blob ", &v)) - parse_cat_blob(v); + parse_cat_blob(state, v); else { parse_and_store_blob(&last_blob, &oid, 0); break; @@ -2512,7 +2534,10 @@ static void file_change_cr(const char *p, struct branch *b, int rename) leaf.tree); } -static void note_change_n(const char *p, struct branch *b, unsigned char *old_fanout) +static void note_change_n(struct fast_import_state *state, + const char *p, + struct branch *b, + unsigned char *old_fanout) { struct object_entry *oe; struct branch *s; @@ -2577,7 +2602,7 @@ static void note_change_n(const char *p, struct branch *b, unsigned char *old_fa die(_("invalid ref name or SHA1 expression: %s"), p); if (inline_data) { - read_next_command(); + read_next_command(state); parse_and_store_blob(&last_blob, &oid, 0); } else if (oe) { if (oe->type != OBJ_BLOB) @@ -2644,7 +2669,9 @@ static void parse_from_existing(struct branch *b) } } -static int parse_objectish(struct branch *b, const char *objectish) +static int parse_objectish(struct fast_import_state *state, + struct branch *b, + const char *objectish) { struct branch *s; struct object_id oid; @@ -2687,31 +2714,34 @@ static int parse_objectish(struct branch *b, const char *objectish) b->branch_tree.tree = NULL; } - read_next_command(); + read_next_command(state); return 1; } -static int parse_from(struct branch *b) +static int parse_from(struct fast_import_state *state, struct branch *b) { const char *from; if (!skip_prefix(command_buf.buf, "from ", &from)) return 0; - return parse_objectish(b, from); + return parse_objectish(state, b, from); } -static int parse_objectish_with_prefix(struct branch *b, const char *prefix) +static int parse_objectish_with_prefix(struct fast_import_state *state, + struct branch *b, + const char *prefix) { const char *base; if (!skip_prefix(command_buf.buf, prefix, &base)) return 0; - return parse_objectish(b, base); + return parse_objectish(state, b, base); } -static struct hash_list *parse_merge(unsigned int *count) +static struct hash_list *parse_merge(struct fast_import_state *state, + unsigned int *count) { struct hash_list *list = NULL, **tail = &list, *n; const char *from; @@ -2745,7 +2775,7 @@ static struct hash_list *parse_merge(unsigned int *count) tail = &n->next; (*count)++; - read_next_command(); + read_next_command(state); } return list; } @@ -2756,7 +2786,9 @@ struct signature_data { struct strbuf data; /* The actual signature data */ }; -static void parse_one_signature(struct signature_data *sig, const char *v) +static void parse_one_signature(struct fast_import_state *state, + struct signature_data *sig, + const char *v) { char *args = xstrdup(v); /* Will be freed when sig->hash_algo is freed */ char *space = strchr(args, ' '); @@ -2781,15 +2813,15 @@ static void parse_one_signature(struct signature_data *sig, const char *v) warning(_("'unknown' signature format in gpgsig")); /* Read signature data */ - read_next_command(); + read_next_command(state); parse_data(&sig->data, 0, NULL); } -static void discard_one_signature(void) +static void discard_one_signature(struct fast_import_state *state) { struct strbuf data = STRBUF_INIT; - read_next_command(); + read_next_command(state); parse_data(&data, 0, NULL); strbuf_release(&data); } @@ -2827,13 +2859,14 @@ static void store_signature(struct signature_data *stored_sig, } } -static void import_one_signature(struct signature_data *sig_sha1, +static void import_one_signature(struct fast_import_state *state, + struct signature_data *sig_sha1, struct signature_data *sig_sha256, const char *v) { struct signature_data sig = { NULL, NULL, STRBUF_INIT }; - parse_one_signature(&sig, v); + parse_one_signature(state, &sig, v); if (!strcmp(sig.hash_algo, "sha1")) store_signature(sig_sha1, &sig, "SHA-1"); @@ -2947,7 +2980,7 @@ static void handle_signature_if_invalid(struct strbuf *new_data, strbuf_release(&tmp_buf); } -static void parse_new_commit(const char *arg) +static void parse_new_commit(struct fast_import_state *state, const char *arg) { static struct strbuf msg = STRBUF_INIT; struct signature_data sig_sha1 = { NULL, NULL, STRBUF_INIT }; @@ -2965,16 +2998,16 @@ static void parse_new_commit(const char *arg) if (!b) b = new_branch(arg); - read_next_command(); - parse_mark(); - parse_original_identifier(); + read_next_command(state); + parse_mark(state); + parse_original_identifier(state); if (skip_prefix(command_buf.buf, "author ", &v)) { author = parse_ident(v); - read_next_command(); + read_next_command(state); } if (skip_prefix(command_buf.buf, "committer ", &v)) { committer = parse_ident(v); - read_next_command(); + read_next_command(state); } if (!committer) die(_("expected committer but didn't get one")); @@ -2990,7 +3023,7 @@ static void parse_new_commit(const char *arg) warning(_("stripping a commit signature")); /* fallthru */ case SIGN_STRIP: - discard_one_signature(); + discard_one_signature(state); break; /* Second, modes that parse the signature */ @@ -3001,24 +3034,24 @@ static void parse_new_commit(const char *arg) case SIGN_STRIP_IF_INVALID: case SIGN_SIGN_IF_INVALID: case SIGN_ABORT_IF_INVALID: - import_one_signature(&sig_sha1, &sig_sha256, v); + import_one_signature(state, &sig_sha1, &sig_sha256, v); break; /* Third, BUG */ default: BUG("invalid signed_commit_mode value %d", signed_commit_mode); } - read_next_command(); + read_next_command(state); } if (skip_prefix(command_buf.buf, "encoding ", &v)) { encoding = xstrdup(v); - read_next_command(); + read_next_command(state); } parse_data(&msg, 0, NULL); - read_next_command(); - parse_from(b); - merge_list = parse_merge(&merge_count); + read_next_command(state); + parse_from(state, b); + merge_list = parse_merge(state, &merge_count); /* ensure the branch is active/loaded */ if (!b->branch_tree.tree || !max_active_branches) { @@ -3031,7 +3064,7 @@ static void parse_new_commit(const char *arg) /* file_change* */ while (command_buf.len > 0) { if (skip_prefix(command_buf.buf, "M ", &v)) - file_change_m(v, b); + file_change_m(state, v, b); else if (skip_prefix(command_buf.buf, "D ", &v)) file_change_d(v, b); else if (skip_prefix(command_buf.buf, "R ", &v)) @@ -3039,18 +3072,18 @@ static void parse_new_commit(const char *arg) else if (skip_prefix(command_buf.buf, "C ", &v)) file_change_cr(v, b, 0); else if (skip_prefix(command_buf.buf, "N ", &v)) - note_change_n(v, b, &prev_fanout); + note_change_n(state, v, b, &prev_fanout); else if (!strcmp("deleteall", command_buf.buf)) file_change_deleteall(b); else if (skip_prefix(command_buf.buf, "ls ", &v)) - parse_ls(v, b); + parse_ls(state, v, b); else if (skip_prefix(command_buf.buf, "cat-blob ", &v)) - parse_cat_blob(v); + parse_cat_blob(state, v); else { unread_command_buf = 1; break; } - if (read_next_command() == EOF) + if (read_next_command(state) == EOF) break; } @@ -3188,7 +3221,7 @@ static void handle_tag_signature(struct strbuf *buf, struct strbuf *msg, const c } } -static void parse_new_tag(const char *arg) +static void parse_new_tag(struct fast_import_state *state, const char *arg) { static struct strbuf msg = STRBUF_INIT; const char *from; @@ -3207,8 +3240,8 @@ static void parse_new_tag(const char *arg) else first_tag = t; last_tag = t; - read_next_command(); - parse_mark(); + read_next_command(state); + parse_mark(state); /* from ... */ if (!skip_prefix(command_buf.buf, "from ", &from)) @@ -3236,15 +3269,15 @@ static void parse_new_tag(const char *arg) type = oe->type; } else die(_("invalid ref name or SHA1 expression: %s"), from); - read_next_command(); + read_next_command(state); /* original-oid ... */ - parse_original_identifier(); + parse_original_identifier(state); /* tagger ... */ if (skip_prefix(command_buf.buf, "tagger ", &v)) { tagger = parse_ident(v); - read_next_command(); + read_next_command(state); } else tagger = NULL; @@ -3275,7 +3308,7 @@ static void parse_new_tag(const char *arg) t->pack_id = pack_id; } -static void parse_reset_branch(const char *arg) +static void parse_reset_branch(struct fast_import_state *state, const char *arg) { struct branch *b; const char *tag_name; @@ -3292,8 +3325,8 @@ static void parse_reset_branch(const char *arg) } else b = new_branch(arg); - read_next_command(); - parse_from(b); + read_next_command(state); + parse_from(state, b); if (b->delete && skip_prefix(b->name, "refs/tags/", &tag_name)) { /* * Elsewhere, we call dump_branches() before dump_tags(), @@ -3333,6 +3366,7 @@ static void cat_blob_write(const char *buf, unsigned long size) static void cat_blob(struct object_entry *oe, struct object_id *oid) { struct strbuf line = STRBUF_INIT; + struct iovec iov[3]; unsigned long size; enum object_type type = 0; char *buf; @@ -3366,10 +3400,21 @@ static void cat_blob(struct object_entry *oe, struct object_id *oid) strbuf_reset(&line); strbuf_addf(&line, "%s %s %"PRIuMAX"\n", oid_to_hex(oid), type_name(type), (uintmax_t)size); - cat_blob_write(line.buf, line.len); + + /* + * Write the header, the payload and the trailing newline with a + * single writev(3p) call instead of three separate write(3p) calls. + */ + iov[0].iov_base = line.buf; + iov[0].iov_len = line.len; + iov[1].iov_base = buf; + iov[1].iov_len = size; + iov[2].iov_base = (void *) "\n"; + iov[2].iov_len = 1; + + if (writev_in_full(cat_blob_fd, iov, ARRAY_SIZE(iov)) < 0) + die_errno(_("write to frontend failed")); strbuf_release(&line); - cat_blob_write(buf, size); - cat_blob_write("\n", 1); if (oe && oe->pack_id == pack_id) { last_blob.offset = oe->idx.offset; strbuf_attach(&last_blob.data, buf, size, size + 1); @@ -3378,7 +3423,8 @@ static void cat_blob(struct object_entry *oe, struct object_id *oid) free(buf); } -static void parse_get_mark(const char *p) +static void parse_get_mark(struct fast_import_state *state UNUSED, + const char *p) { struct object_entry *oe; char output[GIT_MAX_HEXSZ + 2]; @@ -3395,7 +3441,8 @@ static void parse_get_mark(const char *p) cat_blob_write(output, the_hash_algo->hexsz + 1); } -static void parse_cat_blob(const char *p) +static void parse_cat_blob(struct fast_import_state *state UNUSED, + const char *p) { struct object_entry *oe; struct object_id oid; @@ -3560,7 +3607,9 @@ static void print_ls(int mode, const unsigned char *hash, const char *path) cat_blob_write(line.buf, line.len); } -static void parse_ls(const char *p, struct branch *b) +static void parse_ls(struct fast_import_state *state UNUSED, + const char *p, + struct branch *b) { static struct strbuf path = STRBUF_INIT; struct tree_entry *root = NULL; @@ -3607,13 +3656,13 @@ static void checkpoint(void) dump_marks(); } -static void parse_checkpoint(void) +static void parse_checkpoint(struct fast_import_state *state UNUSED) { checkpoint_requested = 1; skip_optional_lf(); } -static void parse_progress(void) +static void parse_progress(struct fast_import_state *state UNUSED) { fwrite(command_buf.buf, 1, command_buf.len, stdout); fputc('\n', stdout); @@ -3621,37 +3670,40 @@ static void parse_progress(void) skip_optional_lf(); } -static void parse_alias(void) +static void parse_alias(struct fast_import_state *state) { struct object_entry *e; struct branch b; skip_optional_lf(); - read_next_command(); + read_next_command(state); /* mark ... */ - parse_mark(); + parse_mark(state); if (!next_mark) die(_("expected 'mark' command, got %s"), command_buf.buf); /* to ... */ memset(&b, 0, sizeof(b)); - if (!parse_objectish_with_prefix(&b, "to ")) + if (!parse_objectish_with_prefix(state, &b, "to ")) die(_("expected 'to' command, got %s"), command_buf.buf); e = find_object(&b.oid); assert(e); insert_mark(&marks, next_mark, e); } -static char* make_fast_import_path(const char *path) +static char* make_fast_import_path(struct fast_import_state *state, + const char *path) { if (!relative_marks_paths || is_absolute_path(path)) - return prefix_filename(global_prefix, path); + return prefix_filename(state->prefix, path); return repo_git_path(the_repository, "info/fast-import/%s", path); } -static void option_import_marks(const char *marks, - int from_stream, int ignore_missing) +static void option_import_marks(struct fast_import_state *state, + const char *marks, + int from_stream, + int ignore_missing) { if (import_marks_file) { if (from_stream) @@ -3663,7 +3715,7 @@ static void option_import_marks(const char *marks, } free(import_marks_file); - import_marks_file = make_fast_import_path(marks); + import_marks_file = make_fast_import_path(state, marks); import_marks_file_from_stream = from_stream; import_marks_file_ignore_missing = ignore_missing; } @@ -3703,13 +3755,15 @@ static void option_active_branches(const char *branches) max_active_branches = ulong_arg("--active-branches", branches); } -static void option_export_marks(const char *marks) +static void option_export_marks(struct fast_import_state *state, + const char *marks) { free(export_marks_file); - export_marks_file = make_fast_import_path(marks); + export_marks_file = make_fast_import_path(state, marks); } -static void option_cat_blob_fd(const char *fd) +static void option_cat_blob_fd(struct fast_import_state *state UNUSED, + const char *fd) { unsigned long n = ulong_arg("--cat-blob-fd", fd); if (n > (unsigned long) INT_MAX) @@ -3717,16 +3771,19 @@ static void option_cat_blob_fd(const char *fd) cat_blob_fd = (int) n; } -static void option_export_pack_edges(const char *edges) +static void option_export_pack_edges(struct fast_import_state *state, + const char *edges) { - char *fn = prefix_filename(global_prefix, edges); + char *fn = prefix_filename(state->prefix, edges); if (pack_edges) fclose(pack_edges); pack_edges = xfopen(fn, "a"); free(fn); } -static void option_rewrite_submodules(const char *arg, struct string_list *list) +static void option_rewrite_submodules(struct fast_import_state *state, + const char *arg, + struct string_list *list) { struct mark_set *ms; FILE *fp; @@ -3738,7 +3795,7 @@ static void option_rewrite_submodules(const char *arg, struct string_list *list) f++; CALLOC_ARRAY(ms, 1); - f = prefix_filename(global_prefix, f); + f = prefix_filename(state->prefix, f); fp = fopen(f, "r"); if (!fp) die_errno(_("cannot read '%s'"), f); @@ -3751,40 +3808,67 @@ static void option_rewrite_submodules(const char *arg, struct string_list *list) free(s); } -static int parse_one_option(const char *option) +static void option_max_pack_size(const char *arg) +{ + unsigned long v; + + if (!git_parse_ulong(arg, &v)) + die(_("--max-pack-size: argument must be a non-negative integer")); + if (v < 8192) { + warning(_("max-pack-size is now in bytes, assuming --max-pack-size=%lum"), v); + v *= 1024 * 1024; + } else if (v < 1024 * 1024) { + warning(_("minimum max-pack-size is 1 MiB")); + v = 1024 * 1024; + } + max_packsize = v; +} + +static void option_big_file_threshold(const char *arg) +{ + unsigned long v; + + if (!git_parse_ulong(arg, &v)) + die(_("--big-file-threshold: argument must be a non-negative integer")); + repo_settings_set_big_file_threshold(the_repository, v); +} + +static void option_signed_commits(const char *arg) +{ + if (parse_sign_mode(arg, &signed_commit_mode, &signed_commit_keyid)) + usagef(_("unknown --signed-commits mode '%s'"), arg); +} + +static void option_signed_tags(const char *arg) +{ + if (parse_sign_mode(arg, &signed_tag_mode, &signed_tag_keyid)) + usagef(_("unknown --signed-tags mode '%s'"), arg); +} + +static void option_quiet(void) +{ + show_stats = 0; + quiet = 1; +} + +static int parse_one_option(struct fast_import_state *state, const char *option) { if (skip_prefix(option, "max-pack-size=", &option)) { - unsigned long v; - if (!git_parse_ulong(option, &v)) - return 0; - if (v < 8192) { - warning(_("max-pack-size is now in bytes, assuming --max-pack-size=%lum"), v); - v *= 1024 * 1024; - } else if (v < 1024 * 1024) { - warning(_("minimum max-pack-size is 1 MiB")); - v = 1024 * 1024; - } - max_packsize = v; + option_max_pack_size(option); } else if (skip_prefix(option, "big-file-threshold=", &option)) { - unsigned long v; - if (!git_parse_ulong(option, &v)) - return 0; - repo_settings_set_big_file_threshold(the_repository, v); + option_big_file_threshold(option); } else if (skip_prefix(option, "depth=", &option)) { option_depth(option); } else if (skip_prefix(option, "active-branches=", &option)) { option_active_branches(option); } else if (skip_prefix(option, "export-pack-edges=", &option)) { - option_export_pack_edges(option); + option_export_pack_edges(state, option); } else if (skip_prefix(option, "signed-commits=", &option)) { - if (parse_sign_mode(option, &signed_commit_mode, &signed_commit_keyid)) - usagef(_("unknown --signed-commits mode '%s'"), option); + option_signed_commits(option); } else if (skip_prefix(option, "signed-tags=", &option)) { - if (parse_sign_mode(option, &signed_tag_mode, &signed_tag_keyid)) - usagef(_("unknown --signed-tags mode '%s'"), option); + option_signed_tags(option); } else if (!strcmp(option, "quiet")) { - show_stats = 0; - quiet = 1; + option_quiet(); } else if (!strcmp(option, "stats")) { show_stats = 1; } else if (!strcmp(option, "allow-unsafe-features")) { @@ -3796,34 +3880,36 @@ static int parse_one_option(const char *option) return 1; } -static void check_unsafe_feature(const char *feature, int from_stream) +static void check_unsafe_feature(struct fast_import_state *state, + const char *feature) { - if (from_stream && !allow_unsafe_features) + if (!state->allow_unsafe_features) die(_("feature '%s' forbidden in input without --allow-unsafe-features"), feature); } -static int parse_one_feature(const char *feature, int from_stream) +static int parse_one_feature(struct fast_import_state *state, + const char *feature) { const char *arg; if (skip_prefix(feature, "date-format=", &arg)) { option_date_format(arg); } else if (skip_prefix(feature, "import-marks=", &arg)) { - check_unsafe_feature("import-marks", from_stream); - option_import_marks(arg, from_stream, 0); + check_unsafe_feature(state, "import-marks"); + option_import_marks(state, arg, 1, 0); } else if (skip_prefix(feature, "import-marks-if-exists=", &arg)) { - check_unsafe_feature("import-marks-if-exists", from_stream); - option_import_marks(arg, from_stream, 1); + check_unsafe_feature(state, "import-marks-if-exists"); + option_import_marks(state, arg, 1, 1); } else if (skip_prefix(feature, "export-marks=", &arg)) { - check_unsafe_feature(feature, from_stream); - option_export_marks(arg); + check_unsafe_feature(state, feature); + option_export_marks(state, arg); } else if (!strcmp(feature, "alias")) { ; /* Don't die - this feature is supported */ } else if (skip_prefix(feature, "rewrite-submodules-to=", &arg)) { - option_rewrite_submodules(arg, &sub_marks_to); + option_rewrite_submodules(state, arg, &sub_marks_to); } else if (skip_prefix(feature, "rewrite-submodules-from=", &arg)) { - option_rewrite_submodules(arg, &sub_marks_from); + option_rewrite_submodules(state, arg, &sub_marks_from); } else if (!strcmp(feature, "get-mark")) { ; /* Don't die - this feature is supported */ } else if (!strcmp(feature, "cat-blob")) { @@ -3845,23 +3931,23 @@ static int parse_one_feature(const char *feature, int from_stream) return 1; } -static void parse_feature(const char *feature) +static void parse_feature(struct fast_import_state *state, const char *feature) { - if (seen_data_command) + if (state->seen_data_command) die(_("got feature command '%s' after data command"), feature); - if (parse_one_feature(feature, 1)) + if (parse_one_feature(state, feature)) return; die(_("this version of fast-import does not support feature %s."), feature); } -static void parse_option(const char *option) +static void parse_option(struct fast_import_state *state, const char *option) { - if (seen_data_command) + if (state->seen_data_command) die(_("got option command '%s' after data command"), option); - if (parse_one_option(option)) + if (parse_one_option(state, option)) return; die(_("this version of fast-import does not support option: %s"), option); @@ -3894,42 +3980,144 @@ static void git_pack_config(void) repo_config(the_repository, git_default_config, NULL); } -static const char fast_import_usage[] = -"git fast-import [--date-format=] [--max-pack-size=] [--big-file-threshold=] [--depth=] [--active-branches=] [--export-marks=]"; +static const char *const fast_import_usage[] = { + N_("git fast-import []"), + NULL +}; -static void parse_argv(void) +static void parse_argv(struct fast_import_state *state) { - unsigned int i; + int argc = parse_options(state->argc, state->argv, state->prefix, + state->option, fast_import_usage, + PARSE_OPT_KEEP_ARGV0); - for (i = 1; i < global_argc; i++) { - const char *a = global_argv[i]; + if (argc > 1) + usage_with_options(fast_import_usage, state->option); - if (*a != '-' || !strcmp(a, "--")) - break; + state->seen_data_command = 1; + if (import_marks_file) + read_marks(); + build_mark_map(&sub_marks_from, &sub_marks_to); +} - if (!skip_prefix(a, "--", &a)) - die(_("unknown option %s"), a); +static int option_parse_date_format(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_date_format(arg); + return 0; +} - if (parse_one_option(a)) - continue; +static int option_parse_export_pack_edges(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_export_pack_edges(opt->value, arg); + return 0; +} - if (parse_one_feature(a, 0)) - continue; +static int option_parse_max_pack_size(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_max_pack_size(arg); + return 0; +} - if (skip_prefix(a, "cat-blob-fd=", &a)) { - option_cat_blob_fd(a); - continue; - } +static int option_parse_big_file_threshold(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_big_file_threshold(arg); + return 0; +} - die(_("unknown option --%s"), a); - } - if (i != global_argc) - usage(fast_import_usage); +static int option_parse_signed_commits(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_signed_commits(arg); + return 0; +} - seen_data_command = 1; - if (import_marks_file) - read_marks(); - build_mark_map(&sub_marks_from, &sub_marks_to); +static int option_parse_signed_tags(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_signed_tags(arg); + return 0; +} + +static int option_parse_rewrite_submodules_from(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_rewrite_submodules(opt->value, arg, &sub_marks_from); + return 0; +} + +static int option_parse_rewrite_submodules_to(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_rewrite_submodules(opt->value, arg, &sub_marks_to); + return 0; +} + +static int option_parse_cat_blob_fd(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_cat_blob_fd(opt->value, arg); + return 0; +} + +static int option_parse_import_marks(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_import_marks(opt->value, arg, 0, 0); + return 0; +} + +static int option_parse_import_marks_if_exists(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_import_marks(opt->value, arg, 0, 1); + return 0; +} + +static int option_parse_export_marks(const struct option *opt, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_export_marks(opt->value, arg); + return 0; +} + +static int option_parse_depth(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_depth(arg); + return 0; +} + +static int option_parse_active_branches(const struct option *opt UNUSED, + const char *arg, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_active_branches(arg); + return 0; +} + +static int option_parse_quiet(const struct option *opt UNUSED, + const char *arg UNUSED, int unset) +{ + BUG_ON_OPT_NEG(unset); + option_quiet(); + return 0; } int cmd_fast_import(int argc, @@ -3937,9 +4125,80 @@ int cmd_fast_import(int argc, const char *prefix, struct repository *repo) { - unsigned int i; - - show_usage_if_asked(argc, argv, fast_import_usage); + struct fast_import_state state; + + struct option fast_import_options[] = { + OPT_GROUP(N_("Common")), + OPT_CALLBACK_F(0, "date-format", NULL, N_("fmt"), + N_("format of the commit/tag dates"), + PARSE_OPT_NONEG, option_parse_date_format), + OPT_BOOL_F(0, "stats", &show_stats, + N_("display some basic statistics (objects, packfiles and memory)"), + PARSE_OPT_NONEG), + OPT_CALLBACK_F(0, "quiet", NULL, NULL, + N_("disable the output shown by --stats"), + PARSE_OPT_NOARG | PARSE_OPT_NONEG, + option_parse_quiet), + OPT_BOOL_F(0, "force", &force_update, + N_("force updating modified existing branches"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "done", &require_explicit_termination, + N_("require a terminating 'done' command"), PARSE_OPT_NONEG), + OPT_CALLBACK_F(0, "max-pack-size", NULL, N_("n"), + N_("maximum size of each output pack file"), + PARSE_OPT_NONEG, option_parse_max_pack_size), + OPT_CALLBACK_F(0, "big-file-threshold", NULL, N_("n"), + N_("maximum size of a blob that will be deltified"), + PARSE_OPT_NONEG, option_parse_big_file_threshold), + OPT_CALLBACK_F(0, "depth", NULL, N_("n"), + N_("maximum delta depth"), + PARSE_OPT_NONEG, option_parse_depth), + OPT_CALLBACK_F(0, "active-branches", NULL, N_("n"), + N_("maximum number of branches to maintain active"), + PARSE_OPT_NONEG, option_parse_active_branches), + OPT_GROUP(N_("Marks")), + OPT_CALLBACK_F(0, "import-marks", &state, N_("file"), + N_("import marks from "), + PARSE_OPT_NONEG, option_parse_import_marks), + OPT_CALLBACK_F(0, "import-marks-if-exists", &state, N_("file"), + N_("import marks from if it exists"), + PARSE_OPT_NONEG, option_parse_import_marks_if_exists), + OPT_CALLBACK_F(0, "export-marks", &state, N_("file"), + N_("dump marks to "), + PARSE_OPT_NONEG, option_parse_export_marks), + OPT_BOOL(0, "relative-marks", &relative_marks_paths, + N_("are --(import|export)-marks= paths relative to '.git/info/fast-import'?")), + OPT_GROUP(N_("Submodule rewrite")), + OPT_CALLBACK_F(0, "rewrite-submodules-from", &state, N_("name:filename"), + N_("rewrite object IDs for submodule from "), + PARSE_OPT_NONEG, option_parse_rewrite_submodules_from), + OPT_CALLBACK_F(0, "rewrite-submodules-to", &state, N_("name:filename"), + N_("rewrite object IDs for submodule to "), + PARSE_OPT_NONEG, option_parse_rewrite_submodules_to), + OPT_GROUP(N_("Signing")), + OPT_CALLBACK_F(0, "signed-commits", NULL, N_("mode"), + N_("how to handle signed commits"), + PARSE_OPT_NONEG, option_parse_signed_commits), + OPT_CALLBACK_F(0, "signed-tags", NULL, N_("mode"), + N_("how to handle signed tags"), + PARSE_OPT_NONEG, option_parse_signed_tags), + OPT_HIDDEN_GROUP(N_("Advanced")), + OPT_BOOL_F(0, "allow-unsafe-features", &state.allow_unsafe_features, + N_("allow unsafe mark commands from the stream"), + PARSE_OPT_HIDDEN | PARSE_OPT_NONEG), + OPT_CALLBACK_F(0, "export-pack-edges", &state, N_("file"), + N_("dump edge commits to "), + PARSE_OPT_HIDDEN | PARSE_OPT_NONEG, + option_parse_export_pack_edges), + OPT_CALLBACK_F(0, "cat-blob-fd", &state, N_("fd"), + N_("write some responses to instead of stdout"), + PARSE_OPT_HIDDEN | PARSE_OPT_NONEG, + option_parse_cat_blob_fd), + OPT_END() + }; + + show_usage_with_options_if_asked(argc, argv, fast_import_usage, fast_import_options); + + fast_import_state_init(&state, argc, argv, prefix, fast_import_options); reset_pack_idx_option(&pack_idx_opts); git_pack_config(); @@ -3958,55 +4217,61 @@ int cmd_fast_import(int argc, * "feature" lines at the start of the stream (which allows the command * line to override stream data). But we must do an early parse of any * command-line options that impact how we interpret the feature lines. + * + * NEEDSWORK: This scan only matches the exact "--allow-unsafe-features" + * spelling and stops at the first argument that doesn't start with a + * dash. As parse_options() below also accepts unambiguous abbreviations + * and values separated by a space from their option, the two disagree + * for command lines like "--allow-unsafe" or "--depth 5 + * --allow-unsafe-features": parse_options() accepts the option, but + * this scan doesn't see it, so unsafe features from the stream are + * still refused. This errs on the safe side, but should be fixed by + * teaching this scan about the options that take a value. */ - for (i = 1; i < argc; i++) { + for (int i = 1; i < argc; i++) { const char *arg = argv[i]; if (*arg != '-' || !strcmp(arg, "--")) break; if (!strcmp(arg, "--allow-unsafe-features")) - allow_unsafe_features = 1; + state.allow_unsafe_features = 1; } - global_argc = argc; - global_argv = argv; - global_prefix = prefix; - rc_free = mem_pool_alloc(&fi_mem_pool, cmd_save * sizeof(*rc_free)); - for (i = 0; i < (cmd_save - 1); i++) + for (unsigned int i = 0; i < (cmd_save - 1); i++) rc_free[i].next = &rc_free[i + 1]; rc_free[cmd_save - 1].next = NULL; start_packfile(); set_die_routine(die_nicely); set_checkpoint_signal(); - while (read_next_command() != EOF) { + while (read_next_command(&state) != EOF) { const char *v; if (!strcmp("blob", command_buf.buf)) - parse_new_blob(); + parse_new_blob(&state); else if (skip_prefix(command_buf.buf, "commit ", &v)) - parse_new_commit(v); + parse_new_commit(&state, v); else if (skip_prefix(command_buf.buf, "tag ", &v)) - parse_new_tag(v); + parse_new_tag(&state, v); else if (skip_prefix(command_buf.buf, "reset ", &v)) - parse_reset_branch(v); + parse_reset_branch(&state, v); else if (skip_prefix(command_buf.buf, "ls ", &v)) - parse_ls(v, NULL); + parse_ls(&state, v, NULL); else if (skip_prefix(command_buf.buf, "cat-blob ", &v)) - parse_cat_blob(v); + parse_cat_blob(&state, v); else if (skip_prefix(command_buf.buf, "get-mark ", &v)) - parse_get_mark(v); + parse_get_mark(&state, v); else if (!strcmp("checkpoint", command_buf.buf)) - parse_checkpoint(); + parse_checkpoint(&state); else if (!strcmp("done", command_buf.buf)) break; else if (!strcmp("alias", command_buf.buf)) - parse_alias(); + parse_alias(&state); else if (starts_with(command_buf.buf, "progress ")) - parse_progress(); + parse_progress(&state); else if (skip_prefix(command_buf.buf, "feature ", &v)) - parse_feature(v); + parse_feature(&state, v); else if (skip_prefix(command_buf.buf, "option git ", &v)) - parse_option(v); + parse_option(&state, v); else if (starts_with(command_buf.buf, "option ")) /* ignore non-git options*/; else @@ -4017,8 +4282,8 @@ int cmd_fast_import(int argc, } /* argv hasn't been parsed yet, do so */ - if (!seen_data_command) - parse_argv(); + if (!state.seen_data_command) + parse_argv(&state); if (require_explicit_termination && feof(stdin)) die(_("stream ends early")); @@ -4035,9 +4300,9 @@ int cmd_fast_import(int argc, if (show_stats) { uintmax_t total_count = 0, duplicate_count = 0; - for (i = 0; i < ARRAY_SIZE(object_count_by_type); i++) + for (size_t i = 0; i < ARRAY_SIZE(object_count_by_type); i++) total_count += object_count_by_type[i]; - for (i = 0; i < ARRAY_SIZE(duplicate_count_by_type); i++) + for (size_t i = 0; i < ARRAY_SIZE(duplicate_count_by_type); i++) duplicate_count += duplicate_count_by_type[i]; fprintf(stderr, "%s statistics:\n", argv[0]); diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 316badd969613f..86754296facc70 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -49,7 +49,7 @@ static void add_sought_entry(struct ref ***sought, int *nr, int *alloc, int cmd_fetch_pack(int argc, const char **argv, const char *prefix UNUSED, - struct repository *repo UNUSED) + struct repository *repo) { int i, ret; struct ref *fetched_refs = NULL, *remote_refs = NULL; @@ -67,8 +67,6 @@ int cmd_fetch_pack(int argc, struct packet_reader reader; enum protocol_version version; - fetch_if_missing = 0; - packet_trace_identity("fetch-pack"); memset(&args, 0, sizeof(args)); @@ -77,6 +75,8 @@ int cmd_fetch_pack(int argc, show_usage_if_asked(argc, argv, fetch_pack_usage); + repo->fetch_if_missing = 0; + for (i = 1; i < argc && *argv[i] == '-'; i++) { const char *arg = argv[i]; diff --git a/builtin/fetch.c b/builtin/fetch.c index ab7db2be06d14c..a492b2809f24d4 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -111,8 +111,30 @@ struct fetch_config { int recurse_submodules; int parallel; int submodule_fetch_jobs; + int submodule_errors; }; +/* really private - use accessors below to parse and format */ +static const char *submodule_error_name[] = { + [SUBMODULE_ERRORS_FAIL] = "fail", + [SUBMODULE_ERRORS_WARN] = "warn", +}; + +static const char *submodule_error(unsigned num) +{ + if (ARRAY_SIZE(submodule_error_name) <= num) + BUG("invalid submodule errors mode %u", num); + return submodule_error_name[num]; +} + +static int parse_submodule_error(const char *name) +{ + for (unsigned num = 0; num < ARRAY_SIZE(submodule_error_name); num++) + if (!strcmp(submodule_error_name[num], name)) + return num; + return -1; +} + static int git_fetch_config(const char *k, const char *v, const struct config_context *ctx, void *cb) { @@ -153,6 +175,19 @@ static int git_fetch_config(const char *k, const char *v, return 0; } + if (!strcmp(k, "fetch.submoduleerrors")) { + int mode; + + if (!v) + return config_error_nonbool(k); + mode = parse_submodule_error(v); + if (mode < 0) + die(_("invalid value for '%s': '%s'"), + "fetch.submoduleErrors", v); + fetch_config->submodule_errors = mode; + return 0; + } + if (!strcmp(k, "fetch.parallel")) { fetch_config->parallel = git_config_int(k, v, ctx->kvi); if (fetch_config->parallel < 0) @@ -2243,6 +2278,9 @@ static void add_options_to_argv(struct strvec *argv, strvec_push(argv, "--no-recurse-submodules"); else if (config->recurse_submodules == RECURSE_SUBMODULES_ON_DEMAND) strvec_push(argv, "--recurse-submodules=on-demand"); + if (config->submodule_errors != -1) + strvec_pushf(argv, "--submodule-errors=%s", + submodule_error(config->submodule_errors)); if (tags == TAGS_SET) strvec_push(argv, "--tags"); else if (tags == TAGS_UNSET) @@ -2502,6 +2540,23 @@ static int fetch_one(struct remote *remote, int argc, const char **argv, return exit_code; } +static int option_parse_submodule_errors(const struct option *opt, + const char *arg, int unset) +{ + int *v = opt->value; + int mode; + + if (unset) { + *v = SUBMODULE_ERRORS_FAIL; + return 0; + } + mode = parse_submodule_error(arg); + if (mode < 0) + die(_("invalid value for '%s': '%s'"), "--submodule-errors", arg); + *v = mode; + return 0; +} + int cmd_fetch(int argc, const char **argv, const char *prefix, @@ -2516,6 +2571,7 @@ int cmd_fetch(int argc, .recurse_submodules = RECURSE_SUBMODULES_DEFAULT, .parallel = 1, .submodule_fetch_jobs = -1, + .submodule_errors = -1, /* unset */ }; const char *submodule_prefix = ""; const char *bundle_uri; @@ -2530,6 +2586,7 @@ int cmd_fetch(int argc, int max_jobs = -1; int recurse_submodules_cli = RECURSE_SUBMODULES_DEFAULT; int recurse_submodules_default = RECURSE_SUBMODULES_ON_DEMAND; + int submodule_errors_cli = -1; /* -1: not set on command line */ int fetch_write_commit_graph = -1; int stdin_refspecs = 0; int negotiate_only = 0; @@ -2566,6 +2623,10 @@ int cmd_fetch(int argc, OPT_CALLBACK_F(0, "recurse-submodules", &recurse_submodules_cli, N_("on-demand"), N_("control recursive fetching of submodules"), PARSE_OPT_OPTARG, option_fetch_parse_recurse_submodules), + OPT_CALLBACK_F(0, "submodule-errors", &submodule_errors_cli, + N_("(fail|warn)"), + N_("control how submodule fetch errors are handled"), + 0, option_parse_submodule_errors), OPT_BOOL(0, "dry-run", &dry_run, N_("dry run")), OPT_BOOL(0, "porcelain", &porcelain, N_("machine-readable output")), @@ -2657,6 +2718,9 @@ int cmd_fetch(int argc, if (recurse_submodules_cli != RECURSE_SUBMODULES_DEFAULT) config.recurse_submodules = recurse_submodules_cli; + if (submodule_errors_cli != -1) + config.submodule_errors = submodule_errors_cli; + if (negotiate_only) { switch (recurse_submodules_cli) { case RECURSE_SUBMODULES_OFF: @@ -2860,11 +2924,14 @@ int cmd_fetch(int argc, if (!result && remote && (config.recurse_submodules != RECURSE_SUBMODULES_OFF)) { struct strvec options = STRVEC_INIT; int max_children = max_jobs; + int submodule_errors = config.submodule_errors; if (max_children < 0) max_children = config.submodule_fetch_jobs; if (max_children < 0) max_children = config.parallel; + if (submodule_errors < 0) + submodule_errors = SUBMODULE_ERRORS_FAIL; add_options_to_argv(&options, &config); trace2_region_enter_printf("fetch", "recurse-submodule", the_repository, "%s", submodule_prefix); @@ -2874,7 +2941,8 @@ int cmd_fetch(int argc, config.recurse_submodules, recurse_submodules_default, verbosity < 0, - max_children); + max_children, + submodule_errors); trace2_region_leave_printf("fetch", "recurse-submodule", the_repository, "%s", submodule_prefix); strvec_clear(&options); } diff --git a/builtin/fsck.c b/builtin/fsck.c index a6c054e45bf8c0..6d368f06f36b50 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -1017,15 +1017,15 @@ int cmd_fsck(int argc, .ref = NULL }; - /* fsck knows how to handle missing promisor objects */ - fetch_if_missing = 0; - errors_found = 0; disable_replace_refs(); save_commit_buffer = 0; argc = parse_options(argc, argv, prefix, fsck_opts, fsck_usage, 0); + /* fsck knows how to handle missing promisor objects */ + repo->fetch_if_missing = 0; + fsck_options_init(&fsck_walk_options, repo, FSCK_OPTIONS_DEFAULT); fsck_walk_options.walk = mark_object; @@ -1069,7 +1069,6 @@ int cmd_fsck(int argc, odb_for_each_object(repo->objects, NULL, mark_object_for_connectivity, repo, 0); } else { - odb_prepare_alternates(repo->objects); for (source = repo->objects->sources; source; source = source->next) fsck_source(repo, source); @@ -1155,7 +1154,6 @@ int cmd_fsck(int argc, if (repo->settings.core_commit_graph) { struct child_process commit_graph_verify = CHILD_PROCESS_INIT; - odb_prepare_alternates(repo->objects); for (source = repo->objects->sources; source; source = source->next) { child_process_init(&commit_graph_verify); commit_graph_verify.git_cmd = 1; @@ -1173,7 +1171,6 @@ int cmd_fsck(int argc, if (repo->settings.core_multi_pack_index) { struct child_process midx_verify = CHILD_PROCESS_INIT; - odb_prepare_alternates(repo->objects); for (source = repo->objects->sources; source; source = source->next) { child_process_init(&midx_verify); midx_verify.git_cmd = 1; diff --git a/builtin/history.c b/builtin/history.c index 000155ad9c24d5..db35a7451a604f 100644 --- a/builtin/history.c +++ b/builtin/history.c @@ -1,6 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "builtin.h" +#include "advice.h" #include "cache-tree.h" #include "commit.h" #include "commit-reach.h" @@ -16,10 +17,12 @@ #include "path.h" #include "read-cache.h" #include "refs.h" +#include "ref-filter.h" #include "replay.h" #include "reset.h" #include "revision.h" #include "sequencer.h" +#include "string-list.h" #include "strvec.h" #include "tree.h" #include "tree-walk.h" @@ -34,6 +37,8 @@ N_("git history reword [--dry-run] [--update-refs=(branches|head)]") #define GIT_HISTORY_SPLIT_USAGE \ N_("git history split [--dry-run] [--update-refs=(branches|head)] [--] [...]") +#define GIT_HISTORY_SQUASH_USAGE \ + N_("git history squash [--dry-run] [--update-refs=(branches|head)] [--[no-]edit] ") static void change_data_free(void *util, const char *str UNUSED) { @@ -108,6 +113,7 @@ enum commit_tree_flags { static int commit_tree_ext(struct repository *repo, const char *action, struct commit *commit_with_message, + const char *message_template, const struct commit_list *parents, const struct object_id *old_tree, const struct object_id *new_tree, @@ -137,13 +143,16 @@ static int commit_tree_ext(struct repository *repo, original_author = xmemdupz(ptr, len); find_commit_subject(original_message, &original_body); + if (!message_template) + message_template = original_body; + if (flags & COMMIT_TREE_EDIT_MESSAGE) { ret = fill_commit_message(repo, old_tree, new_tree, - original_body, action, &commit_message); + message_template, action, &commit_message); if (ret < 0) goto out; } else { - strbuf_addstr(&commit_message, original_body); + strbuf_addstr(&commit_message, message_template); } original_extra_headers = read_commit_extra_headers(commit_with_message, @@ -164,6 +173,25 @@ static int commit_tree_ext(struct repository *repo, return ret; } +static int first_parent_tree_oid(struct repository *repo, + struct commit *commit, + struct object_id *out) +{ + struct commit *parent = commit->parents ? commit->parents->item : NULL; + + if (!parent) { + oidcpy(out, repo->hash_algo->empty_tree); + return 0; + } + + if (repo_parse_commit(repo, parent)) + return error(_("unable to parse parent commit %s"), + oid_to_hex(&parent->object.oid)); + + oidcpy(out, &repo_get_commit_tree(repo, parent)->object.oid); + return 0; +} + static int commit_tree_with_edited_message(struct repository *repo, const char *action, struct commit *original, @@ -171,23 +199,13 @@ static int commit_tree_with_edited_message(struct repository *repo, { struct object_id parent_tree_oid; const struct object_id *tree_oid; - struct commit *parent; tree_oid = &repo_get_commit_tree(repo, original)->object.oid; - parent = original->parents ? original->parents->item : NULL; - if (parent) { - if (repo_parse_commit(repo, parent)) { - return error(_("unable to parse parent commit %s"), - oid_to_hex(&parent->object.oid)); - } - - parent_tree_oid = repo_get_commit_tree(repo, parent)->object.oid; - } else { - oidcpy(&parent_tree_oid, repo->hash_algo->empty_tree); - } + if (first_parent_tree_oid(repo, original, &parent_tree_oid) < 0) + return -1; - return commit_tree_ext(repo, action, original, original->parents, + return commit_tree_ext(repo, action, original, NULL, original->parents, &parent_tree_oid, tree_oid, out, COMMIT_TREE_EDIT_MESSAGE); } @@ -475,18 +493,10 @@ static int commit_became_empty(struct repository *repo, struct commit *original, struct tree *result) { - struct commit *parent = original->parents ? original->parents->item : NULL; struct object_id parent_tree_oid; - if (parent) { - if (repo_parse_commit(repo, parent)) - return error(_("unable to parse parent of %s"), - oid_to_hex(&original->object.oid)); - - parent_tree_oid = repo_get_commit_tree(repo, parent)->object.oid; - } else { - oidcpy(&parent_tree_oid, repo->hash_algo->empty_tree); - } + if (first_parent_tree_oid(repo, original, &parent_tree_oid) < 0) + return -1; return oideq(&result->object.oid, &parent_tree_oid); } @@ -674,7 +684,7 @@ static int cmd_history_fixup(int argc, goto out; if (!skip_commit) { - ret = commit_tree_ext(repo, "fixup", original, original->parents, + ret = commit_tree_ext(repo, "fixup", original, NULL, original->parents, &original_tree->object.oid, &merge_result.tree->object.oid, &rewritten, flags); if (ret < 0) { @@ -830,16 +840,9 @@ static int split_commit(struct repository *repo, struct tree *split_tree; int ret; - if (original->parents) { - if (repo_parse_commit(repo, original->parents->item)) { - ret = error(_("unable to parse parent commit %s"), - oid_to_hex(&original->parents->item->object.oid)); - goto out; - } - - parent_tree_oid = *get_commit_tree_oid(original->parents->item); - } else { - oidcpy(&parent_tree_oid, repo->hash_algo->empty_tree); + if (first_parent_tree_oid(repo, original, &parent_tree_oid) < 0) { + ret = -1; + goto out; } original_commit_tree_oid = get_commit_tree_oid(original); @@ -892,7 +895,7 @@ static int split_commit(struct repository *repo, * The first commit is constructed from the split-out tree. The base * that shall be diffed against is the parent of the original commit. */ - ret = commit_tree_ext(repo, "split-out", original, original->parents, &parent_tree_oid, + ret = commit_tree_ext(repo, "split-out", original, NULL, original->parents, &parent_tree_oid, &split_tree->object.oid, &first_commit, COMMIT_TREE_EDIT_MESSAGE); if (ret < 0) { ret = error(_("failed writing first commit")); @@ -909,7 +912,7 @@ static int split_commit(struct repository *repo, old_tree_oid = &repo_get_commit_tree(repo, first_commit)->object.oid; new_tree_oid = &repo_get_commit_tree(repo, original)->object.oid; - ret = commit_tree_ext(repo, "split-out", original, parents, old_tree_oid, + ret = commit_tree_ext(repo, "split-out", original, NULL, parents, old_tree_oid, new_tree_oid, &second_commit, COMMIT_TREE_EDIT_MESSAGE); if (ret < 0) { ret = error(_("failed writing second commit")); @@ -1006,6 +1009,645 @@ static int cmd_history_split(int argc, return ret; } +#define SQUASH_SEEN (1u << 11) +#define SQUASH_TIP (1u << 12) +#define SQUASH_AMEND_TARGET (1u << 13) + +static bool is_autosquash_subject(const char *s) +{ + return starts_with(s, "amend!") || starts_with(s, "fixup!") || + starts_with(s, "squash!"); +} + +static bool skip_one_autosquash_prefix(const char *s, const char **out) +{ + if (skip_prefix(s, "amend!", out) || skip_prefix(s, "fixup!", out) || + skip_prefix(s, "squash!", out)) { + while (**out == ' ') + (*out)++; + return true; + } + return false; +} + +static void truncate_message_to_subject(struct strbuf *msg) +{ + const char *eos = strstr(msg->buf, "\n\n"); + + if (eos) + strbuf_setlen(msg, eos - msg->buf + 1); +} + +struct subject_data { + struct strintmap subjects; + struct strbuf subject; + struct strbuf squash_message; + const char *message; + bool edit_message; +}; + +#define SUBJECT_DATA_INIT { \ + .subjects = STRINTMAP_INIT, \ + .subject = STRBUF_INIT, \ + .squash_message = STRBUF_INIT, \ +} + +static void subject_data_clear(struct subject_data *data) +{ + strintmap_clear(&data->subjects); + strbuf_release(&data->subject); + strbuf_release(&data->squash_message); +} + +static int squash_amend_message(struct repository *repo, + struct commit *commit, + struct subject_data *data, + unsigned flags) +{ + const char *body = data->message + data->subject.len; + + while (isspace(*body)) + body++; + + if (!*body) { + warning(_("ignoring %s (%s): message body is empty"), + repo_find_unique_abbrev(repo, &commit->object.oid, + DEFAULT_ABBREV), + data->subject.buf); + return 0; + } + + if (data->edit_message) { + return 0; + } else if (flags & SQUASH_AMEND_TARGET) { + if (starts_with(data->squash_message.buf, "squash!")) + return error(_("squashing %s (%s) would overwrite " + "'squash!' message, please combine them " + "using '--edit'"), + repo_find_unique_abbrev(repo, + &commit->object.oid, + DEFAULT_ABBREV), + data->subject.buf); + if (starts_with(data->squash_message.buf, "fixup!")) + strbuf_splice(&data->squash_message, 0, 5, "amend!", 5); + if (starts_with(data->squash_message.buf, "amend!")) { + truncate_message_to_subject(&data->squash_message); + strbuf_addch(&data->squash_message, '\n'); + } else { + strbuf_reset(&data->squash_message); + } + strbuf_addstr(&data->squash_message, body); + strbuf_complete_line(&data->squash_message); + } else { + return error(_("cannot squash %s (%s) that does not target " + "base commit without '--edit'"), + repo_find_unique_abbrev(repo, &commit->object.oid, + DEFAULT_ABBREV), + data->subject.buf); + } + return 0; +} + +static int squash_squash_message(struct repository *repo, + struct commit *commit, + struct subject_data *data, + unsigned flags) +{ + const char *body = data->message + data->subject.len; + + while (isspace(*body)) + body++; + + if (data->edit_message) { + return 0; + } else if (flags & SQUASH_AMEND_TARGET) { + if (starts_with(data->squash_message.buf, "fixup!")) { + truncate_message_to_subject(&data->squash_message); + strbuf_splice(&data->squash_message, 0, 5, "squash", 6); + } + if (starts_with(data->squash_message.buf, "squash!")) { + strbuf_addch(&data->squash_message, '\n'); + strbuf_addstr(&data->squash_message, body); + strbuf_complete_line(&data->squash_message); + } else { + return error(_("squashing %s (%s) would discard its " + "message, please combine them using " + "'--edit'"), + repo_find_unique_abbrev(repo, + &commit->object.oid, + DEFAULT_ABBREV), + data->subject.buf); + } + } else { + return error(_("cannot squash %s (%s) that does not target " + "base commit without '--edit'"), + repo_find_unique_abbrev(repo, &commit->object.oid, + DEFAULT_ABBREV), + data->subject.buf); + } + return 0; +} + +static int squash_check_can_autosquash(struct repository *repo, + struct commit *commit, + struct subject_data *data, + unsigned flags) +{ + commit->object.flags |= flags & SQUASH_AMEND_TARGET; + if (starts_with(data->subject.buf, "amend!")) + return squash_amend_message(repo, commit, data, flags); + else if (starts_with(data->subject.buf, "squash!")) + return squash_squash_message(repo, commit, data, flags); + + return 0; +} + +static int squash_check_autosquash_subject(struct repository *repo, + struct commit *commit, + struct subject_data *data) +{ + const char* s = data->subject.buf; + struct commit *target; + struct hashmap_iter iter; + struct strmap_entry *entry; + /* Try skipping autosquash prefixes one at a time to allow + * squashing + * a commit + * fixup! fixup! a commit + * + * where we may have started with + * a commit + * fixup! a commit + * fixup! fixup! a commit + * + * and squashed the first fixup separately from the second + */ + while (skip_one_autosquash_prefix(s, &s)) { + unsigned flags = strintmap_get(&data->subjects, s); + if (flags) + return squash_check_can_autosquash(repo, commit, data, flags); + } + /* + * Allow "fixup! ", but not "fixup! HEAD^" or + * "fixup! main". If the target is not being squshed check the subject + * to allow "fixup! abc123" and "fixup! " to be + * squashed together. + */ + target = lookup_commit_reference_by_name(s); + if (target && istarts_with(oid_to_hex(&target->object.oid), s)) { + unsigned flags = + target->object.flags & (SQUASH_SEEN | SQUASH_AMEND_TARGET); + if (!flags) { + const char *subject_start; + const char *buffer = repo_logmsg_reencode(repo, target, + NULL, NULL); + size_t subject_len = find_commit_subject(buffer, + &subject_start); + char *subject = xmemdupz(subject_start, subject_len); + + flags = strintmap_get(&data->subjects, subject); + free(subject); + repo_unuse_commit_buffer(repo, target, buffer); + } + if (flags) + return squash_check_can_autosquash(repo, commit, + data, flags); + } + /* Try subject prefix matches */ + strintmap_for_each_entry(&data->subjects, &iter, entry) { + s = data->subject.buf; + while(skip_one_autosquash_prefix(s, &s)) { + if (starts_with(entry->key, s)) { + unsigned value = (intptr_t)entry->value; + + return squash_check_can_autosquash(repo, commit, + data, value); + } + } + } + return error(_("cannot squash %s (%s): its target is not being " + "squashed"), + repo_find_unique_abbrev(repo, &commit->object.oid, + DEFAULT_ABBREV), + data->subject.buf); +} + +static int squash_check_subject(struct repository *repo, + struct commit *commit, + struct subject_data *data) +{ + int ret = 0; + const char *buf = repo_logmsg_reencode(repo, commit, NULL, NULL); + size_t subject_len = find_commit_subject(buf, &data->message); + + strbuf_reset(&data->subject); + strbuf_add(&data->subject, data->message, subject_len); + + if (!strintmap_get_size(&data->subjects)) { + const char *s; + + strbuf_addstr(&data->squash_message, data->message); + strbuf_complete_line(&data->squash_message); + /* + * Strip a single autosquash prefix to allow squashing + * fixup! base + * amend! base + */ + s = data->subject.buf; + skip_one_autosquash_prefix(s, &s); + strintmap_set(&data->subjects, s, SQUASH_AMEND_TARGET | SQUASH_SEEN); + commit->object.flags |= SQUASH_AMEND_TARGET; + } else if (is_autosquash_subject(data->subject.buf)) { + ret = squash_check_autosquash_subject(repo, commit, data); + } else { + strintmap_set(&data->subjects, data->subject.buf, SQUASH_SEEN); + } + repo_unuse_commit_buffer(repo, commit, buf); + return ret; +} + +static int build_squash_message(struct repository *repo, + const struct strbuf *todo_buf, + struct strbuf *out); + +static int setup_squash_revisions(struct repository *repo, + int argc, const char **argv, + struct rev_info *revs) +{ + repo_init_revisions(repo, revs, NULL); + revs->reverse = 1; + revs->topo_order = 1; + revs->sort_order = REV_SORT_IN_GRAPH_ORDER; + revs->simplify_history = 0; + revs->ancestry_path = 1; + revs->limited = 1; + revs->ancestry_path_implicit_bottoms = 1; + + argc = setup_revisions(argc, argv, revs, NULL); + if (argc > 1) + return error(_("unrecognized argument: %s"), argv[1]); + + if (revs->reverse != 1 || revs->topo_order != 1 || + revs->sort_order != REV_SORT_IN_GRAPH_ORDER || + revs->simplify_history != 0 || revs->boundary == 1 || + revs->ancestry_path != 1 || revs->limited != 1 || + revs->ancestry_path_implicit_bottoms != 1) { + warning(_("ignoring rev-list options that would change how the " + "range is walked")); + revs->reverse = 1; + revs->topo_order = 1; + revs->sort_order = REV_SORT_IN_GRAPH_ORDER; + revs->simplify_history = 0; + revs->boundary = 0; + revs->ancestry_path = 1; + revs->limited = 1; + revs->ancestry_path_implicit_bottoms = 1; + } + + /* + * A squash needs a base to reparent onto, so the range has to exclude + * something, as in "..". A revision range with no such + * bottom commit cannot be squashed. + */ + for (size_t i = 0; i < revs->cmdline.nr; i++) + if (revs->cmdline.rev[i].flags & BOTTOM) + return 0; + + return error(_("not a '..' revision range")); +} + +/* + * Resolve a revision range into its oldest commit and single tip. Every + * parent after the oldest commit must either be selected or also be a parent + * of the oldest commit. + */ +static int resolve_squash_range(struct repository *repo, + bool update_branches, + bool edit_message, + int argc, const char **argv, + struct commit **oldest_out, + struct commit **tip_out, + char **message_out) +{ + struct rev_info revs; + struct subject_data subject_data = SUBJECT_DATA_INIT; + struct commit *commit, *oldest = NULL, *tip = NULL; + struct strbuf todo_buf = STRBUF_INIT; + int ret, tip_count = 0; + bool walk_started = false; + struct ref_filter filter = REF_FILTER_INIT; + struct ref_array refs = { 0 }; + + subject_data.edit_message = edit_message; + ret = setup_squash_revisions(repo, argc, argv, &revs); + if (ret < 0) + goto out; + + if (prepare_revision_walk(&revs) < 0) { + ret = error(_("error preparing revisions")); + goto out; + } + walk_started = true; + while ((commit = get_revision(&revs))) { + struct commit_list *p; + + if (edit_message) + strbuf_addf(&todo_buf, "pick %s\n", + oid_to_hex(&commit->object.oid)); + + if (!commit->parents) { + ret = error(_("cannot squash down to root commit")); + goto out; + } + for (p = commit->parents; oldest && p; p = p->next) { + struct commit_list *q; + struct object *o; + bool seen; + + if (repo_parse_commit(repo, p->item)) { + ret = error(_("cannot parse commit")); + goto out; + } + o = &p->item->object; + seen = o->flags & SQUASH_SEEN; + /* + * Allow parents that match the parents of the + * squashed commit. + */ + for (q = oldest->parents; !seen && q; q = q->next) { + if (p->item == q->item) { + seen = true; + commit_list_insert(commit, &filter.with_commit); + } + } + if (!seen) { + ret = error(_("parent %s of commit %s is " + "outside the revision range"), + repo_find_unique_abbrev(repo, &o->oid, + DEFAULT_ABBREV), + repo_find_unique_abbrev(repo, + &commit->object.oid, + DEFAULT_ABBREV)); + goto out; + } + if (o->flags & SQUASH_TIP) { + tip_count--; + o->flags &= ~SQUASH_TIP; + } + } + if (!oldest) { + commit_list_insert(commit, &filter.with_commit); + oldest = commit; + } + if (squash_check_subject(repo, commit, &subject_data)) { + ret = -1; + goto out; + } + tip = commit; + tip->object.flags |= SQUASH_SEEN | SQUASH_TIP; + tip_count++; + } + clear_object_flags(repo, SQUASH_SEEN | SQUASH_TIP); + reset_revision_walk(); + walk_started = false; + + if (!tip_count) { + ret = error(_("the revision range is empty")); + goto out; + } else if (tip_count != 1) { + ret = error(_("the revision range contains more than one tip " + "commit")); + goto out; + } else if (oldest == tip) { + ret = error(_("the revision range holds a single commit; " + "nothing to squash")); + goto out; + } else if (!oldest->parents) { + BUG("an in-range commit must have a parent"); + } + + commit_list_insert(tip, &filter.no_commit); + filter.kind = FILTER_REFS_BRANCHES; + if (update_branches && + filter_refs(&refs, &filter, filter.kind)) { + ret = error(_("could not filter refs")); + goto out; + } + if (refs.nr) { + struct ref_format format = REF_FORMAT_INIT; + struct ref_sorting *sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; + struct strbuf branches = STRBUF_INIT; + struct strbuf err = STRBUF_INIT; + + format.format = "%(refname:short)"; + if (verify_ref_format(&format)) + BUG("invalid branch format"); + string_list_append(&sorting_options, "refname"); + sorting = ref_sorting_options(&sorting_options); + ref_array_sort(sorting, &refs); + for (int i = 0; i < refs.nr; i++) { + strbuf_reset(&err); + strbuf_addstr(&branches, "\n "); + if (format_ref_array_item(refs.items[i], &format, + &branches, &err)) + BUG("could not format branch name: %s", err.buf); + } + /* + * TODO: also check HEADS from other worktrees. + */ + ret = error(_("the following branches cannot be rewritten as " + "descendants of the squashed commit:%s"), branches.buf); + advise_if_enabled(ADVICE_HISTORY_UPDATE_REFS, + _("Use --update-refs=head to rewrite only " + "the current branch and leave such branches " + "untouched.")); + strbuf_release(&err); + strbuf_release(&branches); + ref_sorting_release(sorting); + string_list_clear(&sorting_options, 0); + goto out; + } + if (edit_message) { + strbuf_reset(&subject_data.squash_message); + ret = build_squash_message(repo, &todo_buf, + &subject_data.squash_message); + if (ret < 0) + goto out; + } + + *oldest_out = oldest; + *tip_out = tip; + *message_out = strbuf_detach(&subject_data.squash_message, NULL); + ret = 0; + +out: + strbuf_release(&todo_buf); + clear_object_flags(repo, SQUASH_SEEN | SQUASH_TIP | + SQUASH_AMEND_TARGET); + if (walk_started) + reset_revision_walk(); + subject_data_clear(&subject_data); + release_revisions(&revs); + ref_filter_clear(&filter); + ref_array_clear(&refs); + return ret; +} + +static bool amend_replaces_target(struct todo_list *todo, int target) +{ + for (int i = target + 1; i < todo->nr && + todo->items[i].command != TODO_PICK; i++) { + if (todo->items[i].command == TODO_SQUASH) + return false; + if (todo->items[i].flags & TODO_REPLACE_FIXUP_MSG) + return true; + } + return false; +} + +static int build_squash_message(struct repository *repo, + const struct strbuf *todo_buf, + struct strbuf *out) +{ + struct todo_list todo = TODO_LIST_INIT; + struct replay_opts opts = REPLAY_OPTS_INIT; + int nr_commits, ret; + + if (todo_list_parse_insn_buffer(repo, &opts, todo_buf->buf, &todo) < 0 || + todo_list_rearrange_squash(&todo) < 0) { + ret = error(_("could not prepare the squash message")); + goto out; + } + + nr_commits = todo.nr; + for (int i = 0; i < nr_commits; i++) { + struct todo_item *item = &todo.items[i]; + const char *message, *body; + size_t commented_len; + bool skip, squashing; + + squashing = item->command == TODO_SQUASH || + (item->flags & TODO_REPLACE_FIXUP_MSG); + if (item->command == TODO_PICK) + skip = amend_replaces_target(&todo, i); + else + skip = !squashing; + + message = repo_logmsg_reencode(repo, item->commit, NULL, NULL); + find_commit_subject(message, &body); + + if (skip) + commented_len = strlen(body); + else if (squashing) + commented_len = squash_subject_comment_len(body, 1); + else + commented_len = 0; + + if (!i) + add_squash_combination_header(out, nr_commits); + strbuf_addch(out, '\n'); + add_squash_message_header(out, i + 1, skip); + strbuf_addstr(out, "\n\n"); + strbuf_add_commented_lines(out, body, commented_len, comment_line_str); + strbuf_addstr(out, body + commented_len); + strbuf_complete_line(out); + + repo_unuse_commit_buffer(repo, item->commit, message); + } + + ret = 0; + +out: + todo_list_release(&todo); + replay_opts_release(&opts); + return ret; +} + +static int cmd_history_squash(int argc, + const char **argv, + const char *prefix, + struct repository *repo) +{ + const char * const usage[] = { + GIT_HISTORY_SQUASH_USAGE, + NULL, + }; + enum ref_action action = REF_ACTION_DEFAULT; + int dry_run = 0; + int edit = 1; + struct option options[] = { + OPT_CALLBACK_F(0, "update-refs", &action, "(branches|head)", + N_("control which refs should be updated"), + PARSE_OPT_NONEG, parse_ref_action), + OPT_BOOL('n', "dry-run", &dry_run, + N_("perform a dry-run without updating any refs")), + OPT_BOOL('e', "edit", &edit, + N_("edit the commit message")), + OPT_END(), + }; + struct strbuf reflog_msg = STRBUF_INIT; + struct commit *oldest, *tip, *rewritten; + const struct object_id *base_tree_oid, *tip_tree_oid; + char *message_template = NULL; + struct rev_info revs = { 0 }; + int ret; + + argc = parse_options(argc, argv, prefix, options, usage, + PARSE_OPT_KEEP_UNKNOWN_OPT | PARSE_OPT_KEEP_ARGV0); + if (argc < 2) { + ret = error(_("command expects a revision range")); + goto out; + } + repo_config(repo, git_default_config, NULL); + + if (action == REF_ACTION_DEFAULT) + action = REF_ACTION_BRANCHES; + + strbuf_addstr(&reflog_msg, "squash: updating "); + strbuf_join_argv(&reflog_msg, argc - 1, argv + 1, ' '); + + ret = resolve_squash_range(repo, action == REF_ACTION_BRANCHES, + edit, + argc, argv, &oldest, &tip, + &message_template); + if (ret < 0) + goto out; + + ret = setup_revwalk(repo, action, tip, &revs); + if (ret < 0) + goto out; + + base_tree_oid = &repo_get_commit_tree(repo, + oldest->parents->item)->object.oid; + tip_tree_oid = &repo_get_commit_tree(repo, tip)->object.oid; + + ret = commit_tree_ext(repo, "squash", oldest, message_template, + oldest->parents, base_tree_oid, tip_tree_oid, + &rewritten, + edit ? COMMIT_TREE_EDIT_MESSAGE : 0); + if (ret < 0) { + ret = error(_("failed writing squashed commit")); + goto out; + } + + ret = handle_reference_updates(&revs, action, tip, rewritten, + reflog_msg.buf, dry_run, + REPLAY_EMPTY_COMMIT_ABORT); + if (ret < 0) { + ret = error(_("failed replaying descendants")); + goto out; + } + + ret = 0; + +out: + strbuf_release(&reflog_msg); + release_revisions(&revs); + free(message_template); + return ret; +} + static int update_worktree(struct repository *repo, const struct commit *old_head, const struct commit *new_head, @@ -1194,6 +1836,7 @@ int cmd_history(int argc, GIT_HISTORY_FIXUP_USAGE, GIT_HISTORY_REWORD_USAGE, GIT_HISTORY_SPLIT_USAGE, + GIT_HISTORY_SQUASH_USAGE, NULL, }; parse_opt_subcommand_fn *fn = NULL; @@ -1202,6 +1845,7 @@ int cmd_history(int argc, OPT_SUBCOMMAND("fixup", &fn, cmd_history_fixup), OPT_SUBCOMMAND("reword", &fn, cmd_history_reword), OPT_SUBCOMMAND("split", &fn, cmd_history_split), + OPT_SUBCOMMAND("squash", &fn, cmd_history_squash), OPT_END(), }; diff --git a/builtin/index-pack.c b/builtin/index-pack.c index bc86925ad04340..44ac72ef1e622f 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -763,7 +763,7 @@ static void find_ref_delta_children(const struct object_id *oid, struct compare_data { struct object_entry *entry; - struct odb_read_stream *st; + struct odb_stream *st; unsigned char *buf; unsigned long buf_size; }; @@ -780,7 +780,7 @@ static int compare_objects(const unsigned char *buf, unsigned long size, } while (size) { - ssize_t len = odb_read_stream_read(data->st, data->buf, size); + ssize_t len = odb_stream_read(data->st, data->buf, size); if (len == 0) die(_("SHA1 COLLISION FOUND WITH %s !"), oid_to_hex(&data->entry->idx.oid)); @@ -806,14 +806,14 @@ static int check_collison(struct object_entry *entry) memset(&data, 0, sizeof(data)); data.entry = entry; - data.st = odb_read_stream_open(the_repository->objects, &entry->idx.oid, NULL); + data.st = odb_stream_from_object(the_repository->objects, &entry->idx.oid, NULL); if (!data.st) return -1; if (data.st->size != entry->size || data.st->type != entry->type) die(_("SHA1 COLLISION FOUND WITH %s !"), oid_to_hex(&entry->idx.oid)); unpack_data(entry, compare_objects, &data); - odb_read_stream_close(data.st); + odb_stream_close(data.st); free(data.buf); return 0; } @@ -1866,7 +1866,7 @@ static void repack_local_links(void) while (strbuf_getline_lf(&line, out) != EOF) { unsigned char binary[GIT_MAX_RAWSZ]; if (line.len != the_hash_algo->hexsz || - !hex_to_bytes(binary, line.buf, line.len)) + !hex_to_bytes(binary, line.buf, line.len, HEX_KIND_MIXED)) die(_("index-pack: Expecting full hex object ID lines only from pack-objects.")); /* @@ -1886,7 +1886,7 @@ static void repack_local_links(void) int cmd_index_pack(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { int i, fix_thin_pack = 0, verify = 0, stat_only = 0, rev_index; const char *curr_index; @@ -1903,15 +1903,15 @@ int cmd_index_pack(int argc, int report_end_of_input = 0; int hash_algo = 0; + show_usage_if_asked(argc, argv, index_pack_usage); + /* * index-pack never needs to fetch missing objects except when * REF_DELTA bases are missing (which are explicitly handled). It only * accesses the repo to do hash collision checks and to check which * REF_DELTA bases need to be fetched. */ - fetch_if_missing = 0; - - show_usage_if_asked(argc, argv, index_pack_usage); + (repo ? repo : the_repository)->fetch_if_missing = 0; disable_replace_refs(); diff --git a/builtin/last-modified.c b/builtin/last-modified.c index 5478182f2e95c2..fe14f4493c5743 100644 --- a/builtin/last-modified.c +++ b/builtin/last-modified.c @@ -272,6 +272,18 @@ static bool maybe_changed_path(struct last_modified *lm, if (!filter) return true; + /* + * With --show-trees we also track the tree entries containing the + * paths, so a change to any of those parent directories matters too. + */ + if (lm->show_trees) { + if (!revs_maybe_changed_in_bloom_with_parents(&lm->rev, filter)) + return false; + } else { + if (!revs_maybe_changed_in_bloom(&lm->rev, filter)) + return false; + } + hashmap_for_each_entry(&lm->paths, &iter, ent, hashent) { if (active && !bitmap_get(active, ent->diff_idx)) continue; @@ -290,7 +302,8 @@ static void process_parent(struct last_modified *lm, { struct bitmap *active_p; - repo_parse_commit(lm->rev.repo, parent); + if (repo_parse_commit(lm->rev.repo, parent)) + return; active_p = active_paths_for(lm, parent); /* @@ -357,6 +370,14 @@ static int last_modified_run(struct last_modified *lm) prepare_revision_walk(&lm->rev); + /* + * prepare_revision_walk() clears bloom_filter_settings for pathspecs + * without a Bloom key. Restore it so the per-path check keeps working. + */ + if (!lm->rev.bloom_filter_settings) + lm->rev.bloom_filter_settings = + get_bloom_filter_settings(lm->rev.repo); + max_count = lm->rev.max_count; init_active_paths_for_commit(&lm->active_paths); @@ -414,12 +435,14 @@ static int last_modified_run(struct last_modified *lm) * Otherwise, make sure that 'c' isn't reachable from anything * in the '--not' queue. */ - repo_parse_commit(lm->rev.repo, c); + if (repo_parse_commit(lm->rev.repo, c)) + goto cleanup; while ((n = prio_queue_get(¬_queue))) { struct commit_list *np; - repo_parse_commit(lm->rev.repo, n); + if (repo_parse_commit(lm->rev.repo, n)) + continue; for (np = n->parents; np; np = np->next) { if (!(np->item->object.flags & PARENT2)) { diff --git a/builtin/log.c b/builtin/log.c index 350b35c556362d..59ecd34334b3bd 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -209,6 +209,7 @@ static void cmd_log_init_defaults(struct rev_info *rev, init_diffstat_widths(&rev->diffopt); rev->diffopt.flags.recursive = 1; rev->diffopt.flags.allow_textconv = 1; + rev->diffopt.flags.allow_diff_process = 1; rev->abbrev_commit = cfg->default_abbrev_commit; rev->show_root_diff = cfg->default_show_root; rev->subject_prefix = cfg->fmt_patch_subject_prefix; @@ -693,8 +694,11 @@ int cmd_show(int argc, opt.tweak = show_setup_revisions_tweak; cmd_log_init(argc, argv, prefix, &rev, &opt, &cfg); + diff_hunks_attach(&rev.diffopt); + if (!rev.no_walk) { ret = cmd_log_walk(&rev); + diff_hunks_detach(&rev.diffopt); release_revisions(&rev); log_config_release(&cfg); return ret; @@ -767,6 +771,7 @@ int cmd_show(int argc, } rev.diffopt.no_free = 0; + diff_hunks_detach(&rev.diffopt); diff_free(&rev.diffopt); release_revisions(&rev); log_config_release(&cfg); @@ -846,8 +851,11 @@ int cmd_log(int argc, opt.tweak = log_setup_revisions_tweak; cmd_log_init(argc, argv, prefix, &rev, &opt, &cfg); + diff_hunks_attach(&rev.diffopt); + ret = cmd_log_walk(&rev); + diff_hunks_detach(&rev.diffopt); release_revisions(&rev); log_config_release(&cfg); return ret; @@ -2218,6 +2226,17 @@ int cmd_format_patch(int argc, if (argc > 1) die(_("unrecognized argument: %s"), argv[1]); + /* + * Patches generated by format-patch must be based on the builtin + * diff so recipients without the store or the process can apply + * them, and so the emitted diffstat does not depend on the sender's + * local cache: the precomputed-hunks store is not consulted for the + * diffstat, and the diff process is not consulted even when + * --ext-diff enables the external diff command. + */ + rev.diffopt.flags.no_precomputed_hunks = 1; + rev.diffopt.flags.allow_diff_process = 0; + if (rev.diffopt.output_format & DIFF_FORMAT_NAME) die(_("--name-only does not make sense")); if (rev.diffopt.output_format & DIFF_FORMAT_NAME_STATUS) diff --git a/builtin/merge.c b/builtin/merge.c index 5b4eb23a833295..e21b0d8ffbc5c7 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -538,7 +538,7 @@ static void finish(struct commit *head_commit, if (new_head) apply_autostash_ref(the_repository, "MERGE_AUTOSTASH", - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); strbuf_release(&reflog_message); } @@ -1682,7 +1682,7 @@ int cmd_merge(int argc, &commit->object.oid, overwrite_ignore)) { apply_autostash_ref(the_repository, "MERGE_AUTOSTASH", - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); ret = 1; goto done; } @@ -1846,7 +1846,7 @@ int cmd_merge(int argc, fprintf(stderr, _("Merge with strategy %s failed.\n"), use_strategies[0]->name); apply_autostash_ref(the_repository, "MERGE_AUTOSTASH", - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); ret = 2; goto done; } else if (best_strategy == wt_strategy) diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 6e73c85cde324f..84d2467cc7520c 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -225,7 +225,8 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, } ret = write_midx_file(source->packed, opts.preferred_pack, - opts.refs_snapshot, opts.flags); + opts.refs_snapshot, opts.incremental_base, + opts.flags); free(opts.refs_snapshot); return ret; diff --git a/builtin/name-rev.c b/builtin/name-rev.c index 60cbbfb4b7d190..0c9014ca594e2a 100644 --- a/builtin/name-rev.c +++ b/builtin/name-rev.c @@ -21,6 +21,7 @@ #include "revision.h" #include "notes.h" #include "write-or-die.h" +#include "date.h" /* * One day. See the 'name a rev shortly after epoch' test in t6120 when @@ -772,16 +773,21 @@ int cmd_name_rev(int argc, return 0; } -struct format_nul_data { +struct format_rev_data { + const char *format; + const char *stdin_mode; bool nul_input; bool nul_output; + struct string_list notes; + struct rev_info rev; + int color; }; static int format_nul_cb(const struct option *option, const char *arg, int unset) { - struct format_nul_data *data = option->value; + struct format_rev_data *data = option->value; data->nul_input = 1; data->nul_output = 1; BUG_ON_OPT_NEG(unset); @@ -789,6 +795,17 @@ static int format_nul_cb(const struct option *option, return 0; } +static int date_cb(const struct option *option, + const char *arg, + int unset) +{ + struct rev_info *data = option->value; + parse_date_format(arg, &data->date_mode); + data->date_mode_explicit = 1; + BUG_ON_OPT_NEG(unset); + return 0; +} + static enum stdin_mode parse_stdin_mode(const char *stdin_mode) { if (!strcmp(stdin_mode, "text")) @@ -802,9 +819,8 @@ static enum stdin_mode parse_stdin_mode(const char *stdin_mode) } static char const *const format_rev_usage[] = { - N_("(EXPERIMENTAL!) git format-rev --stdin-mode= " - "--format= [--[no-]notes=] " - "[-z] [--[no-]null-output] [--[no-]null-input]"), + N_("(EXPERIMENTAL!) git format-rev [] " + "--stdin-mode= --format="), NULL }; @@ -813,32 +829,36 @@ int cmd_format_rev(int argc, const char *prefix, struct repository *repo UNUSED) { - const char *format = NULL; + struct format_rev_data data = { + NULL, NULL, 0, 0, STRING_LIST_INIT_NODUP, + REV_INFO_INIT, GIT_COLOR_AUTO + }; enum stdin_mode stdin_mode; - const char *stdin_mode_arg = NULL; - struct format_nul_data nul_data = { 0, 0 }; char output_terminator; strbuf_getline_fn getline_fn; struct display_notes_opt format_notes_opt; - struct rev_info format_rev = REV_INFO_INIT; struct pretty_format format_pp = { 0 }; - struct string_list notes = STRING_LIST_INIT_NODUP; struct strbuf scratch_buf = STRBUF_INIT; struct command cmd; struct option opts[] = { - OPT_STRING(0, "format", &format, N_("format"), + OPT_STRING(0, "format", &data.format, N_("format"), N_("pretty format to use")), - OPT_STRING(0, "stdin-mode", &stdin_mode_arg, N_("stdin-mode"), + OPT_STRING(0, "stdin-mode", &data.stdin_mode, N_("stdin-mode"), N_("how revs are processed")), - OPT_STRING_LIST(0, "notes", ¬es, N_("notes"), + OPT_STRING_LIST(0, "notes", &data.notes, N_("notes"), N_("display notes for pretty format")), - OPT_CALLBACK_F('z', "null", &nul_data, N_("z"), - N_("Use NUL for input and output termination"), + OPT__ABBREV(&data.rev.abbrev), + OPT__COLOR(&data.color, N_("use colored output")), + OPT_CALLBACK_F(0, "date", &data.rev, N_("date"), + N_("date format"), + PARSE_OPT_NONEG, date_cb), + OPT_CALLBACK_F('z', "null", &data, N_("z"), + N_("use NUL for input and output termination"), PARSE_OPT_NOARG | PARSE_OPT_NONEG, format_nul_cb), - OPT_BOOL(0, "null-input", &nul_data.nul_input, - N_("Use NUL for input termination")), - OPT_BOOL(0, "null-output", &nul_data.nul_output, - N_("Use NUL for output termination")), + OPT_BOOL(0, "null-input", &data.nul_input, + N_("use NUL for input termination")), + OPT_BOOL(0, "null-output", &data.nul_output, + N_("use NUL for output termination")), OPT_END(), }; @@ -849,32 +869,32 @@ int cmd_format_rev(int argc, usage_with_options(format_rev_usage, opts); } - if (!format) + if (!data.format) die(_("'%s' is required"), "--format"); - if (!stdin_mode_arg) + if (!data.stdin_mode) die(_("'%s' is required"), "--stdin-mode"); - getline_fn = nul_data.nul_input ? strbuf_getline_nul : strbuf_getline_lf; - output_terminator = nul_data.nul_output ? '\0' : '\n'; + getline_fn = data.nul_input ? strbuf_getline_nul : strbuf_getline_lf; + output_terminator = data.nul_output ? '\0' : '\n'; init_display_notes(&format_notes_opt); - stdin_mode = parse_stdin_mode(stdin_mode_arg); + stdin_mode = parse_stdin_mode(data.stdin_mode); - get_commit_format(format, &format_rev); - format_pp.ctx.rev = &format_rev; - format_pp.ctx.fmt = format_rev.commit_format; - format_pp.ctx.abbrev = format_rev.abbrev; - format_pp.ctx.date_mode_explicit = format_rev.date_mode_explicit; - format_pp.ctx.date_mode = format_rev.date_mode; - format_pp.ctx.color = GIT_COLOR_AUTO; + get_commit_format(data.format, &data.rev); + format_pp.ctx.rev = &data.rev; + format_pp.ctx.fmt = data.rev.commit_format; + format_pp.ctx.abbrev = data.rev.abbrev; + format_pp.ctx.date_mode_explicit = data.rev.date_mode_explicit; + format_pp.ctx.date_mode = data.rev.date_mode; + format_pp.ctx.color = data.color; - userformat_find_requirements(format, + userformat_find_requirements(data.format, &format_pp.want); if (format_pp.want.notes) { int ignore_show_notes = 0; struct string_list_item *n; - for_each_string_list_item(n, ¬es) + for_each_string_list_item(n, &data.notes) enable_ref_display_notes(&format_notes_opt, &ignore_show_notes, n->string); @@ -934,7 +954,7 @@ int cmd_format_rev(int argc, } strbuf_release(&scratch_buf); - string_list_clear(¬es, 0); + string_list_clear(&data.notes, 0); release_display_notes(&format_notes_opt); return 0; } diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 1ec5b6f206366e..11f53f3043d622 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -261,8 +261,8 @@ static int exclude_promisor_objects_best_effort; static int use_delta_islands; -static unsigned long delta_cache_size = 0; -static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE; +static size_t delta_cache_size = 0; +static size_t max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE; static unsigned long cache_max_small_delta_size = 1000; static unsigned long window_memory_limit = 0; @@ -354,7 +354,8 @@ static void index_commit_for_bitmap(struct commit *commit) static void *get_delta(struct object_entry *entry) { - unsigned long size, base_size, delta_size; + unsigned long size, base_size; + size_t delta_size; void *buf, *base_buf, *delta_buf; enum object_type type; size_t size_st = 0, base_size_st = 0; @@ -411,7 +412,7 @@ static unsigned long do_compress(void **pptr, unsigned long size) return stream.total_out; } -static unsigned long write_large_blob_data(struct odb_read_stream *st, struct hashfile *f, +static unsigned long write_large_blob_data(struct odb_stream *st, struct hashfile *f, const struct object_id *oid) { git_zstream stream; @@ -425,7 +426,7 @@ static unsigned long write_large_blob_data(struct odb_read_stream *st, struct ha for (;;) { ssize_t readlen; int zret = Z_OK; - readlen = odb_read_stream_read(st, ibuf, sizeof(ibuf)); + readlen = odb_stream_read(st, ibuf, sizeof(ibuf)); if (readlen == -1) die(_("unable to read %s"), oid_to_hex(oid)); @@ -488,7 +489,7 @@ static void copy_pack_data(struct hashfile *f, off_t len) { unsigned char *in; - unsigned long avail; + size_t avail; while (len) { in = use_pack(p, w_curs, offset, &avail); @@ -521,15 +522,15 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent unsigned hdrlen; enum object_type type; void *buf; - struct odb_read_stream *st = NULL; + struct odb_stream *st = NULL; const unsigned hashsz = the_hash_algo->rawsz; if (!usable_delta) { if (oe_type(entry) == OBJ_BLOB && oe_size_greater_than(&to_pack, entry, repo_settings_get_big_file_threshold(the_repository)) && - (st = odb_read_stream_open(the_repository->objects, &entry->idx.oid, - NULL)) != NULL) { + (st = odb_stream_from_object(the_repository->objects, &entry->idx.oid, + NULL)) != NULL) { buf = NULL; type = st->type; size = st->size; @@ -589,7 +590,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent dheader[--pos] = 128 | (--ofs & 127); if (limit && hdrlen + sizeof(dheader) - pos + datalen + hashsz >= limit) { if (st) - odb_read_stream_close(st); + odb_stream_close(st); free(buf); return 0; } @@ -603,7 +604,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent */ if (limit && hdrlen + hashsz + datalen + hashsz >= limit) { if (st) - odb_read_stream_close(st); + odb_stream_close(st); free(buf); return 0; } @@ -613,7 +614,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent } else { if (limit && hdrlen + datalen + hashsz >= limit) { if (st) - odb_read_stream_close(st); + odb_stream_close(st); free(buf); return 0; } @@ -621,7 +622,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent } if (st) { datalen = write_large_blob_data(st, f, &entry->idx.oid); - odb_read_stream_close(st); + odb_stream_close(st); } else { hashwrite(f, buf, datalen); free(buf); @@ -1183,12 +1184,12 @@ static size_t write_reused_pack_verbatim(struct bitmapped_pack *reuse_packfile, size_t pos = 0; size_t end; - if (reuse_packfile->bitmap_pos) { + if (reuse_packfile->bitmap_pos || + reuse_packfile->bitmap_nr != reuse_packfile->p->num_objects) { /* - * We can't reuse whole chunks verbatim out of - * non-preferred packs since we can't guarantee that - * all duplicate objects were resolved in favor of - * that pack. + * We can't reuse whole chunks verbatim from non-preferred + * packs or packs with entries missing from the bitmap because + * bitmap and pack positions may differ. * * Even if we have a whole eword_t worth of bits that * could be reused, there may be objects between the @@ -1197,7 +1198,7 @@ static size_t write_reused_pack_verbatim(struct bitmapped_pack *reuse_packfile, * pack, causing us to send duplicate or unwanted * objects. * - * Handle non-preferred packs from within + * Handle these packs from within * write_reused_pack(), which inspects and reuses * individual bits. */ @@ -1264,20 +1265,18 @@ static void write_reused_pack(struct bitmapped_pack *reuse_packfile, if (pos + offset >= reuse_packfile->bitmap_pos + reuse_packfile->bitmap_nr) goto done; - if (reuse_packfile->bitmap_pos) { + if (reuse_packfile->bitmap_pos || + reuse_packfile->bitmap_nr != reuse_packfile->p->num_objects) { /* - * When doing multi-pack reuse on a - * non-preferred pack, translate bit positions - * from the MIDX pseudo-pack order back to their - * pack-relative positions before attempting - * reuse. + * Translate MIDX bitmap positions which do not + * correspond directly to physical pack positions. */ struct multi_pack_index *m = reuse_packfile->from_midx; uint32_t midx_pos; off_t pack_ofs; if (!m) - BUG("non-zero bitmap position without MIDX"); + BUG("cannot translate bitmap position without MIDX"); midx_pos = pack_pos_to_midx(m, pos + offset); pack_ofs = nth_midxed_offset(m, midx_pos); @@ -1779,8 +1778,6 @@ static int want_object_in_pack_mtime(const struct object_id *oid, *found_offset = 0; } - odb_prepare_alternates(the_repository->objects); - for (source = the_repository->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); struct multi_pack_index *m = get_multi_pack_index(files->packed); @@ -2259,8 +2256,7 @@ static void check_object(struct object_entry *entry, uint32_t object_index) int have_base = 0; struct object_id base_ref; struct object_entry *base_entry; - unsigned long used, used_0; - unsigned long avail; + size_t used, used_0, avail; off_t ofs; unsigned char *buf, c; enum object_type type; @@ -2688,8 +2684,8 @@ struct unpacked { unsigned depth; }; -static int delta_cacheable(unsigned long src_size, unsigned long trg_size, - unsigned long delta_size) +static int delta_cacheable(size_t src_size, size_t trg_size, + size_t delta_size) { if (max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size) return 0; @@ -2772,8 +2768,7 @@ size_t oe_get_size_slow(struct packing_data *pack, struct pack_window *w_curs; unsigned char *buf; enum object_type type; - unsigned long used, avail; - size_t size; + size_t used, avail, size; if (e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) { size_t sz; @@ -2804,11 +2799,12 @@ size_t oe_get_size_slow(struct packing_data *pack, } static int try_delta(struct unpacked *trg, struct unpacked *src, - unsigned max_depth, unsigned long *mem_usage) + unsigned max_depth, size_t *mem_usage) { struct object_entry *trg_entry = trg->entry; struct object_entry *src_entry = src->entry; - unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz; + unsigned long trg_size, src_size, sizediff, max_size, sz; + size_t delta_size; unsigned ref_depth; enum object_type type; void *delta_buf; @@ -2972,9 +2968,9 @@ static unsigned int check_delta_limit(struct object_entry *me, unsigned int n) return m; } -static unsigned long free_unpacked(struct unpacked *n) +static size_t free_unpacked(struct unpacked *n) { - unsigned long freed_mem = sizeof_delta_index(n->index); + size_t freed_mem = sizeof_delta_index(n->index); free_delta_index(n->index); n->index = NULL; if (n->data) { @@ -2991,7 +2987,7 @@ static void find_deltas(struct object_entry **list, unsigned *list_size, { uint32_t i, idx = 0, count = 0; struct unpacked *array; - unsigned long mem_usage = 0; + size_t mem_usage = 0; CALLOC_ARRAY(array, window); @@ -3701,7 +3697,7 @@ static int git_pack_config(const char *k, const char *v, return 0; } if (!strcmp(k, "pack.deltacachesize")) { - max_delta_cache_size = git_config_int(k, v, ctx->kvi); + max_delta_cache_size = git_config_size_t(k, v, ctx->kvi); return 0; } if (!strcmp(k, "pack.deltacachelimit")) { @@ -4087,9 +4083,10 @@ static void stdin_packs_read_input(struct rev_info *revs, static void add_unreachable_loose_objects(struct rev_info *revs); -static void read_stdin_packs(enum stdin_packs_mode mode, int rev_list_unpacked) +static void read_stdin_packs(struct repository *repo, + enum stdin_packs_mode mode, int rev_list_unpacked) { - int prev_fetch_if_missing = fetch_if_missing; + int prev_fetch_if_missing = repo->fetch_if_missing; struct rev_info revs; /* @@ -4097,9 +4094,9 @@ static void read_stdin_packs(enum stdin_packs_mode mode, int rev_list_unpacked) * walk is best-effort though we don't want to perform backfill fetches * for them. */ - fetch_if_missing = 0; + repo->fetch_if_missing = 0; - repo_init_revisions(the_repository, &revs, NULL); + repo_init_revisions(repo, &revs, NULL); /* * Use a revision walk to fill in the namehash of objects in the include * packs. To save time, we'll avoid traversing through objects that are @@ -4145,7 +4142,7 @@ static void read_stdin_packs(enum stdin_packs_mode mode, int rev_list_unpacked) trace2_data_intmax("pack-objects", the_repository, "stdin_packs_hints", stdin_packs_hints_nr); - fetch_if_missing = prev_fetch_if_missing; + repo->fetch_if_missing = prev_fetch_if_missing; } static void add_cruft_object_entry(const struct object_id *oid, enum object_type type, @@ -4455,9 +4452,11 @@ static void show_object__ma_allow_promisor(struct object *obj, const char *name, show_object(obj, name, data); } -static int option_parse_missing_action(const struct option *opt UNUSED, +static int option_parse_missing_action(const struct option *opt, const char *arg, int unset) { + struct repository *repo = opt->value; + assert(arg); assert(!unset); @@ -4469,14 +4468,14 @@ static int option_parse_missing_action(const struct option *opt UNUSED, if (!strcmp(arg, "allow-any")) { arg_missing_action = MA_ALLOW_ANY; - fetch_if_missing = 0; + repo->fetch_if_missing = 0; fn_show_object = show_object__ma_allow_any; return 0; } if (!strcmp(arg, "allow-promisor")) { arg_missing_action = MA_ALLOW_PROMISOR; - fetch_if_missing = 0; + repo->fetch_if_missing = 0; fn_show_object = show_object__ma_allow_promisor; return 0; } @@ -4520,7 +4519,6 @@ static void add_objects_in_unpacked_packs(void) .source_infop = &source_info, }; - odb_prepare_alternates(to_pack.repo->objects); for (source = to_pack.repo->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); @@ -5118,7 +5116,7 @@ static int parse_stdin_packs_mode(const struct option *opt, const char *arg, int cmd_pack_objects(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { int use_internal_rev_list = 0; int all_progress_implied = 0; @@ -5225,7 +5223,7 @@ int cmd_pack_objects(int argc, N_("write a bitmap index if possible"), WRITE_BITMAP_QUIET, PARSE_OPT_HIDDEN), OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options), - OPT_CALLBACK_F(0, "missing", NULL, N_("action"), + OPT_CALLBACK_F(0, "missing", repo, N_("action"), N_("handling for missing objects"), PARSE_OPT_NONEG, option_parse_missing_action), OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects, @@ -5345,7 +5343,7 @@ int cmd_pack_objects(int argc, exclude_promisor_objects_best_effort, "--exclude-promisor-objects-best-effort"); if (exclude_promisor_objects) { - fetch_if_missing = 0; + repo->fetch_if_missing = 0; /* --stdin-packs handles promisor objects separately. */ if (!stdin_packs) { @@ -5354,8 +5352,9 @@ int cmd_pack_objects(int argc, } } else if (exclude_promisor_objects_best_effort) { use_internal_rev_list = 1; - fetch_if_missing = 0; - option_parse_missing_action(NULL, "allow-any", 0); + arg_missing_action = MA_ALLOW_ANY; + repo->fetch_if_missing = 0; + fn_show_object = show_object__ma_allow_any; /* revs configured below */ } if (unpack_unreachable || keep_unreachable || pack_loose_unreachable) @@ -5471,7 +5470,7 @@ int cmd_pack_objects(int argc, progress_state = start_progress(the_repository, _("Enumerating objects"), 0); if (stdin_packs) { - read_stdin_packs(stdin_packs, rev_list_unpacked); + read_stdin_packs(repo, stdin_packs, rev_list_unpacked); } else if (cruft) { read_cruft_objects(); } else if (!use_internal_rev_list) { diff --git a/builtin/prune.c b/builtin/prune.c index 55635a891f37d6..a7e4678d117072 100644 --- a/builtin/prune.c +++ b/builtin/prune.c @@ -194,7 +194,7 @@ int cmd_prune(int argc, if (show_progress == -1) show_progress = isatty(2); if (exclude_promisor_objects) { - fetch_if_missing = 0; + repo->fetch_if_missing = 0; revs.exclude_promisor_objects = 1; } diff --git a/builtin/rebase.c b/builtin/rebase.c index 10a306310cd439..5827b20bafce54 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -43,7 +43,7 @@ static char const * const builtin_rebase_usage[] = { "[--onto | --keep-base] [ []]"), N_("git rebase [-i] [options] [--exec ] [--onto ] " "--root []"), - "git rebase --continue | --abort | --skip | --edit-todo", + "git rebase --continue [--[no-]edit] | --abort | --skip | --edit-todo", NULL }; @@ -135,6 +135,8 @@ struct rebase_options { int config_autosquash; int config_rebase_merges; int config_update_refs; + int config_no_edit; + int edit; }; #define REBASE_OPTIONS_INIT { \ @@ -156,6 +158,8 @@ struct rebase_options { .update_refs = -1, \ .config_update_refs = -1, \ .strategy_opts = STRING_LIST_INIT_NODUP,\ + .config_no_edit = -1, \ + .edit = -1, \ } static void rebase_options_release(struct rebase_options *opts) @@ -215,6 +219,13 @@ static struct replay_opts get_replay_opts(const struct rebase_options *opts) replay.have_squash_onto = 1; } + if (opts->action == ACTION_CONTINUE) { + if (opts->edit >= 0) + replay.edit = opts->edit; + else if (opts->config_no_edit > 0) + replay.edit = 0; + } + return replay; } @@ -841,6 +852,11 @@ static int rebase_config(const char *var, const char *value, return 0; } + if (!strcmp(var, "rebase.noedit")) { + opts->config_no_edit = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "rebase.forkpoint")) { opts->fork_point = git_config_bool(var, value) ? -1 : 0; return 0; @@ -1171,6 +1187,8 @@ int cmd_rebase(int argc, ACTION_CONTINUE), OPT_CMDMODE(0, "skip", &options.action, N_("skip current patch and continue"), ACTION_SKIP), + OPT_BOOL('e', "edit", &options.edit, + N_("edit the commit message")), OPT_CMDMODE(0, "abort", &options.action, N_("abort and check out the original branch"), ACTION_ABORT), @@ -1311,10 +1329,15 @@ int cmd_rebase(int argc, "which is no longer supported; use 'merges' instead")); if (options.action != ACTION_NONE && total_argc != 2) { - usage_with_options(builtin_rebase_usage, - builtin_rebase_options); + if (options.action != ACTION_CONTINUE || + options.edit < 0 || total_argc != 3) + usage_with_options(builtin_rebase_usage, + builtin_rebase_options); } + if (options.edit >= 0 && options.action != ACTION_CONTINUE) + die(_("--edit and --no-edit can only be used with --continue")); + if (argc > 2) usage_with_options(builtin_rebase_usage, builtin_rebase_options); diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 86933d8d7e4b33..e6e54ba55f7a8c 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -15,7 +15,6 @@ #include "gpg-interface.h" #include "hex.h" #include "hook.h" -#include "lockfile.h" #include "object.h" #include "object-file.h" #include "object-name.h" @@ -23,7 +22,6 @@ #include "oid-array.h" #include "oidset.h" #include "pack.h" -#include "packfile.h" #include "parse-options.h" #include "pkt-line.h" #include "protocol.h" @@ -62,12 +60,9 @@ static enum deny_action deny_delete_current = DENY_UNCONFIGURED; static int receive_fsck_objects = -1; static int transfer_fsck_objects = -1; static struct strbuf fsck_msg_types = STRBUF_INIT; -static int receive_unpack_limit = -1; -static int transfer_unpack_limit = -1; static int advertise_atomic_push = 1; static int advertise_push_options; static int advertise_sid; -static int unpack_limit = 100; static off_t max_input_size; static int report_status; static int report_status_v2; @@ -86,7 +81,6 @@ static const char *head_name; static void *head_name_to_free; static int sent_capabilities; static int shallow_update; -static const char *alt_shallow_file; static struct strbuf push_cert = STRBUF_INIT; static struct object_id push_cert_oid; static struct signature_check sigcheck; @@ -158,16 +152,6 @@ static int receive_pack_config(const char *var, const char *value, return 0; } - if (strcmp(var, "receive.unpacklimit") == 0) { - receive_unpack_limit = git_config_int(var, value, ctx->kvi); - return 0; - } - - if (strcmp(var, "transfer.unpacklimit") == 0) { - transfer_unpack_limit = git_config_int(var, value, ctx->kvi); - return 0; - } - if (strcmp(var, "receive.fsck.skiplist") == 0) { char *path; @@ -2029,7 +2013,7 @@ static void execute_commands_atomic(struct command *commands, } static void execute_commands(struct command *commands, - const char *unpacker_error, + int unpacker_error, struct shallow_info *si, struct odb_transaction *transaction, const struct string_list *push_options) @@ -2306,150 +2290,44 @@ static void read_push_options(struct packet_reader *reader, } } -static const char *parse_pack_header(struct pack_header *hdr) -{ - switch (read_pack_header(0, hdr)) { - case PH_ERROR_EOF: - return "eof before pack header was fully read"; - - case PH_ERROR_PACK_SIGNATURE: - return "protocol error (pack signature mismatch detected)"; - - case PH_ERROR_PROTOCOL: - return "protocol error (pack version unsupported)"; - - default: - return "unknown error in parse_pack_header"; - - case 0: - return NULL; - } -} - -static struct tempfile *pack_lockfile; - -static void push_header_arg(struct strvec *args, struct pack_header *hdr) -{ - strvec_pushf(args, "--pack_header=%"PRIu32",%"PRIu32, - ntohl(hdr->hdr_version), ntohl(hdr->hdr_entries)); -} - -static const char *unpack(int err_fd, struct shallow_info *si, - struct odb_transaction *transaction) -{ - struct pack_header hdr; - const char *hdr_err; - int status; - struct child_process child = CHILD_PROCESS_INIT; - int fsck_objects = (receive_fsck_objects >= 0 - ? receive_fsck_objects - : transfer_fsck_objects >= 0 - ? transfer_fsck_objects - : 0); - - hdr_err = parse_pack_header(&hdr); - if (hdr_err) { - if (err_fd > 0) - close(err_fd); - return hdr_err; - } - - if (si->nr_ours || si->nr_theirs) { - alt_shallow_file = setup_temporary_shallow(si->shallow); - strvec_push(&child.args, "--shallow-file"); - strvec_push(&child.args, alt_shallow_file); - } - - odb_transaction_env(transaction, &child.env); - - if (ntohl(hdr.hdr_entries) < unpack_limit) { - strvec_push(&child.args, "unpack-objects"); - push_header_arg(&child.args, &hdr); - if (quiet) - strvec_push(&child.args, "-q"); - if (fsck_objects) - strvec_pushf(&child.args, "--strict%s", - fsck_msg_types.buf); - if (max_input_size) - strvec_pushf(&child.args, "--max-input-size=%"PRIuMAX, - (uintmax_t)max_input_size); - child.no_stdout = 1; - child.err = err_fd; - child.git_cmd = 1; - status = run_command(&child); - if (status) - return "unpack-objects abnormal exit"; - } else { - char hostname[HOST_NAME_MAX + 1]; - char *lockfile; - - strvec_pushl(&child.args, "index-pack", "--stdin", NULL); - push_header_arg(&child.args, &hdr); - - if (xgethostname(hostname, sizeof(hostname))) - xsnprintf(hostname, sizeof(hostname), "localhost"); - strvec_pushf(&child.args, - "--keep=receive-pack %"PRIuMAX" on %s", - (uintmax_t)getpid(), - hostname); - - if (!quiet && err_fd) - strvec_push(&child.args, "--show-resolving-progress"); - if (use_sideband) - strvec_push(&child.args, "--report-end-of-input"); - if (fsck_objects) - strvec_pushf(&child.args, "--strict%s", - fsck_msg_types.buf); - if (!reject_thin) - strvec_push(&child.args, "--fix-thin"); - if (max_input_size) - strvec_pushf(&child.args, "--max-input-size=%"PRIuMAX, - (uintmax_t)max_input_size); - child.out = -1; - child.err = err_fd; - child.git_cmd = 1; - status = start_command(&child); - if (status) - return "index-pack fork failed"; - - lockfile = index_pack_lockfile(the_repository, child.out, NULL); - if (lockfile) { - pack_lockfile = register_tempfile(lockfile); - free(lockfile); - } - close(child.out); - - status = finish_command(&child); - if (status) - return "index-pack abnormal exit"; - odb_reprepare(the_repository->objects); - } - return NULL; -} - -static const char *unpack_with_sideband(struct shallow_info *si, - struct odb_transaction *transaction) +static int unpack_with_sideband(struct odb_transaction *transaction, + const char *shallow_file, + struct strbuf *err_msg) { + struct odb_transaction_write_pack_opts opts = { + .fsck_objects = (receive_fsck_objects >= 0 + ? receive_fsck_objects + : transfer_fsck_objects >= 0 + ? transfer_fsck_objects + : 0), + .fsck_msg_types = fsck_msg_types.buf, + .max_input_size = max_input_size, + .shallow_file = shallow_file, + .reject_thin = reject_thin, + .quiet = quiet, + }; struct async muxer; - const char *ret; + int ret; if (!use_sideband) - return unpack(0, si, transaction); + return odb_transaction_write_pack(transaction, 0, err_msg, &opts); use_keepalive = KEEPALIVE_AFTER_NUL; memset(&muxer, 0, sizeof(muxer)); muxer.proc = copy_to_sideband; muxer.in = -1; if (start_async(&muxer)) - return NULL; + return 0; - ret = unpack(muxer.in, si, transaction); + opts.err_fd = muxer.in; + ret = odb_transaction_write_pack(transaction, 0, err_msg, &opts); finish_async(&muxer); return ret; } -static void prepare_shallow_update(struct shallow_info *si) +static void prepare_shallow_update(struct shallow_info *si, + const char *shallow_file) { int i, j, k, bitmap_size = DIV_ROUND_UP(si->ref->nr, 32); @@ -2489,12 +2367,13 @@ static void prepare_shallow_update(struct shallow_info *si) * command. check_connected() will be done with * true .git/shallow though. */ - setenv(GIT_SHALLOW_FILE_ENVIRONMENT, alt_shallow_file, 1); + setenv(GIT_SHALLOW_FILE_ENVIRONMENT, shallow_file, 1); } static void update_shallow_info(struct command *commands, struct shallow_info *si, - struct oid_array *ref) + struct oid_array *ref, + const char *shallow_file) { struct command *cmd; int *ref_status; @@ -2513,7 +2392,7 @@ static void update_shallow_info(struct command *commands, si->ref = ref; if (shallow_update) { - prepare_shallow_update(si); + prepare_shallow_update(si, shallow_file); return; } @@ -2530,13 +2409,13 @@ static void update_shallow_info(struct command *commands, free(ref_status); } -static void report(struct command *commands, const char *unpack_status) +static void report(struct command *commands, const struct strbuf *unpack_status) { struct command *cmd; struct strbuf buf = STRBUF_INIT; packet_buf_write(&buf, "unpack %s\n", - unpack_status ? unpack_status : "ok"); + unpack_status->len ? unpack_status->buf : "ok"); for (cmd = commands; cmd; cmd = cmd->next) { if (!cmd->error_string) packet_buf_write(&buf, "ok %s\n", @@ -2554,14 +2433,14 @@ static void report(struct command *commands, const char *unpack_status) strbuf_release(&buf); } -static void report_v2(struct command *commands, const char *unpack_status) +static void report_v2(struct command *commands, const struct strbuf *unpack_status) { struct command *cmd; struct strbuf buf = STRBUF_INIT; struct ref_push_report *report; packet_buf_write(&buf, "unpack %s\n", - unpack_status ? unpack_status : "ok"); + unpack_status->len ? unpack_status->buf : "ok"); for (cmd = commands; cmd; cmd = cmd->next) { int count = 0; @@ -2652,11 +2531,6 @@ int cmd_receive_pack(int argc, if (cert_nonce_seed) push_cert_nonce = prepare_push_cert_nonce(service_dir, time(NULL)); - if (0 <= receive_unpack_limit) - unpack_limit = receive_unpack_limit; - else if (0 <= transfer_unpack_limit) - unpack_limit = transfer_unpack_limit; - switch (determine_protocol_version_server()) { case protocol_v2: /* @@ -2690,8 +2564,8 @@ int cmd_receive_pack(int argc, PACKET_READ_DIE_ON_ERR_PACKET); if ((commands = read_head_info(&reader, &shallow))) { - const char *unpack_status = NULL; struct string_list push_options = STRING_LIST_INIT_DUP; + struct strbuf unpack_status = STRBUF_INIT; if (use_push_options) read_push_options(&reader, &push_options); @@ -2705,21 +2579,27 @@ int cmd_receive_pack(int argc, if (!si.nr_ours && !si.nr_theirs) shallow_update = 0; if (!delete_only(commands)) { + const char *alt_shallow_file = NULL; + + if (si.nr_ours || si.nr_theirs) + alt_shallow_file = setup_temporary_shallow(si.shallow); + if (odb_transaction_begin(the_repository->objects, &transaction, ODB_TRANSACTION_RECEIVE)) - unpack_status = "unable to start object transaction"; + strbuf_addstr(&unpack_status, "unable to start object transaction"); else - unpack_status = unpack_with_sideband(&si, transaction); - update_shallow_info(commands, &si, &ref); + unpack_with_sideband(transaction, alt_shallow_file, &unpack_status); + + update_shallow_info(commands, &si, &ref, alt_shallow_file); } use_keepalive = KEEPALIVE_ALWAYS; - execute_commands(commands, unpack_status, &si, transaction, + execute_commands(commands, !!unpack_status.len, &si, transaction, &push_options); - delete_tempfile(&pack_lockfile); + odb_transaction_finalize(transaction); sigchain_push(SIGPIPE, SIG_IGN); if (report_status_v2) - report_v2(commands, unpack_status); + report_v2(commands, &unpack_status); else if (report_status) - report(commands, unpack_status); + report(commands, &unpack_status); sigchain_pop(SIGPIPE); run_receive_hook(commands, "post-receive", 1, NULL, &push_options); @@ -2744,6 +2624,7 @@ int cmd_receive_pack(int argc, if (auto_update_server_info) update_server_info(the_repository, 0); clear_shallow_info(&si); + strbuf_release(&unpack_status); } if (use_sideband) packet_flush(1); diff --git a/builtin/repack.c b/builtin/repack.c index db504d673fcf52..2a4e1cf6a311bc 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -14,6 +14,11 @@ #include "promisor-remote.h" #include "repack.h" #include "shallow.h" +#include "list-objects-filter-options.h" +#include "oidset.h" +#include "hex.h" +#include "wt-status.h" +#include "read-cache-ll.h" #define ALL_INTO_ONE 1 #define LOOSEN_UNREACHABLE 2 @@ -28,11 +33,15 @@ static int use_delta_islands; static int run_update_server_info = 1; static char *packdir, *packtmp_name, *packtmp; static int midx_must_contain_cruft = 1; +static int drop_filtered; +static int dry_run; +static int write_bitmaps_given; static const char *const git_repack_usage[] = { N_("git repack [-a] [-A] [-d] [-f] [-F] [-l] [-n] [-q] [-b] [-m]\n" "[--window=] [--depth=] [--threads=] [--keep-pack=]\n" - "[--write-midx[=]] [--name-hash-version=] [--path-walk]"), + "[--write-midx[=]] [--name-hash-version=] [--path-walk]\n" + "[--filter=] [--drop-filtered [--dry-run]]"), NULL }; @@ -111,6 +120,21 @@ static int repack_config(const char *var, const char *value, return git_default_config(var, value, ctx, cb); } +static int option_parse_write_bitmaps(const struct option *opt, const char *arg, + int unset) +{ + int *value = opt->value; + + BUG_ON_OPT_ARG(arg); + if (unset) + *value = 0; + else + *value = 1; + + write_bitmaps_given = 1; + return 0; +} + static int option_parse_write_midx(const struct option *opt, const char *arg, int unset) { @@ -140,6 +164,7 @@ int cmd_repack(int argc, struct string_list_item *item; struct string_list names = STRING_LIST_INIT_DUP; struct existing_packs existing = EXISTING_PACKS_INIT; + struct oidset drop_oids = OIDSET_INIT; struct pack_geometry geometry = { 0 }; struct tempfile *refs_snapshot = NULL; int i, ret; @@ -194,8 +219,9 @@ int cmd_repack(int argc, OPT__QUIET(&po_args.quiet, N_("be quiet")), OPT_BOOL('l', "local", &po_args.local, N_("pass --local to git-pack-objects")), - OPT_BOOL('b', "write-bitmap-index", &write_bitmaps, - N_("write bitmap index")), + OPT_CALLBACK_F('b', "write-bitmap-index", &write_bitmaps, NULL, + N_("write bitmap index"), + PARSE_OPT_NOARG, option_parse_write_bitmaps), OPT_BOOL('i', "delta-islands", &use_delta_islands, N_("pass --delta-islands to git-pack-objects")), OPT_STRING(0, "unpack-unreachable", &unpack_unreachable, N_("approxidate"), @@ -231,6 +257,10 @@ int cmd_repack(int argc, N_("pack prefix to store a pack containing pruned objects")), OPT_STRING(0, "filter-to", &filter_to, N_("dir"), N_("pack prefix to store a pack containing filtered out objects")), + OPT_BOOL(0, "drop-filtered", &drop_filtered, + N_("delete filtered out objects (requires --filter)")), + OPT_BOOL(0, "dry-run", &dry_run, + N_("only show which objects would be dropped")), OPT_END() }; @@ -252,6 +282,118 @@ int cmd_repack(int argc, po_args.depth = xstrdup_or_null(opt_depth); po_args.threads = xstrdup_or_null(opt_threads); + die_for_incompatible_opt2(drop_filtered, "--drop-filtered", + !!filter_to, "--filter-to"); + + if (dry_run && !drop_filtered) + die(_("--dry-run only takes effect with --drop-filtered")); + + if (drop_filtered) { + if (!po_args.filter_options.choice) + die(_("--drop-filtered requires --filter")); + + if (!(pack_everything & ALL_INTO_ONE)) + die(_("--drop-filtered requires -a")); + + /* + * Only blob:limit= is supported for now. Reject other + * filter choices early, before walking the object database. + */ + if (po_args.filter_options.choice != LOFC_BLOB_LIMIT) + die(_("--drop-filtered only supports --filter=blob:limit= for now")); + + /* + * An explicit -b on the command line is a conflict we have to + * report; a bitmap setting from config is silently overridden + * for the duration of the command. + */ + if (write_bitmaps_given && write_bitmaps > 0) + die(_("options '%s' and '%s' cannot be used together"), + "--drop-filtered", "--write-bitmap-index"); + + /* + * Without a promisor remote there is nowhere to re-fetch the + * dropped objects from, so dropping them would be permanent + * data loss. + */ + if (!repo_has_promisor_remote(repo)) + die(_("--drop-filtered requires a promisor remote")); + + /* + * Refuse to run while another operation is in progress. A + * dropped object would just be lazily re-fetched when the + * operation resumes, but triggering a network fetch in the + * middle of a half-finished + * merge/rebase/cherry-pick/revert/bisect is a poor + * experience, so this is a UX convenience rather than a + * safety measure. Bare repositories have no such state, so + * the check is skipped there. + */ + if (!is_bare_repository(repo)) { + struct wt_status_state state = { 0 }; + + wt_status_get_state(repo, &state, 0); + if (state.merge_in_progress || state.revert_in_progress || + state.rebase_in_progress || state.bisect_in_progress || + state.cherry_pick_in_progress || state.am_in_progress || + state.rebase_interactive_in_progress) { + wt_status_state_free_buffers(&state); + die(_("--drop-filtered cannot be used while " + "another operation (merge, rebase, am, " + "cherry-pick, revert, or bisect) is in " + "progress")); + } + wt_status_state_free_buffers(&state); + } + + write_bitmaps = 0; + + /* + * Dropping objects means rebuilding the promisor packs + * without them and then removing the old packs, so the + * redundant packs must be deleted. Imply -d on a real run. + */ + if (!dry_run) + delete_redundant = 1; + + ret = enumerate_promisor_blobs(repo, &po_args.filter_options, &drop_oids); + + if (ret) + goto cleanup; + + /* + * Refuse to drop blobs that the current index references. + * Such a blob would only be lazily re-fetched by the next + * command that touches the worktree, so dropping it reclaims + * nothing. This guard just avoids that churn. Bare + * repositories have no index, so the check is skipped there. + */ + if (!is_bare_repository(repo) && oidset_size(&drop_oids)) { + struct index_state *istate = repo->index; + unsigned int i; + + if (repo_read_index(repo) < 0) + die(_("could not read the index")); + + for (i = 0; i < istate->cache_nr; i++) { + const struct cache_entry *ce = istate->cache[i]; + + if (oidset_contains(&drop_oids, &ce->oid)) + die(_("cannot drop '%s' (%s): it is referenced by the current index"), + ce->name, oid_to_hex(&ce->oid)); + } + } + + if (dry_run) { + struct oidset_iter iter; + const struct object_id *oid; + + oidset_iter_init(&drop_oids, &iter); + while ((oid = oidset_iter_next(&iter))) + printf("%s\n", oid_to_hex(oid)); + } + } + if (delete_redundant && repo->repository_format_precious_objects) die(_("cannot delete packs in a precious-objects repo")); @@ -362,7 +504,8 @@ int cmd_repack(int argc, strvec_push(&cmd.args, "--delta-islands"); if (pack_everything & ALL_INTO_ONE) { - repack_promisor_objects(repo, &po_args, &names, packtmp); + repack_promisor_objects(repo, &po_args, &names, packtmp, + (drop_filtered && !dry_run) ? &drop_oids : NULL); if (existing_packs_has_non_kept(&existing) && delete_redundant && @@ -545,7 +688,7 @@ int cmd_repack(int argc, } } - if (po_args.filter_options.choice) { + if (po_args.filter_options.choice && !drop_filtered) { struct write_pack_opts opts = { .po_args = &po_args, .destination = filter_to, @@ -632,12 +775,13 @@ int cmd_repack(int argc, if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0)) flags |= MIDX_WRITE_INCREMENTAL; - write_midx_file(files->packed, NULL, NULL, flags); + write_midx_file(files->packed, NULL, NULL, NULL, flags); } cleanup: string_list_clear(&keep_pack_list, 0); string_list_clear(&names, 1); + oidset_clear(&drop_oids); existing_packs_release(&existing); pack_geometry_release(&geometry); pack_objects_args_release(&po_args); diff --git a/builtin/replay.c b/builtin/replay.c index 39e3a86f6c10ab..d39626a37d055c 100644 --- a/builtin/replay.c +++ b/builtin/replay.c @@ -85,7 +85,7 @@ int cmd_replay(int argc, const char *const replay_usage[] = { N_("(EXPERIMENTAL!) git replay " "([--contained] --onto= | --advance= | --revert=)\n" - "[--ref=] [--ref-action=] "), + "[--ref=] [--ref-action=] [--linearize] "), NULL }; struct option replay_options[] = { @@ -111,6 +111,8 @@ int cmd_replay(int argc, N_("mode"), N_("control ref update behavior (update|print)"), PARSE_OPT_NONEG), + OPT_BOOL(0, "linearize", &opts.linearize, + N_("drop merge commits, replaying only non-merge commits")), OPT_END() }; @@ -132,6 +134,8 @@ int cmd_replay(int argc, opts.contained, "--contained"); die_for_incompatible_opt2(!!opts.ref, "--ref", !!opts.contained, "--contained"); + die_for_incompatible_opt2(opts.linearize, "--linearize", + !!opts.contained, "--contained"); /* Parse ref action mode from command line or config */ ref_mode = get_ref_action_mode(repo, ref_action); diff --git a/builtin/repo.c b/builtin/repo.c index 84e012f83f65ae..8e9f6296b846b3 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -14,10 +14,10 @@ #include "ref-filter.h" #include "refs.h" #include "revision.h" -#include "setup.h" #include "strbuf.h" #include "string-list.h" #include "shallow.h" +#include "submodule.h" #include "tree.h" #include "tree-walk.h" #include "utf8.h" @@ -99,6 +99,16 @@ static int get_path_commondir_relative(struct repository *repo, struct strbuf *b return 0; } +static int get_path_git_prefix(struct repository *repo, struct strbuf *buf) +{ + /* + * repo->prefix is NULL when the current working directory is + * the worktree root. + */ + strbuf_addstr(buf, repo->prefix ? repo->prefix : ""); + return 0; +} + static int get_path_gitdir_absolute(struct repository *repo, struct strbuf *buf) { const char *git_dir = repo_get_git_dir(repo); @@ -121,6 +131,120 @@ static int get_path_gitdir_relative(struct repository *repo, struct strbuf *buf) return 0; } +static int get_path_grafts_absolute(struct repository *repo, struct strbuf *buf) +{ + const char *graft_file = repo_get_graft_file(repo); + + if (!graft_file) + return error(_("unable to get graft file")); + + format_path(buf, graft_file, "", PATH_FORMAT_CANONICAL); + return 0; +} + +static int get_path_grafts_relative(struct repository *repo, struct strbuf *buf) +{ + const char *graft_file = repo_get_graft_file(repo); + + if (!graft_file) + return error(_("unable to get graft file")); + + format_path(buf, graft_file, repo->prefix, PATH_FORMAT_RELATIVE); + return 0; +} + +static int get_path_hooks_absolute(struct repository *repo, struct strbuf *buf) +{ + struct strbuf hooks_path = STRBUF_INIT; + + repo_git_path_replace(repo, &hooks_path, "hooks"); + format_path(buf, hooks_path.buf, "", PATH_FORMAT_CANONICAL); + strbuf_release(&hooks_path); + return 0; +} + +static int get_path_hooks_relative(struct repository *repo, struct strbuf *buf) +{ + struct strbuf hooks_path = STRBUF_INIT; + + repo_git_path_replace(repo, &hooks_path, "hooks"); + format_path(buf, hooks_path.buf, repo->prefix, PATH_FORMAT_RELATIVE); + strbuf_release(&hooks_path); + return 0; +} + +static int get_path_index_absolute(struct repository *repo, struct strbuf *buf) +{ + const char *index_file = repo_get_index_file(repo); + + if (!index_file) + return error(_("unable to get index file")); + + format_path(buf, index_file, "", PATH_FORMAT_CANONICAL); + return 0; +} + +static int get_path_index_relative(struct repository *repo, struct strbuf *buf) +{ + const char *index_file = repo_get_index_file(repo); + + if (!index_file) + return error(_("unable to get index file")); + + format_path(buf, index_file, repo->prefix, PATH_FORMAT_RELATIVE); + return 0; +} + +static int get_path_superproject_absolute(struct repository *repo UNUSED, struct strbuf *buf) +{ + struct strbuf superproject = STRBUF_INIT; + + if (!get_superproject_working_tree(&superproject)) { + strbuf_release(&superproject); + return 0; + } + + format_path(buf, superproject.buf, "", PATH_FORMAT_CANONICAL); + strbuf_release(&superproject); + return 0; +} + +static int get_path_superproject_relative(struct repository *repo, struct strbuf *buf) +{ + struct strbuf superproject = STRBUF_INIT; + + if (!get_superproject_working_tree(&superproject)) { + strbuf_release(&superproject); + return 0; + } + + format_path(buf, superproject.buf, repo->prefix, PATH_FORMAT_RELATIVE); + strbuf_release(&superproject); + return 0; +} + +static int get_path_toplevel_absolute(struct repository *repo, struct strbuf *buf) +{ + const char *work_tree = repo_get_work_tree(repo); + + if (!work_tree) + return 0; + + format_path(buf, work_tree, "", PATH_FORMAT_CANONICAL); + return 0; +} + +static int get_path_toplevel_relative(struct repository *repo, struct strbuf *buf) +{ + const char *work_tree = repo_get_work_tree(repo); + + if (!work_tree) + return 0; + + format_path(buf, work_tree, repo->prefix, PATH_FORMAT_RELATIVE); + return 0; +} + static int get_references_format(struct repository *repo, struct strbuf *buf) { strbuf_addstr(buf, @@ -135,8 +259,19 @@ static const struct repo_info_field repo_info_field[] = { { "object.format", get_object_format }, { "path.commondir.absolute", get_path_commondir_absolute }, { "path.commondir.relative", get_path_commondir_relative }, + { "path.git-prefix", get_path_git_prefix }, { "path.gitdir.absolute", get_path_gitdir_absolute }, { "path.gitdir.relative", get_path_gitdir_relative }, + { "path.grafts.absolute", get_path_grafts_absolute }, + { "path.grafts.relative", get_path_grafts_relative }, + { "path.hooks.absolute", get_path_hooks_absolute }, + { "path.hooks.relative", get_path_hooks_relative }, + { "path.index.absolute", get_path_index_absolute }, + { "path.index.relative", get_path_index_relative }, + { "path.superproject-root.absolute", get_path_superproject_absolute }, + { "path.superproject-root.relative", get_path_superproject_relative }, + { "path.toplevel.absolute", get_path_toplevel_absolute }, + { "path.toplevel.relative", get_path_toplevel_relative }, { "references.format", get_references_format }, }; diff --git a/builtin/rev-list.c b/builtin/rev-list.c index 02818b81c63fd7..6b596231ab1ab0 100644 --- a/builtin/rev-list.c +++ b/builtin/rev-list.c @@ -500,7 +500,8 @@ static void print_disk_usage(off_t size) strbuf_release(&sb); } -static inline int parse_missing_action_value(const char *value) +static inline int parse_missing_action_value(struct repository *repo, + const char *value) { if (!strcmp(value, "error")) { arg_missing_action = MA_ERROR; @@ -509,25 +510,25 @@ static inline int parse_missing_action_value(const char *value) if (!strcmp(value, "allow-any")) { arg_missing_action = MA_ALLOW_ANY; - fetch_if_missing = 0; + repo->fetch_if_missing = 0; return 1; } if (!strcmp(value, "print")) { arg_missing_action = MA_PRINT; - fetch_if_missing = 0; + repo->fetch_if_missing = 0; return 1; } if (!strcmp(value, "print-info")) { arg_missing_action = MA_PRINT_INFO; - fetch_if_missing = 0; + repo->fetch_if_missing = 0; return 1; } if (!strcmp(value, "allow-promisor")) { arg_missing_action = MA_ALLOW_PROMISOR; - fetch_if_missing = 0; + repo->fetch_if_missing = 0; return 1; } @@ -692,7 +693,7 @@ static void prepare_maximal_independent(struct rev_info *revs) int cmd_rev_list(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { struct rev_info revs; struct rev_list_info info; @@ -745,10 +746,10 @@ int cmd_rev_list(int argc, for (i = 1; i < argc; i++) { const char *arg = argv[i]; if (!strcmp(arg, "--exclude-promisor-objects")) { - fetch_if_missing = 0; + repo->fetch_if_missing = 0; revs.exclude_promisor_objects = 1; } else if (skip_prefix(arg, "--missing=", &arg)) { - parse_missing_action_value(arg); + parse_missing_action_value(repo, arg); } else if (!strcmp(arg, "-z")) { line_term = '\0'; info_term = '\0'; diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index 4263edfbecdd39..351948724ab703 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -358,51 +358,55 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size, write_object(nr, type, buf, size); } -struct input_zstream_data { +struct zlib_stream { + struct odb_stream base; git_zstream *zstream; int status; }; -static ssize_t feed_input_zstream(struct odb_write_stream *in_stream, - unsigned char *buf, size_t buf_len) +static ssize_t zlib_stream_read(struct odb_stream *in_stream, + char *buf, size_t buf_len) { - struct input_zstream_data *data = in_stream->data; + struct zlib_stream *data = container_of(in_stream, struct zlib_stream, base); git_zstream *zstream = data->zstream; - void *in = fill(1); - if (in_stream->is_finished) + if (data->status != Z_OK) return 0; - zstream->next_out = buf; + zstream->next_out = (unsigned char *) buf; zstream->avail_out = buf_len; - zstream->next_in = in; - zstream->avail_in = len; - data->status = git_inflate(zstream, 0); + while (data->status == Z_OK && zstream->avail_out == buf_len) { + zstream->next_in = fill(1); + zstream->avail_in = len; + data->status = git_inflate(zstream, 0); + use(len - zstream->avail_in); + } - in_stream->is_finished = data->status != Z_OK; - use(len - zstream->avail_in); return buf_len - zstream->avail_out; } static void stream_blob(unsigned long size, unsigned nr) { git_zstream zstream = { 0 }; - struct input_zstream_data data = { 0 }; - struct odb_write_stream in_stream = { - .read = feed_input_zstream, - .data = &data, + struct zlib_stream in_stream = { + .base = { + .read = zlib_stream_read, + .size = size, + .type = OBJ_BLOB, + }, + .zstream = &zstream, + .status = Z_OK, }; struct obj_info *info = &obj_list[nr]; - data.zstream = &zstream; git_inflate_init(&zstream); - if (odb_write_object_stream(the_repository->objects, &in_stream, size, &info->oid)) + if (odb_write_object_stream(the_repository->objects, &in_stream.base, &info->oid)) die(_("failed to write object in stream")); - if (data.status != Z_STREAM_END) - die(_("inflate returned (%d)"), data.status); + if (in_stream.status != Z_STREAM_END) + die(_("inflate returned (%d)"), in_stream.status); git_inflate_end(&zstream); if (strict) { @@ -603,7 +607,7 @@ static void unpack_all(void) unpack_one(i); display_progress(progress, i + 1); } - odb_transaction_commit(transaction); + odb_transaction_commit_and_finalize_or_die(transaction); stop_progress(&progress); if (delta_list) diff --git a/builtin/update-index.c b/builtin/update-index.c index 241abd4332dcf9..bf70f76c49830b 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -130,7 +130,7 @@ static void xrmdir(const char *path) static void avoid_racy(void) { /* - * not use if we could usleep(10) if USE_NSEC is defined. The + * not use if we could usleep(10) if core.useNanosec is defined. The * field nsec could be there, but the OS could choose to * ignore it? */ @@ -1156,7 +1156,7 @@ int cmd_update_index(int argc, * a transaction. */ if (transaction && verbose) { - odb_transaction_commit(transaction); + odb_transaction_commit_and_finalize_or_die(transaction); transaction = NULL; } @@ -1224,7 +1224,7 @@ int cmd_update_index(int argc, /* * By now we have added all of the new objects */ - odb_transaction_commit(transaction); + odb_transaction_commit_and_finalize_or_die(transaction); if (split_index > 0) { if (repo_config_get_split_index(the_repository) == 0) diff --git a/builtin/upload-pack.c b/builtin/upload-pack.c index 32831fb8796acc..8b531ca724faf6 100644 --- a/builtin/upload-pack.c +++ b/builtin/upload-pack.c @@ -42,10 +42,13 @@ int cmd_upload_pack(int argc, OPT_END() }; unsigned enter_repo_flags = ENTER_REPO_ANY_OWNER_OK; + bool no_lazy_fetch_set; packet_trace_identity("upload-pack"); disable_replace_refs(); save_commit_buffer = 0; + + no_lazy_fetch_set = !!getenv(NO_LAZY_FETCH_ENVIRONMENT); xsetenv(NO_LAZY_FETCH_ENVIRONMENT, "1", 0); argc = parse_options(argc, argv, prefix, options, upload_pack_usage, 0); @@ -62,6 +65,14 @@ int cmd_upload_pack(int argc, if (!enter_repo(the_repository, dir, enter_repo_flags)) die("'%s' does not appear to be a git repository", dir); + /* + * Relax the GIT_NO_LAZY_FETCH=1 default if the served repo is in + * the "uploadpack.lazyFetchTrusted" protected allowlist and + * GIT_NO_LAZY_FETCH was not already set explicitly. + */ + if (!no_lazy_fetch_set && upload_pack_lazy_fetch_trusted(the_repository)) + xsetenv(NO_LAZY_FETCH_ENVIRONMENT, "0", 1); + switch (determine_protocol_version_server()) { case protocol_v2: if (advertise_refs) diff --git a/builtin/worktree.c b/builtin/worktree.c index 654d27c3e1ce99..b29c3a3755c58e 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -64,6 +64,19 @@ "\n" \ " git worktree add --orphan %s\n") +#define WORKTREE_ADD_AMBIGUOUS_REMOTE_BRANCH_NAME_HINT_TEXT \ + _("Matched multiple remote tracking branches, you can list them by:\n" \ + "\n" \ + " git branch -r --list \"*/%s\"\n" \ + "\n" \ + "If you meant to create a worktree from a remote tracking branch on,\n" \ + "e.g. 'origin', you can do so by:\n" \ + "\n" \ + " git worktree add -b %s %s origin/%s\n" \ + "\n" \ + "If you'd like to always prefer some remote, e.g. 'origin',\n" \ + "consider setting checkout.defaultRemote=origin in your config.") + static const char * const git_worktree_usage[] = { BUILTIN_WORKTREE_ADD_USAGE, BUILTIN_WORKTREE_LIST_USAGE, @@ -904,10 +917,18 @@ static int add(int ac, const char **av, const char *prefix, commit = lookup_commit_reference_by_name(branch); if (!commit) { - remote = unique_tracking_name(branch, &oid, NULL); + int num_matches = 0; + remote = unique_tracking_name(branch, &oid, &num_matches); if (remote) { new_branch = branch; branch = new_branch_to_free = remote; + } else if (num_matches > 1) { + if (!opts.quiet) + advise_if_enabled(ADVICE_CHECKOUT_AMBIGUOUS_REMOTE_BRANCH_NAME, + WORKTREE_ADD_AMBIGUOUS_REMOTE_BRANCH_NAME_HINT_TEXT, + branch, branch, path, branch); + die(_("'%s' matched multiple (%d) remote tracking branches"), + branch, num_matches); } } diff --git a/bundle.c b/bundle.c index b64716f252b78f..f55a521b2a1f12 100644 --- a/bundle.c +++ b/bundle.c @@ -1,4 +1,3 @@ -#define USE_THE_REPOSITORY_VARIABLE #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" @@ -21,6 +20,13 @@ #include "connected.h" #include "write-or-die.h" +/* + * NEEDSWORK: this function implicitly depends on `the_repository` and is not + * available because we dropped USE_THE_REPOSITORY_VARIABLE. We can remove the + * declaration once it's accessible via `repo_config_values`. + */ +extern const char *get_log_output_encoding(void); + static const char v2_bundle_signature[] = "# v2 git bundle\n"; static const char v3_bundle_signature[] = "# v3 git bundle\n"; static struct { @@ -294,7 +300,8 @@ int list_bundle_refs(struct bundle_header *header, int argc, const char **argv) return list_refs(&header->references, argc, argv); } -static int is_tag_in_date_range(struct object *tag, struct rev_info *revs) +static int is_tag_in_date_range(struct repository *repo, + struct object *tag, struct rev_info *revs) { size_t size; enum object_type type; @@ -305,7 +312,7 @@ static int is_tag_in_date_range(struct object *tag, struct rev_info *revs) if (revs->max_age == -1 && revs->min_age == -1) goto out; - buf = odb_read_object(the_repository->objects, &tag->oid, &type, &size); + buf = odb_read_object(repo->objects, &tag->oid, &type, &size); if (!buf) goto out; line = memmem(buf, size, "\ntagger ", 8); @@ -325,50 +332,52 @@ static int is_tag_in_date_range(struct object *tag, struct rev_info *revs) /* Write the pack data to bundle_fd */ -static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *pack_options) +static int write_pack_data(int bundle_fd, struct rev_info *revs, int progress) { - struct child_process pack_objects = CHILD_PROCESS_INIT; + struct odb_generate_pack_options opts = ODB_GENERATE_PACK_OPTIONS_INIT; + struct odb_pack_generator *generator; + int ret = 0; int i; - strvec_pushl(&pack_objects.args, - "pack-objects", - "--stdout", "--thin", "--delta-base-offset", - NULL); - strvec_pushv(&pack_objects.args, pack_options->v); + opts.thin = 1; + opts.ofs_delta = 1; + if (progress) + opts.progress = ODB_GENERATE_PACK_PROGRESS_VERBOSE; if (revs->filter.choice) - strvec_pushf(&pack_objects.args, "--filter=%s", - list_objects_filter_spec(&revs->filter)); - pack_objects.in = -1; - pack_objects.out = bundle_fd; - pack_objects.git_cmd = 1; + opts.filter_spec = list_objects_filter_spec(&revs->filter); /* - * start_command() will close our descriptor if it's >1. Duplicate it - * to avoid surprising the caller. + * The pack generator will consume our descriptor if it's >1. + * Duplicate it to avoid surprising the caller. */ - if (pack_objects.out > 1) { - pack_objects.out = dup(pack_objects.out); - if (pack_objects.out < 0) { - error_errno(_("unable to dup bundle descriptor")); - child_process_clear(&pack_objects); - return -1; - } + opts.pack_fd = bundle_fd; + if (opts.pack_fd > 1) { + opts.pack_fd = dup(bundle_fd); + if (opts.pack_fd < 0) + return error_errno(_("unable to dup bundle descriptor")); } - if (start_command(&pack_objects)) - return error(_("Could not spawn pack-objects")); - for (i = 0; i < revs->pending.nr; i++) { struct object *object = revs->pending.objects[i].item; if (object->flags & UNINTERESTING) - write_or_die(pack_objects.in, "^", 1); - write_or_die(pack_objects.in, oid_to_hex(&object->oid), the_hash_algo->hexsz); - write_or_die(pack_objects.in, "\n", 1); + oid_array_append(&opts.haves, &object->oid); + else + oid_array_append(&opts.wants, &object->oid); } - close(pack_objects.in); - if (finish_command(&pack_objects)) - return error(_("pack-objects died")); - return 0; + + if (odb_generate_pack(revs->repo->objects, &generator, &opts)) { + ret = error(_("Could not spawn pack-objects")); + goto out; + } + + if (odb_pack_generator_finish(generator)) { + ret = error(_("pack-objects died")); + goto out; + } + +out: + odb_generate_pack_options_release(&opts); + return ret; } /* @@ -395,10 +404,10 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs) if (e->item->flags & UNINTERESTING) continue; - if (repo_dwim_ref(the_repository, e->name, strlen(e->name), + if (repo_dwim_ref(revs->repo, e->name, strlen(e->name), &oid, &ref, 0) != 1) goto skip_write_ref; - if (refs_read_ref_full(get_main_ref_store(the_repository), e->name, RESOLVE_REF_READING, &oid, &flag)) + if (refs_read_ref_full(get_main_ref_store(revs->repo), e->name, RESOLVE_REF_READING, &oid, &flag)) flag = 0; display_ref = (flag & REF_ISSYMREF) ? e->name : ref; @@ -406,7 +415,7 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs) goto skip_write_ref; if (e->item->type == OBJ_TAG && - !is_tag_in_date_range(e->item, revs)) { + !is_tag_in_date_range(revs->repo, e->item, revs)) { e->item->flags |= UNINTERESTING; goto skip_write_ref; } @@ -428,7 +437,8 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs) ref_count++; strset_add(&objects, display_ref); - write_or_die(bundle_fd, oid_to_hex(&e->item->oid), the_hash_algo->hexsz); + write_or_die(bundle_fd, oid_to_hex(&e->item->oid), + revs->repo->hash_algo->hexsz); write_or_die(bundle_fd, " ", 1); write_or_die(bundle_fd, display_ref, strlen(display_ref)); write_or_die(bundle_fd, "\n", 1); @@ -476,7 +486,7 @@ static void write_bundle_prerequisites(struct commit *commit, void *data) } int create_bundle(struct repository *r, const char *path, - int argc, const char **argv, struct strvec *pack_options, int version) + int argc, const char **argv, int version, int progress) { struct lock_file lock = LOCK_INIT; int bundle_fd = -1; @@ -507,7 +517,7 @@ int create_bundle(struct repository *r, const char *path, * SHA1. * 2. @filter is required because we parsed an object filter. */ - if (the_hash_algo != &hash_algos[GIT_HASH_SHA1_LEGACY] || revs.filter.choice) + if (r->hash_algo != &hash_algos[GIT_HASH_SHA1_LEGACY] || revs.filter.choice) min_version = 3; if (argc > 1) { @@ -528,14 +538,15 @@ int create_bundle(struct repository *r, const char *path, if (version < 2 || version > 3) { die(_("unsupported bundle version %d"), version); } else if (version < min_version) { - die(_("cannot write bundle version %d with algorithm %s"), version, the_hash_algo->name); + die(_("cannot write bundle version %d with algorithm %s"), version, + r->hash_algo->name); } else if (version == 2) { write_or_die(bundle_fd, v2_bundle_signature, strlen(v2_bundle_signature)); } else { const char *capability = "@object-format="; write_or_die(bundle_fd, v3_bundle_signature, strlen(v3_bundle_signature)); write_or_die(bundle_fd, capability, strlen(capability)); - write_or_die(bundle_fd, the_hash_algo->name, strlen(the_hash_algo->name)); + write_or_die(bundle_fd, r->hash_algo->name, strlen(r->hash_algo->name)); write_or_die(bundle_fd, "\n", 1); if (revs.filter.choice) { @@ -584,7 +595,7 @@ int create_bundle(struct repository *r, const char *path, } /* write pack */ - if (write_pack_data(bundle_fd, &revs_copy, pack_options)) { + if (write_pack_data(bundle_fd, &revs_copy, progress)) { ret = -1; goto out; } diff --git a/bundle.h b/bundle.h index d664b2f2d61e20..471da23d1bc9e4 100644 --- a/bundle.h +++ b/bundle.h @@ -27,8 +27,7 @@ int read_bundle_header(const char *path, struct bundle_header *header); int read_bundle_header_fd(int fd, struct bundle_header *header, const char *report_path); int create_bundle(struct repository *r, const char *path, - int argc, const char **argv, struct strvec *pack_options, - int version); + int argc, const char **argv, int version, int progress); enum verify_bundle_flags { VERIFY_BUNDLE_VERBOSE = (1 << 0), diff --git a/cache-tree.c b/cache-tree.c index d92f5132865f13..a220372a420197 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -538,7 +538,7 @@ int cache_tree_update(struct index_state *istate, int flags) i = update_one(istate->cache_tree, istate->cache, istate->cache_nr, "", 0, &skip, flags); if (!inflight) - odb_transaction_commit(transaction); + odb_transaction_commit_and_finalize_or_die(transaction); trace2_region_leave("cache_tree", "update", istate->repo); trace_performance_leave("cache_tree_update"); if (i < 0) diff --git a/chdir-notify.c b/chdir-notify.c index 1237a45e2e6492..55773c24c96eda 100644 --- a/chdir-notify.c +++ b/chdir-notify.c @@ -7,25 +7,22 @@ #include "trace.h" struct chdir_notify_entry { - const char *name; chdir_notify_callback cb; void *data; struct list_head list; }; static LIST_HEAD(chdir_notify_entries); -void chdir_notify_register(const char *name, - chdir_notify_callback cb, +void chdir_notify_register(chdir_notify_callback cb, void *data) { struct chdir_notify_entry *e = xmalloc(sizeof(*e)); - e->name = name; e->cb = cb; e->data = data; list_add_tail(&e->list, &chdir_notify_entries); } -void chdir_notify_unregister(const char *name, chdir_notify_callback cb, +void chdir_notify_unregister(chdir_notify_callback cb, void *data) { struct list_head *pos, *p; @@ -34,8 +31,7 @@ void chdir_notify_unregister(const char *name, chdir_notify_callback cb, struct chdir_notify_entry *e = list_entry(pos, struct chdir_notify_entry, list); - if (e->cb != cb || e->data != data || !e->name != !name || - (e->name && strcmp(e->name, name))) + if (e->cb != cb || e->data != data) continue; list_del(pos); @@ -64,7 +60,7 @@ int chdir_notify(const char *new_cwd) list_for_each(pos, &chdir_notify_entries) { struct chdir_notify_entry *e = list_entry(pos, struct chdir_notify_entry, list); - e->cb(e->name, old_cwd.buf, new_cwd, e->data); + e->cb(old_cwd.buf, new_cwd, e->data); } strbuf_release(&old_cwd); diff --git a/chdir-notify.h b/chdir-notify.h index 36b4114472e31d..e4ae38e12dd8b9 100644 --- a/chdir-notify.h +++ b/chdir-notify.h @@ -33,13 +33,11 @@ * $GIT_TRACE_SETUP. It may be NULL, but if non-NULL should point to * storage which lasts as long as the registration is active. */ -typedef void (*chdir_notify_callback)(const char *name, - const char *old_cwd, +typedef void (*chdir_notify_callback)(const char *old_cwd, const char *new_cwd, void *data); -void chdir_notify_register(const char *name, chdir_notify_callback cb, void *data); -void chdir_notify_unregister(const char *name, chdir_notify_callback cb, - void *data); +void chdir_notify_register(chdir_notify_callback cb, void *data); +void chdir_notify_unregister(chdir_notify_callback cb, void *data); /* * diff --git a/chunk-format.c b/chunk-format.c index 51b5a2c959a1cf..34ab2750f7f14a 100644 --- a/chunk-format.c +++ b/chunk-format.c @@ -101,12 +101,14 @@ int write_chunkfile(struct chunkfile *cf, void *data) return result; } -int read_table_of_contents(struct chunkfile *cf, - const unsigned char *mfile, - size_t mfile_size, - uint64_t toc_offset, - int toc_length, - unsigned expected_alignment) +static int read_table_of_contents_1(struct chunkfile *cf, + const unsigned char *mfile, + size_t mfile_size, + uint64_t toc_offset, + int toc_length, + unsigned expected_alignment, + const struct git_hash_algo *algo, + int quiet) { int i; uint32_t chunk_id; @@ -121,12 +123,14 @@ int read_table_of_contents(struct chunkfile *cf, chunk_offset = get_be64(table_of_contents + 4); if (!chunk_id) { - error(_("terminating chunk id appears earlier than expected")); + if (!quiet) + error(_("terminating chunk id appears earlier than expected")); return 1; } if (chunk_offset % expected_alignment != 0) { - error(_("chunk id %"PRIx32" not %d-byte aligned"), - chunk_id, expected_alignment); + if (!quiet) + error(_("chunk id %"PRIx32" not %d-byte aligned"), + chunk_id, expected_alignment); return 1; } @@ -134,16 +138,18 @@ int read_table_of_contents(struct chunkfile *cf, next_chunk_offset = get_be64(table_of_contents + 4); if (next_chunk_offset < chunk_offset || - next_chunk_offset > mfile_size - the_hash_algo->rawsz) { - error(_("improper chunk offset(s) %"PRIx64" and %"PRIx64""), - chunk_offset, next_chunk_offset); + next_chunk_offset > mfile_size - algo->rawsz) { + if (!quiet) + error(_("improper chunk offset(s) %"PRIx64" and %"PRIx64""), + chunk_offset, next_chunk_offset); return -1; } for (i = 0; i < cf->chunks_nr; i++) { if (cf->chunks[i].id == chunk_id) { - error(_("duplicate chunk ID %"PRIx32" found"), - chunk_id); + if (!quiet) + error(_("duplicate chunk ID %"PRIx32" found"), + chunk_id); return -1; } } @@ -156,13 +162,39 @@ int read_table_of_contents(struct chunkfile *cf, chunk_id = get_be32(table_of_contents); if (chunk_id) { - error(_("final chunk has non-zero id %"PRIx32""), chunk_id); + if (!quiet) + error(_("final chunk has non-zero id %"PRIx32""), chunk_id); return -1; } return 0; } +int read_table_of_contents(struct chunkfile *cf, + const unsigned char *mfile, + size_t mfile_size, + uint64_t toc_offset, + int toc_length, + unsigned expected_alignment) +{ + return read_table_of_contents_1(cf, mfile, mfile_size, toc_offset, + toc_length, expected_alignment, + the_hash_algo, 0); +} + +int read_table_of_contents_quiet(struct chunkfile *cf, + const unsigned char *mfile, + size_t mfile_size, + uint64_t toc_offset, + int toc_length, + unsigned expected_alignment, + const struct git_hash_algo *algo) +{ + return read_table_of_contents_1(cf, mfile, mfile_size, toc_offset, + toc_length, expected_alignment, + algo, 1); +} + struct pair_chunk_data { const unsigned char **p; size_t *size; diff --git a/chunk-format.h b/chunk-format.h index 212a0a6af18356..bc31302ed09f1b 100644 --- a/chunk-format.h +++ b/chunk-format.h @@ -39,6 +39,20 @@ int read_table_of_contents(struct chunkfile *cf, int toc_length, unsigned expected_alignment); +/* + * Like read_table_of_contents(), for a reader that treats a malformed + * table as an absent file rather than reporting it: nothing is printed + * on failure, and the trailing-checksum bound is computed with the + * given hash algorithm instead of the_hash_algo. + */ +int read_table_of_contents_quiet(struct chunkfile *cf, + const unsigned char *mfile, + size_t mfile_size, + uint64_t toc_offset, + int toc_length, + unsigned expected_alignment, + const struct git_hash_algo *algo); + #define CHUNK_NOT_FOUND (-2) /* diff --git a/color.c b/color.c index 00b53f97acbcc7..9015d0faf18259 100644 --- a/color.c +++ b/color.c @@ -72,7 +72,7 @@ static int get_hex_color(const char **inp, int width, unsigned char *out) unsigned int val; assert(width == 1 || width == 2); - val = (hexval(in[0]) << 4) | hexval(in[width - 1]); + val = (hexval(in[0], HEX_KIND_MIXED) << 4) | hexval(in[width - 1], HEX_KIND_MIXED); if (val & ~0xff) return -1; *inp += width; diff --git a/command-list.txt b/command-list.txt index 21b802c42026b3..e7b241e6ad8498 100644 --- a/command-list.txt +++ b/command-list.txt @@ -95,6 +95,7 @@ git-describe mainporcelain git-diagnose ancillaryinterrogators git-diff mainporcelain info git-diff-files plumbinginterrogators +git-diff-hunks plumbingmanipulators git-diff-index plumbinginterrogators git-diff-pairs plumbinginterrogators git-diff-tree plumbinginterrogators @@ -223,6 +224,7 @@ gitfaq guide gitformat-bundle developerinterfaces gitformat-chunk developerinterfaces gitformat-commit-graph developerinterfaces +gitformat-diff-hunks developerinterfaces gitformat-index developerinterfaces gitformat-pack developerinterfaces gitformat-signature developerinterfaces diff --git a/commit-graph.c b/commit-graph.c index 49e8f639305212..983c11ce853459 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -651,8 +651,6 @@ struct commit_graph *load_commit_graph_chain_fd_st(struct object_database *odb, count = st->st_size / (odb->repo->hash_algo->hexsz + 1); CALLOC_ARRAY(oids, count); - odb_prepare_alternates(odb); - for (i = 0; i < count; i++) { struct odb_source *source; @@ -768,7 +766,6 @@ static struct commit_graph *prepare_commit_graph(struct repository *r) if (!commit_graph_compatible(r)) return NULL; - odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { r->objects->commit_graph = read_commit_graph_one(source); if (r->objects->commit_graph) @@ -2018,7 +2015,6 @@ static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) _("Finding commits for commit graph among packed objects"), ctx->approx_nr_objects); - odb_prepare_alternates(ctx->r->objects); for (source = ctx->r->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); odb_source_for_each_object(&files->packed->base, &oi, add_packed_commits_oi, diff --git a/commit-reach.c b/commit-reach.c index b53c6b1cdff3f3..b16eea2355356f 100644 --- a/commit-reach.c +++ b/commit-reach.c @@ -11,6 +11,7 @@ #include "tag.h" #include "commit-reach.h" #include "ewah/ewok.h" +#include "trace2.h" /* Remember to update object flag allocation in object.h */ #define PARENT1 (1u<<16) @@ -78,25 +79,111 @@ static void clear_nonstale_queue(struct nonstale_queue *queue) queue->max_nonstale = NULL; } -static void nonstale_queue_put_dedup(struct nonstale_queue *queue, - struct commit *c) +/* + * Priority queue with per-side commit counters for paint_down_to_common(). + * Each non-stale queued commit occupies exactly one bucket: PARENT1-only, + * PARENT2-only, or both (a pending merge-base candidate). + */ +struct paint_state { + struct prio_queue queue; + size_t parent1_count; + size_t parent2_count; + size_t mb_candidate_count; + timestamp_t min_generation; + timestamp_t last_gen; + timestamp_t topo_ceiling; +}; + +static void paint_count_update(struct paint_state *state, + unsigned flags, int delta) { - if (c->object.flags & ENQUEUED) - return; - c->object.flags |= ENQUEUED; - nonstale_queue_put(queue, c); + switch (flags & (PARENT1 | PARENT2 | STALE)) { + case PARENT1: + state->parent1_count += delta; + break; + + case PARENT2: + state->parent2_count += delta; + break; + + case PARENT1 | PARENT2: + state->mb_candidate_count += delta; + break; + + case PARENT1 | PARENT2 | STALE: + break; + + default: + BUG("unexpected paint state"); + } } -static struct commit *nonstale_queue_get_dedup(struct nonstale_queue *queue) +static void paint_queue_put(struct paint_state *state, + struct commit *c, unsigned add_flags) { - struct commit *commit = nonstale_queue_get(queue); + unsigned old_flags = c->object.flags; + c->object.flags |= add_flags; + + if (old_flags & ENQUEUED) { + paint_count_update(state, old_flags, -1); + paint_count_update(state, c->object.flags, 1); + } else { + c->object.flags |= ENQUEUED; + prio_queue_put(&state->queue, c); + paint_count_update(state, c->object.flags, 1); + } +} + +/* + * Dequeue the next commit for the paint walk, or return NULL when + * no more merge bases can be discovered. + */ +static struct commit *paint_queue_get(struct paint_state *state) +{ + struct commit *commit = prio_queue_get(&state->queue); + timestamp_t generation; + + if (!commit) + return NULL; + + commit->object.flags &= ~ENQUEUED; + generation = commit_graph_generation(commit); - if (commit) - commit->object.flags &= ~ENQUEUED; + if (state->min_generation && generation > state->last_gen) + BUG("bad generation skip %"PRItime" > %"PRItime" at %s", + generation, state->last_gen, + oid_to_hex(&commit->object.oid)); + state->last_gen = generation; + + /* generation cutoff */ + if (generation < state->min_generation) + return NULL; + + /* + * Check exit condition before decrementing: the counters + * still include this commit, so the last non-stale commit + * sees a non-zero count and is returned for processing. + */ + if (!state->mb_candidate_count) { + /* only stale entries remain */ + if (!state->parent1_count && !state->parent2_count) + return NULL; + + /* one side is exhausted */ + if ((!state->parent1_count || !state->parent2_count) && + generation < state->topo_ceiling) + return NULL; + } + + paint_count_update(state, commit->object.flags, -1); return commit; } -/* all input commits in one and twos[] must have been parsed! */ +/* + * See Documentation/technical/paint-down-to-common.adoc + * + * All input commits in one and twos[] must have been parsed! + */ static int paint_down_to_common(struct repository *r, struct commit *one, int n, struct commit **twos, @@ -104,45 +191,40 @@ static int paint_down_to_common(struct repository *r, enum merge_base_flags mb_flags, struct commit_list **result) { - struct nonstale_queue queue = { - { compare_commits_by_gen_then_commit_date } + /* + * Generation ordering is required for the side-exhaustion and + * single-result early exits, which rely on topological traversal + * order (children visited before parents) in the ordered region. + */ + struct paint_state state = { + .queue = { compare_commits_by_gen_then_commit_date } }; + struct commit *commit; int i; - int gen_ordered = 1; - timestamp_t last_gen = GENERATION_NUMBER_INFINITY; + int steps = 0; struct commit_list **tail = result; - if (!min_generation && !corrected_commit_dates_enabled(r)) { - queue.pq.compare = compare_commits_by_commit_date; - gen_ordered = 0; - } + state.min_generation = min_generation; + state.last_gen = GENERATION_NUMBER_INFINITY; + state.topo_ceiling = corrected_commit_dates_enabled(r) + ? GENERATION_NUMBER_INFINITY + : GENERATION_NUMBER_V1_MAX; + one->object.flags |= PARENT1; if (!n) { commit_list_append(one, result); return 0; } - nonstale_queue_put_dedup(&queue, one); + paint_queue_put(&state, one, 0); - for (i = 0; i < n; i++) { - twos[i]->object.flags |= PARENT2; - nonstale_queue_put_dedup(&queue, twos[i]); - } + for (i = 0; i < n; i++) + paint_queue_put(&state, twos[i], PARENT2); - while (queue.max_nonstale) { - struct commit *commit = nonstale_queue_get_dedup(&queue); + while ((commit = paint_queue_get(&state))) { struct commit_list *parents; int flags; - timestamp_t generation = commit_graph_generation(commit); - - if (min_generation && generation > last_gen) - BUG("bad generation skip %"PRItime" > %"PRItime" at %s", - generation, last_gen, - oid_to_hex(&commit->object.oid)); - last_gen = generation; - - if (generation < min_generation) - break; + steps++; flags = commit->object.flags & (PARENT1 | PARENT2 | STALE); if (flags == (PARENT1 | PARENT2)) { @@ -155,8 +237,7 @@ static int paint_down_to_common(struct repository *r, * descendant of this one. */ if (!(mb_flags & MERGE_BASE_FIND_ALL) && - gen_ordered && - generation < GENERATION_NUMBER_INFINITY) + state.last_gen < state.topo_ceiling) break; } /* Mark parents of a found merge stale */ @@ -169,7 +250,7 @@ static int paint_down_to_common(struct repository *r, if ((p->object.flags & flags) == flags) continue; if (repo_parse_commit(r, p)) { - clear_nonstale_queue(&queue); + clear_prio_queue(&state.queue); commit_list_free(*result); *result = NULL; /* @@ -184,12 +265,13 @@ static int paint_down_to_common(struct repository *r, return error(_("could not parse commit %s"), oid_to_hex(&p->object.oid)); } - p->object.flags |= flags; - nonstale_queue_put_dedup(&queue, p); + paint_queue_put(&state, p, flags); } } - clear_nonstale_queue(&queue); + clear_prio_queue(&state.queue); + trace2_data_intmax("paint_down_to_common", r, + "steps", steps); commit_list_sort_by_date(result); return 0; } diff --git a/common-init.c b/common-init.c index d26c9c1f20239e..4a3fa4d7bee3be 100644 --- a/common-init.c +++ b/common-init.c @@ -47,7 +47,7 @@ static void setup_environment(void) update_ref_namespace(NAMESPACE_REPLACE, git_replace_ref_base); if (git_env_bool(NO_LAZY_FETCH_ENVIRONMENT, 0)) - fetch_if_missing = 0; + the_repository->fetch_if_missing = 0; } void init_git(const char **argv) diff --git a/compat/darwin.h b/compat/darwin.h new file mode 100644 index 00000000000000..6fbdc345608ec4 --- /dev/null +++ b/compat/darwin.h @@ -0,0 +1,8 @@ +#ifndef COMPAT_DARWIN_H +#define COMPAT_DARWIN_H + +int darwin_regexec_buf(const regex_t *preg, const char *buf, size_t size, + size_t nmatch, regmatch_t pmatch[], int eflags); +#define regexec_buf darwin_regexec_buf + +#endif diff --git a/compat/darwin/regexec.c b/compat/darwin/regexec.c new file mode 100644 index 00000000000000..13fb7d5946bc8c --- /dev/null +++ b/compat/darwin/regexec.c @@ -0,0 +1,91 @@ +#include "git-compat-util.h" + +#include + +/* + * Darwin's TRE regex engine leaks an internal buffer when it encounters an + * invalid multibyte sequence. Since the leak has already happened when + * regexec() reports REG_ILLSEQ, keep invalid bytes out of regexec() by + * searching each valid segment separately. + */ + +/* + * Search buf[start, end), where size is the full size of buf. REG_STARTEND + * keeps match offsets relative to buf. Do not let an internal segment create + * a false beginning or end of line. + */ +static int regexec_segment(const regex_t *preg, const char *buf, + size_t size, size_t start, size_t end, + size_t nmatch, regmatch_t pmatch[], int eflags) +{ + eflags |= REG_STARTEND; + if (start > 0) + eflags |= REG_NOTBOL; + if (end < size) + eflags |= REG_NOTEOL; + pmatch[0].rm_so = start; + pmatch[0].rm_eo = end; + return regexec(preg, buf, nmatch, pmatch, eflags); +} + +int darwin_regexec_buf(const regex_t *preg, const char *buf, size_t size, + size_t nmatch, regmatch_t pmatch[], int eflags) +{ + size_t seg_start = 0, i = 0; + mbstate_t mbs; + + assert(nmatch > 0 && pmatch); + + /* + * A single-byte locale cannot contain an invalid multibyte sequence, + * so use regexec() directly. + */ + if (MB_CUR_MAX == 1) { + pmatch[0].rm_so = 0; + pmatch[0].rm_eo = size; + return regexec(preg, buf, nmatch, pmatch, eflags | REG_STARTEND); + } + + memset(&mbs, 0, sizeof(mbs)); + while (i < size) { + unsigned char c = (unsigned char)buf[i]; + size_t n; + + if (c < 0x80) { + i++; + continue; + } + + n = mbrtowc(NULL, buf + i, size - i, &mbs); + if (!n) + n = 1; + if (n != (size_t)-1 && n != (size_t)-2) { + i += n; + continue; + } + + /* + * -1 denotes an encoding error; -2 denotes an incomplete + * trailing sequence. In either case, buf[i] cannot begin a + * complete valid character within this buffer. Search an + * empty initial segment to preserve zero-width matches at the + * true beginning. + */ + if (i > seg_start || i == 0) { + int ret = regexec_segment(preg, buf, size, seg_start, i, + nmatch, pmatch, eflags); + if (ret != REG_NOMATCH) + return ret; + } + i++; + seg_start = i; + memset(&mbs, 0, sizeof(mbs)); + } + + /* + * Search the final segment even when it is empty, so an empty buffer + * or a buffer ending in invalid bytes still has its true end. + */ + return regexec_segment(preg, buf, size, seg_start, size, + nmatch, pmatch, eflags); +} diff --git a/compat/mingw.c b/compat/mingw.c index 4c2f26d4548147..0c8179f3458bb8 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -3157,6 +3157,45 @@ int xwcstoutf(char *utf, const wchar_t *wcs, size_t utflen) return -1; } +#ifdef ENSURE_MSYSTEM_IS_SET +#if !defined(RUNTIME_PREFIX) || !defined(HAVE_WPGMPTR) || !defined(MINGW_PREFIX) +static size_t append_system_bin_dirs(char *path UNUSED, size_t size UNUSED) +{ + return 0; +} +#else +static size_t append_system_bin_dirs(char *path, size_t size) +{ + char prefix[32768]; + const char *slash; + size_t len = xwcstoutf(prefix, _wpgmptr, sizeof(prefix)), off = 0; + + if (len == 0 || len >= sizeof(prefix) || + !(slash = find_last_dir_sep(prefix))) + return 0; + /* strip trailing `git.exe` */ + len = slash - prefix; + + /* strip trailing `cmd` or `\bin` or `bin` or `libexec\git-core` */ + if (strip_suffix_mem(prefix, &len, "\\" MINGW_PREFIX "\\libexec\\git-core") || + strip_suffix_mem(prefix, &len, "\\" MINGW_PREFIX "\\bin")) + off += xsnprintf(path + off, size - off, + "%.*s\\" MINGW_PREFIX "\\bin;", (int)len, prefix); + else if (strip_suffix_mem(prefix, &len, "\\cmd") || + strip_suffix_mem(prefix, &len, "\\bin") || + strip_suffix_mem(prefix, &len, "\\libexec\\git-core")) + off += xsnprintf(path + off, size - off, + "%.*s\\" MINGW_PREFIX "\\bin;", (int)len, prefix); + else + return 0; + + off += xsnprintf(path + off, size - off, + "%.*s\\usr\\bin;", (int)len, prefix); + return off; +} +#endif +#endif + static void setup_windows_environment(void) { char *tmp = getenv("TMPDIR"); @@ -3208,6 +3247,35 @@ static void setup_windows_environment(void) if (!tmp && (tmp = getenv("USERPROFILE"))) setenv("HOME", tmp, 1); } + + if (!getenv("PLINK_PROTOCOL")) + setenv("PLINK_PROTOCOL", "ssh", 0); + +#ifdef ENSURE_MSYSTEM_IS_SET + if (!(tmp = getenv("MSYSTEM")) || !tmp[0]) { + const char *home = getenv("HOME"), *path = getenv("PATH"); + char buf[32768]; + size_t off = 0; + + setenv("MSYSTEM", ENSURE_MSYSTEM_IS_SET, 1); + + if (home) + off += xsnprintf(buf + off, sizeof(buf) - off, + "%s\\bin;", home); + off += append_system_bin_dirs(buf + off, sizeof(buf) - off); + if (path) + off += xsnprintf(buf + off, sizeof(buf) - off, + "%s", path); + else if (off > 0) + buf[off - 1] = '\0'; + else + buf[0] = '\0'; + setenv("PATH", buf, 1); + } +#endif + + if (!getenv("LC_ALL") && !getenv("LC_CTYPE") && !getenv("LANG")) + setenv("LC_CTYPE", "C.UTF-8", 1); } int mingw_platform_has_symlinks(void) diff --git a/compat/posix.h b/compat/posix.h index e2e794cad7d419..108d38e0df58d7 100644 --- a/compat/posix.h +++ b/compat/posix.h @@ -148,6 +148,9 @@ #include #include #include +#ifndef NO_WRITEV +#include +#endif #include #ifndef NO_SYS_SELECT_H #include @@ -334,6 +337,17 @@ int git_lstat(const char *, struct stat *); ssize_t git_pread(int fd, void *buf, size_t count, off_t offset); #endif +#ifdef NO_WRITEV +#define writev git_writev +#define iovec git_iovec +struct git_iovec { + void *iov_base; + size_t iov_len; +}; + +ssize_t git_writev(int fd, const struct iovec *iov, int iovcnt); +#endif + #ifdef NO_SETENV #define setenv gitsetenv int gitsetenv(const char *, const char *, int); @@ -487,7 +501,6 @@ int git_qsort_s(void *base, size_t nmemb, size_t size, } while (0) #ifdef NO_NSEC -#undef USE_NSEC #define ST_CTIME_NSEC(st) 0 #define ST_MTIME_NSEC(st) 0 #else diff --git a/compat/pread.c b/compat/pread.c index 484e6d4c716ef6..ac7d058cb895db 100644 --- a/compat/pread.c +++ b/compat/pread.c @@ -7,6 +7,8 @@ ssize_t git_pread(int fd, void *buf, size_t count, off_t offset) ssize_t rc; current_offset = lseek(fd, 0, SEEK_CUR); + if (current_offset < 0) + return -1; if (lseek(fd, offset, SEEK_SET) < 0) return -1; diff --git a/compat/writev.c b/compat/writev.c new file mode 100644 index 00000000000000..540f66de61fb09 --- /dev/null +++ b/compat/writev.c @@ -0,0 +1,41 @@ +#include "../git-compat-util.h" +#include "../wrapper.h" + +ssize_t git_writev(int fd, const struct iovec *iov, int iovcnt) +{ + size_t sum = 0; + + if (iovcnt <= 0) { + errno = EINVAL; + return -1; + } + + /* + * According to writev(3p), the syscall shall error with EINVAL in case + * the sum of `iov_len` overflows `ssize_t`. + */ + for (int i = 0; i < iovcnt; i++) { + if (iov[i].iov_len > maximum_signed_value_of_type(ssize_t) || + unsigned_add_overflows(iov[i].iov_len, sum) || + iov[i].iov_len + sum > maximum_signed_value_of_type(ssize_t)) { + errno = EINVAL; + return -1; + } + + sum += iov[i].iov_len; + } + + /* + * We only ever write the first non-empty vector so that we can + * guarantee the call to be non-interleaving as guaranteed by POSIX. + * This works just fine as callers have to loop around writev anyway. + */ + for (int i = 0; i < iovcnt; i++) { + if (!iov[i].iov_len) + continue; + return xwrite(fd, iov[i].iov_base, iov[i].iov_len); + } + + /* When all iovec members were zero we ought to return 0 according to POSIX. */ + return 0; +} diff --git a/config.c b/config.c index 1bdd702e7a3969..d9019e7e6c34b0 100644 --- a/config.c +++ b/config.c @@ -1271,6 +1271,15 @@ ssize_t git_config_ssize_t(const char *name, const char *value, return ret; } +size_t git_config_size_t(const char *name, const char *value, + const struct key_value_info *kvi) +{ + size_t ret; + if (!git_parse_size_t(value, &ret)) + die_bad_number(name, value, kvi); + return ret; +} + double git_config_double(const char *name, const char *value, const struct key_value_info *kvi) { diff --git a/config.h b/config.h index 31fe3e29611e11..b66dd08007c97a 100644 --- a/config.h +++ b/config.h @@ -282,6 +282,9 @@ unsigned long git_config_ulong(const char *, const char *, ssize_t git_config_ssize_t(const char *, const char *, const struct key_value_info *); +size_t git_config_size_t(const char *, const char *, + const struct key_value_info *); + /** * Identically to `git_config_double`, but for double-precision floating point * values. diff --git a/config.mak.uname b/config.mak.uname index 9ebd240378ca59..d28ec12a2575aa 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -154,6 +154,7 @@ ifeq ($(uname_S),Darwin) HAVE_DEV_TTY = YesPlease COMPAT_OBJS += compat/precompose_utf8.o BASIC_CFLAGS += -DPRECOMPOSE_UNICODE + DARWIN_REGEXEC = YesPlease BASIC_CFLAGS += -DPROTECT_HFS_DEFAULT=1 HAVE_BSD_SYSCTL = YesPlease FREAD_READS_DIRECTORIES = UnfortunatelyYes @@ -465,14 +466,8 @@ ifeq ($(uname_S),Windows) GIT_VERSION := $(GIT_VERSION).MSVC pathsep = ; # Assume that this is built in Git for Windows' SDK - ifeq (MINGW32,$(MSYSTEM)) - prefix = /mingw32 - else - ifeq (CLANGARM64,$(MSYSTEM)) - prefix = /clangarm64 - else - prefix = /mingw64 - endif + ifneq (,$(MSYSTEM)) + prefix = $(MINGW_PREFIX) endif # Prepend MSVC 64-bit tool-chain to PATH. # @@ -483,6 +478,7 @@ ifeq ($(uname_S),Windows) SANE_TOOL_PATH ?= $(msvc_bin_dir_msys) HAVE_ALLOCA_H = YesPlease NO_PREAD = YesPlease + NO_WRITEV = YesPlease NEEDS_CRYPTO_WITH_SSL = YesPlease NO_LIBGEN_H = YesPlease NO_POLL = YesPlease @@ -524,7 +520,8 @@ ifeq ($(uname_S),Windows) NO_POSIX_GOODIES = UnfortunatelyYes NATIVE_CRLF = YesPlease DEFAULT_HELP_FORMAT = html -ifeq (/mingw64,$(subst 32,64,$(subst clangarm,mingw,$(prefix)))) + SKIP_DASHED_BUILT_INS = YabbaDabbaDoo +ifneq (,$(MINGW_PREFIX)) # Move system config into top-level /etc/ ETC_GITCONFIG = ../etc/gitconfig ETC_GITATTRIBUTES = ../etc/gitattributes @@ -540,7 +537,9 @@ endif compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/trace2_win32_process_info.o \ compat/win32/dirent.o - COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" + COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY \ + -DENSURE_MSYSTEM_IS_SET="\"$(MSYSTEM)\"" -DMINGW_PREFIX="\"$(patsubst /%,%,$(MINGW_PREFIX))\"" \ + -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -ENTRY:wmainCRTStartup -SUBSYSTEM:CONSOLE # invalidcontinue.obj allows Git's source code to close the same file # handle twice, or to access the osfhandle of an already-closed stdout @@ -697,6 +696,7 @@ ifeq ($(uname_S),MINGW) pathsep = ; HAVE_ALLOCA_H = YesPlease NO_PREAD = YesPlease + NO_WRITEV = YesPlease NEEDS_CRYPTO_WITH_SSL = YesPlease NO_LIBGEN_H = YesPlease NO_POLL = YesPlease @@ -716,6 +716,7 @@ ifeq ($(uname_S),MINGW) FSMONITOR_DAEMON_BACKEND = win32 FSMONITOR_OS_SETTINGS = win32 + SKIP_DASHED_BUILT_INS = YabbaDabbaDoo RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease @@ -754,26 +755,25 @@ ifeq ($(uname_S),MINGW) ifneq (,$(findstring -O,$(filter-out -O0 -Og,$(CFLAGS)))) BASIC_LDFLAGS += -Wl,--dynamicbase endif - ifeq (MINGW32,$(MSYSTEM)) - prefix = /mingw32 - HOST_CPU = i686 - BASIC_LDFLAGS += -Wl,--pic-executable,-e,_mainCRTStartup - endif - ifeq (MINGW64,$(MSYSTEM)) - prefix = /mingw64 - HOST_CPU = x86_64 - BASIC_LDFLAGS += -Wl,--pic-executable,-e,mainCRTStartup - else ifeq (CLANGARM64,$(MSYSTEM)) - prefix = /clangarm64 - HOST_CPU = aarch64 - BASIC_LDFLAGS += -Wl,--pic-executable,-e,mainCRTStartup - else - COMPAT_CFLAGS += -D_USE_32BIT_TIME_T - BASIC_LDFLAGS += -Wl,--large-address-aware + ifneq (,$(MSYSTEM)) + ifeq ($(MINGW_PREFIX),$(filter-out /%,$(MINGW_PREFIX))) + # Override if empty or does not start with a slash + MINGW_PREFIX := /$(shell echo '$(MSYSTEM)' | tr A-Z a-z) + endif + prefix = $(MINGW_PREFIX) + HOST_CPU = $(patsubst %-w64-mingw32,%,$(MINGW_CHOST)) + BASIC_LDFLAGS += -Wl,--pic-executable + COMPAT_CFLAGS += -DDETECT_MSYS_TTY \ + -DENSURE_MSYSTEM_IS_SET="\"$(MSYSTEM)\"" \ + -DMINGW_PREFIX="\"$(patsubst /%,%,$(MINGW_PREFIX))\"" + ifeq (MINGW32,$(MSYSTEM)) + BASIC_LDFLAGS += -Wl,--large-address-aware + endif + # Move system config into top-level /etc/ + ETC_GITCONFIG = ../etc/gitconfig + ETC_GITATTRIBUTES = ../etc/gitattributes endif - CC = gcc - COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY \ - -fstack-protector-strong + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -fstack-protector-strong EXTLIBS += -lntdll EXTRA_PROGRAMS += headless-git$X INSTALL = /bin/install @@ -781,11 +781,7 @@ ifeq ($(uname_S),MINGW) HAVE_LIBCHARSET_H = YesPlease USE_GETTEXT_SCHEME = fallthrough USE_LIBPCRE = YesPlease - ifeq (/mingw64,$(subst 32,64,$(subst clangarm,mingw,$(prefix)))) - # Move system config into top-level /etc/ - ETC_GITCONFIG = ../etc/gitconfig - ETC_GITATTRIBUTES = ../etc/gitattributes - endif + NO_PYTHON = endif ifeq ($(uname_S),QNX) COMPAT_CFLAGS += -DSA_RESTART=0 diff --git a/configure.ac b/configure.ac index cfb50112bf81ff..fc956776ab0445 100644 --- a/configure.ac +++ b/configure.ac @@ -351,12 +351,6 @@ GIT_PARSE_WITH(iconv)) ## --enable-FEATURE[=ARG] and --disable-FEATURE # -# Define USE_NSEC below if you want git to care about sub-second file mtimes -# and ctimes. Note that you need recent glibc (at least 2.2.4) for this, and -# it will BREAK YOUR LOCAL DIFFS! show-diff and anything using it will likely -# randomly break unless your underlying filesystem supports those sub-second -# times (my ext3 doesn't). -# # Define USE_STDEV below if you want git to care about the underlying device # change being considered an inode change from the update-index perspective. diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index a57c4b464fa456..1af245c15a89d2 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -256,7 +256,14 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows") _CONSOLE DETECT_MSYS_TTY STRIP_EXTENSION=".exe" NO_SYMLINK_HEAD UNRELIABLE_FSTAT NOGDI OBJECT_CREATION_MODE=1 __USE_MINGW_ANSI_STDIO=0 OVERRIDE_STRDUP MMAP_PREVENTS_DELETE USE_WIN32_MMAP - HAVE_WPGMPTR ENSURE_MSYSTEM_IS_SET HAVE_RTLGENRANDOM) + HAVE_WPGMPTR HAVE_RTLGENRANDOM) + if(CMAKE_GENERATOR_PLATFORM STREQUAL "x64") + add_compile_definitions(ENSURE_MSYSTEM_IS_SET="MINGW64" MINGW_PREFIX="mingw64") + elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "arm64") + add_compile_definitions(ENSURE_MSYSTEM_IS_SET="CLANGARM64" MINGW_PREFIX="clangarm64") + elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "x86") + add_compile_definitions(ENSURE_MSYSTEM_IS_SET="MINGW32" MINGW_PREFIX="mingw32") + endif() list(APPEND compat_SOURCES compat/mingw.c compat/winansi.c @@ -378,7 +385,7 @@ endif() #function checks set(function_checks strcasestr memmem strlcpy strtoimax strtoumax strtoull - setenv mkdtemp poll pread memmem) + setenv mkdtemp poll pread memmem writev) #unsetenv,hstrerror are incompatible with windows build if(NOT WIN32) @@ -423,6 +430,10 @@ if(NOT HAVE_MEMMEM) list(APPEND compat_SOURCES compat/memmem.c) endif() +if(NOT HAVE_WRITEV) + list(APPEND compat_SOURCES compat/writev.c) +endif() + if(NOT WIN32) if(NOT HAVE_UNSETENV) list(APPEND compat_SOURCES compat/unsetenv.c) @@ -519,6 +530,9 @@ if(NOT HAVE_REGEX) include_directories(${CMAKE_SOURCE_DIR}/compat/regex) list(APPEND compat_SOURCES compat/regex/regex.c ) add_compile_definitions(NO_REGEX NO_MBSUPPORT GAWK) +elseif(APPLE) + list(APPEND compat_SOURCES compat/darwin/regexec.c) + add_compile_definitions(DARWIN_REGEXEC) endif() diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index e8757877104eb9..9f8b9b50ff834c 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -1735,49 +1735,59 @@ __git_checkout_default_dwim_mode () _git_checkout () { - __git_has_doubledash && return + if ! __git_has_doubledash; then + local dwim_opt="$(__git_checkout_default_dwim_mode)" - local dwim_opt="$(__git_checkout_default_dwim_mode)" + case "$prev" in + -b|-B|--orphan) + # Complete local branches (and DWIM branch + # remote branch names) for an option argument + # specifying a new branch name. This is for + # convenience, assuming new branches are + # possibly based on pre-existing branch names. + __git_complete_refs $dwim_opt --mode="heads" + return + ;; + *) + ;; + esac - case "$prev" in - -b|-B|--orphan) - # Complete local branches (and DWIM branch - # remote branch names) for an option argument - # specifying a new branch name. This is for - # convenience, assuming new branches are - # possibly based on pre-existing branch names. - __git_complete_refs $dwim_opt --mode="heads" - return - ;; - *) - ;; - esac + case "$cur" in + --conflict=*) + __gitcomp "diff3 merge zdiff3" "" "${cur##--conflict=}" + return + ;; + --*) + __gitcomp_builtin checkout + return + ;; + *) + # At this point, we've already handled special completion for + # the arguments to -b/-B, and --orphan. There are 3 main + # things left we can possibly complete: + # 1) a start-point for -b/-B, -d/--detach, or --orphan + # 2) a remote head, for --track + # 3) an arbitrary reference, possibly including DWIM names + # + + if [ -n "$(__git_find_on_cmdline "-b -B -d --detach --orphan")" ]; then + __git_complete_refs --mode="refs" + elif [ -n "$(__git_find_on_cmdline "-t --track")" ]; then + __git_complete_refs --mode="remote-heads" + else + __git_complete_refs $dwim_opt --mode="refs" + fi + ;; + esac + fi - case "$cur" in - --conflict=*) - __gitcomp "diff3 merge zdiff3" "" "${cur##--conflict=}" - ;; - --*) - __gitcomp_builtin checkout - ;; - *) - # At this point, we've already handled special completion for - # the arguments to -b/-B, and --orphan. There are 3 main - # things left we can possibly complete: - # 1) a start-point for -b/-B, -d/--detach, or --orphan - # 2) a remote head, for --track - # 3) an arbitrary reference, possibly including DWIM names - # + if [ ${#COMPREPLY[@]} -eq 0 ]; then + __git_complete_index_file "" + fi - if [ -n "$(__git_find_on_cmdline "-b -B -d --detach --orphan")" ]; then - __git_complete_refs --mode="refs" - elif [ -n "$(__git_find_on_cmdline "-t --track")" ]; then - __git_complete_refs --mode="remote-heads" - else - __git_complete_refs $dwim_opt --mode="refs" - fi - ;; - esac + if [ ${#COMPREPLY[@]} -eq 0 ]; then + __git_complete_index_file "--others --directory" + fi } __git_sequencer_inprogress_options="--continue --quit --abort --skip" @@ -1947,35 +1957,48 @@ __git_diff_difftool_options="--cached --staged _git_diff () { - __git_has_doubledash && return + if ! __git_has_doubledash; then + case "$cur" in + --diff-algorithm=*) + __gitcomp "$__git_diff_algorithms" \ + "" "${cur##--diff-algorithm=}" + return + ;; + --submodule=*) + __gitcomp "$__git_diff_submodule_formats" \ + "" "${cur##--submodule=}" + return + ;; + --color-moved=*) + __gitcomp "$__git_color_moved_opts" \ + "" "${cur##--color-moved=}" + return + ;; + --color-moved-ws=*) + __gitcomp "$__git_color_moved_ws_opts" \ + "" "${cur##--color-moved-ws=}" + return + ;; + --ws-error-highlight=*) + __gitcomp "$__git_ws_error_highlight_opts" \ + "" "${cur##--ws-error-highlight=}" + return + ;; + --*) + __gitcomp "$__git_diff_difftool_options" + return + ;; + esac + __git_complete_revlist_file + fi - case "$cur" in - --diff-algorithm=*) - __gitcomp "$__git_diff_algorithms" "" "${cur##--diff-algorithm=}" - return - ;; - --submodule=*) - __gitcomp "$__git_diff_submodule_formats" "" "${cur##--submodule=}" - return - ;; - --color-moved=*) - __gitcomp "$__git_color_moved_opts" "" "${cur##--color-moved=}" - return - ;; - --color-moved-ws=*) - __gitcomp "$__git_color_moved_ws_opts" "" "${cur##--color-moved-ws=}" - return - ;; - --ws-error-highlight=*) - __gitcomp "$__git_ws_error_highlight_opts" "" "${cur##--ws-error-highlight=}" - return - ;; - --*) - __gitcomp "$__git_diff_difftool_options" - return - ;; - esac - __git_complete_revlist_file + if [ ${#COMPREPLY[@]} -eq 0 ]; then + __git_complete_index_file "" + fi + + if [ ${#COMPREPLY[@]} -eq 0 ]; then + __git_complete_index_file "--others --directory" + fi } __git_mergetools_common="diffuse diffmerge ecmerge emerge kdiff3 meld opendiff @@ -2137,6 +2160,71 @@ _git_help () fi } +__git_history_has_revision () +{ + local i + + for ((i = __git_cmd_idx + 2; i < cword; i++)); do + case "${words[i]}" in + -*) + ;; + *) + return 0 + ;; + esac + done + return 1 +} + +_git_history () +{ + local subcommands subcommand + + __git_resolve_builtins "history" + + subcommands="$___git_resolved_builtins" + subcommand="$(__git_find_subcommand "$subcommands")" + + if [ -z "$subcommand" ]; then + __gitcomp "$subcommands" + return + fi + + if ! __git_has_doubledash; then + case "$cur" in + --empty=*) + case "$subcommand" in + drop|fixup) + __gitcomp "drop keep abort" "" \ + "${cur##--empty=}" + ;; + esac + return + ;; + --update-refs=*) + __gitcomp "branches head" "" \ + "${cur##--update-refs=}" + return + ;; + --*) + __gitcomp_builtin "history_$subcommand" + return + ;; + esac + fi + + if ! __git_history_has_revision; then + __git_complete_refs + return + fi + + case "$subcommand" in + split) + __git_complete_index_file "--cached" + ;; + esac +} + _git_init () { case "$cur" in diff --git a/contrib/subtree/git-subtree.sh b/contrib/subtree/git-subtree.sh index 791fd8260c4703..e9c7ca7cf513b1 100755 --- a/contrib/subtree/git-subtree.sh +++ b/contrib/subtree/git-subtree.sh @@ -278,6 +278,20 @@ main () { "cmd_$arg_command" "$@" } +# Usage: reject_if_v2_config REV +# +# Bails if we find .git-subtree/config. This file is used by the RIIR +# git-subtree, which can read data from this script, but which generates +# data that this script cannot cope with. So if we find that the user's +# project has already been processed with the new tool, we stop, to +# avoid generating broken output. +reject_if_v2_config () { + local config=.git-subtree/config + if git rev-parse --verify -q "$rev:$config"; then + die "fatal: tree contains $config: has been processed with new standalone (Rust) git-subtree; use that tool instead of this one. See https://codeberg.org/diziet/git-subtree https://crates.io/crates/git-subtree" + fi +} + # Usage: cache_setup cache_setup () { assert test $# = 0 @@ -846,6 +860,7 @@ process_split_commit () { # Or: cmd_add REPOSITORY REF cmd_add () { + reject_if_v2_config HEAD ensure_clean if test $# -eq 1 @@ -934,6 +949,8 @@ cmd_split () { die "fatal: you must provide exactly one revision, and optionally a repository. Got: '$*'" fi + reject_if_v2_config "$rev" + # Now validate prefix against the commit, not the working tree if ! git cat-file -e "$rev:$dir" 2>/dev/null then @@ -1034,6 +1051,7 @@ cmd_merge () { then repository="$2" fi + reject_if_v2_config HEAD ensure_clean if test -n "$arg_addmerge_squash" diff --git a/contrib/subtree/t/t7900-subtree.sh b/contrib/subtree/t/t7900-subtree.sh index c10f283b38d92d..7c06da476b0485 100755 --- a/contrib/subtree/t/t7900-subtree.sh +++ b/contrib/subtree/t/t7900-subtree.sh @@ -439,6 +439,24 @@ test_expect_success 'split sub dir/ with --rejoin' ' ) ' +test_expect_success 'split fail on RIIR git subtree data' ' + subtree_test_create_repo "$test_count" && + subtree_test_create_repo "$test_count/sub proj" && + test_create_commit "$test_count" main1 && + test_create_commit "$test_count/sub proj" sub1 && + ( + cd "$test_count" && + git fetch ./"sub proj" HEAD && + git subtree add --prefix="sub dir" FETCH_HEAD && + # simulate RIIR git-subtree generated data + mkdir .git-subtree && + echo "# sabotage" >.git-subtree/config && + git add .git-subtree/config && + git commit -m sabotage && + test_must_fail git subtree split -P "sub dir" HEAD + ) +' + # Tests that commits from other subtrees are not processed as # part of a split. # diff --git a/delta.h b/delta.h index eb5c6d2fdb9c51..42a211905dd2b7 100644 --- a/delta.h +++ b/delta.h @@ -14,7 +14,7 @@ struct delta_index; * using free_delta_index(). */ struct delta_index * -create_delta_index(const void *buf, unsigned long bufsize); +create_delta_index(const void *buf, size_t bufsize); /* * free_delta_index: free the index created by create_delta_index() @@ -28,7 +28,7 @@ void free_delta_index(struct delta_index *index); * * Given pointer must be what create_delta_index() returned, or NULL. */ -unsigned long sizeof_delta_index(struct delta_index *index); +size_t sizeof_delta_index(struct delta_index *index); /* * create_delta: create a delta from given index for the given buffer @@ -42,8 +42,8 @@ unsigned long sizeof_delta_index(struct delta_index *index); */ void * create_delta(const struct delta_index *index, - const void *buf, unsigned long bufsize, - unsigned long *delta_size, unsigned long max_delta_size); + const void *buf, size_t bufsize, + size_t *delta_size, size_t max_delta_size); /* * diff_delta: create a delta from source buffer to target buffer @@ -54,9 +54,9 @@ create_delta(const struct delta_index *index, * updated with its size. The returned buffer must be freed by the caller. */ static inline void * -diff_delta(const void *src_buf, unsigned long src_bufsize, - const void *trg_buf, unsigned long trg_bufsize, - unsigned long *delta_size, unsigned long max_delta_size) +diff_delta(const void *src_buf, size_t src_bufsize, + const void *trg_buf, size_t trg_bufsize, + size_t *delta_size, size_t max_delta_size) { struct delta_index *index = create_delta_index(src_buf, src_bufsize); if (index) { diff --git a/diagnose.c b/diagnose.c index 5092bf80d35fdd..9c652d36a6649c 100644 --- a/diagnose.c +++ b/diagnose.c @@ -112,7 +112,7 @@ static void loose_objs_stats(struct strbuf *buf, const char *path) while ((e = readdir_skip_dot_and_dotdot(dir)) != NULL) if (get_dtype(e, &count_path, 0) == DT_DIR && strlen(e->d_name) == 2 && - !hex_to_bytes(&c, e->d_name, 1)) { + !hex_to_bytes(&c, e->d_name, 1, HEX_KIND_OID)) { strbuf_setlen(&count_path, base_path_len); strbuf_addf(&count_path, "%s/", e->d_name); total += (count = count_files(&count_path)); diff --git a/diff-delta.c b/diff-delta.c index 43c339f01061ca..7cbedeb5071bcc 100644 --- a/diff-delta.c +++ b/diff-delta.c @@ -125,14 +125,14 @@ struct unpacked_index_entry { }; struct delta_index { - unsigned long memsize; + size_t memsize; const void *src_buf; - unsigned long src_size; + size_t src_size; unsigned int hash_mask; struct index_entry *hash[FLEX_ARRAY]; }; -struct delta_index * create_delta_index(const void *buf, unsigned long bufsize) +struct delta_index * create_delta_index(const void *buf, size_t bufsize) { unsigned int i, hsize, hmask, entries, prev_val, *hash_count; const unsigned char *data, *buffer = buf; @@ -140,7 +140,7 @@ struct delta_index * create_delta_index(const void *buf, unsigned long bufsize) struct unpacked_index_entry *entry, **hash; struct index_entry *packed_entry, **packed_hash; void *mem; - unsigned long memsize; + size_t memsize; if (!buf || !bufsize) return NULL; @@ -302,7 +302,7 @@ void free_delta_index(struct delta_index *index) free(index); } -unsigned long sizeof_delta_index(struct delta_index *index) +size_t sizeof_delta_index(struct delta_index *index) { if (index) return index->memsize; @@ -318,8 +318,8 @@ unsigned long sizeof_delta_index(struct delta_index *index) void * create_delta(const struct delta_index *index, - const void *trg_buf, unsigned long trg_size, - unsigned long *delta_size, unsigned long max_size) + const void *trg_buf, size_t trg_size, + size_t *delta_size, size_t max_size) { unsigned int i, val; off_t outpos, moff; diff --git a/diff-hunks.c b/diff-hunks.c new file mode 100644 index 00000000000000..9830df07fa1aa5 --- /dev/null +++ b/diff-hunks.c @@ -0,0 +1,1034 @@ +/* + * Precomputed diff hunks, keyed by diff input. + * + * A single store at .git/objects/info/diff-hunks maps an (old blob, + * new blob, xdl_opts) key to the hunk coordinates of diffing the pair. + * The key determines the diff result (only trim-stable pairs are + * recorded; see diff-hunks.h), so an entry is valid in any context it + * recurs in, independent of path. Reading is on by default + * (core.diffHunks); writing is off by default and enabled per run or + * by configuration (see diff_hunks_write_enabled), so an ordinary + * command populates the store only during a warming run the + * repository owner opts into. + * + * File layout: + * Header: "DHPF"(4) + version(1) + hash_version(1) + * + num_chunks(1) + reserved(1) + * Table of contents (chunk-format) + * DHIX chunk: sorted entries, each + * old_blob_oid, new_blob_oid, xdl_opts(4), hdat_offset(4) + * DHDT chunk: per entry, num_hunks(4) followed by that many 16-byte hunks + * Trailing hash checksum + */ +#include "git-compat-util.h" +#include "chunk-format.h" +#include "config.h" +#include "csum-file.h" +#include "diff-hunks.h" +#include "diff-provider-internal.h" +#include "diff.h" +#include "gettext.h" +#include "hash.h" +#include "hashmap.h" +#include "lockfile.h" +#include "odb.h" +#include "path.h" +#include "repo-settings.h" +#include "repository.h" +#include "strbuf.h" +#include "wrapper.h" + +#define DIFF_HUNKS_SIGNATURE 0x44485046 /* "DHPF" */ +/* + * Bump when the on-disk format changes, or when xdiff's emitted hunk + * coordinates change for a fixed (blobs, xdl_opts) key: an old store + * would otherwise serve stale hunks and change command output. + */ +#define DIFF_HUNKS_VERSION 1 +#define DIFF_HUNKS_HEADER_SIZE 8 + +#define DIFF_HUNKS_CHUNKID_INDEX 0x44484958 /* "DHIX" */ +#define DIFF_HUNKS_CHUNKID_DATA 0x44484454 /* "DHDT" */ + +/* + * Each hunk is 16 bytes on disk: + * old_start(4) old_count(4) new_start(4) new_count(4) + */ +#define DIFF_HUNKS_HUNK_SIZE (4 * sizeof(uint32_t)) + +/* + * Result of a store lookup: num_hunks records encoded in the store's mmap, + * valid until the store is freed. Read them with nth_precomputed_hunk(). + */ +struct precomputed_entry { + uint32_t num_hunks; + const unsigned char *hunk_data; +}; + +/* Decode a single hunk from the raw on-disk format. */ +static inline void decode_precomputed_hunk(const unsigned char *data, + struct precomputed_hunk *h) +{ + h->old_start = get_be32(data); + h->old_count = get_be32(data + 4); + h->new_start = get_be32(data + 8); + h->new_count = get_be32(data + 12); +} + +/* Decode the nth hunk of a lookup result into *h. */ +static inline void nth_precomputed_hunk(const struct precomputed_entry *e, + uint32_t n, struct precomputed_hunk *h) +{ + decode_precomputed_hunk(e->hunk_data + (size_t)n * DIFF_HUNKS_HUNK_SIZE, h); +} + +/* Byte length of the (old_oid, new_oid, xdl_opts) lookup key. */ +static size_t store_index_key_size(const struct git_hash_algo *algo) +{ + return 2 * algo->rawsz + sizeof(uint32_t); +} + +/* Index entry: the lookup key followed by the 4-byte offset into DHDT. */ +static size_t store_index_entry_size(const struct git_hash_algo *algo) +{ + return store_index_key_size(algo) + sizeof(uint32_t); +} + +/* + * The smallest a valid store file can be: the header, a table of contents + * with one entry per chunk plus a terminating entry, and the trailing + * checksum. + */ +static size_t store_min_size(const struct git_hash_algo *algo, + uint8_t num_chunks) +{ + size_t toc_size = (num_chunks + 1) * CHUNK_TOC_ENTRY_SIZE; + + return DIFF_HUNKS_HEADER_SIZE + toc_size + algo->rawsz; +} + +/* + * Decode an index entry's key into pointers to the two oids and the + * xdl_opts value (on-disk: old_oid, new_oid, then xdl_opts as a + * big-endian uint32). + */ +static void decode_store_index_key(const unsigned char *entry, unsigned int rawsz, + const unsigned char **old_hash, + const unsigned char **new_hash, + uint32_t *xdl_opts) +{ + *old_hash = entry; + *new_hash = entry + rawsz; + *xdl_opts = get_be32(entry + 2 * rawsz); +} + +/* The DHDT offset stored in an index entry, in the field after its key. */ +static uint32_t index_entry_hdat_offset(const unsigned char *entry, size_t keysz) +{ + return get_be32(entry + keysz); +} + +static char *diff_hunks_store_path(struct repository *r) +{ + return xstrfmt("%s/info/diff-hunks", repo_get_object_directory(r)); +} + +struct diff_hunks_store { + const unsigned char *data; + size_t data_len; + const struct git_hash_algo *hash_algo; + const unsigned char *index; + uint32_t num_entries; + const unsigned char *hdat; + size_t hdat_size; + + /* Consultation counters; see diff_hunks_read_stats(). */ + unsigned long read_hits; + unsigned long read_misses; +}; + +static void free_store(struct diff_hunks_store *s) +{ + if (!s) + return; + if (s->data) + munmap((void *)s->data, s->data_len); + free(s); +} + +/* + * Open, mmap, and parse the store at fname. Returns the parsed store + * or NULL on any error. The diff output is unaffected either way; + * corruption is reported by verify, not treated as fatal here. + */ +static struct diff_hunks_store *load_store_at( + const struct git_hash_algo *repo_algo, const char *fname) +{ + struct diff_hunks_store *s; + struct chunkfile *cf; + int fd; + struct stat st; + void *data; + const unsigned char *p; + uint8_t num_chunks; + size_t index_size, entry_size, data_len; + + fd = git_open(fname); + if (fd < 0) + return NULL; + if (fstat(fd, &st) || st.st_size < DIFF_HUNKS_HEADER_SIZE) { + close(fd); + return NULL; + } + data_len = xsize_t(st.st_size); + data = xmmap(NULL, data_len, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + p = data; + + num_chunks = p[6]; + + /* + * Reject a file that is not a readable store: wrong signature, + * version, or object hash, or too small to hold the table of + * contents that read_table_of_contents() walks (it dereferences + * each entry before range-checking its offset). + */ + if (get_be32(p) != DIFF_HUNKS_SIGNATURE || + p[4] != DIFF_HUNKS_VERSION || + p[5] != oid_version(repo_algo) || + data_len < store_min_size(repo_algo, num_chunks)) { + munmap(data, data_len); + return NULL; + } + + /* + * The trailing checksum is not verified here: the writer fsyncs + * and commits atomically, so a committed file is intact, and + * every record is bounds-checked at read (see precomputed_entry_at). + * The checksum is checked separately, by diff_hunks_verify(). + */ + + CALLOC_ARRAY(s, 1); + s->data = data; + s->data_len = data_len; + s->hash_algo = repo_algo; + + cf = init_chunkfile(NULL); + if (read_table_of_contents_quiet(cf, p, data_len, + DIFF_HUNKS_HEADER_SIZE, num_chunks, 1, + repo_algo) || + pair_chunk(cf, DIFF_HUNKS_CHUNKID_INDEX, &s->index, &index_size) || + pair_chunk(cf, DIFF_HUNKS_CHUNKID_DATA, &s->hdat, &s->hdat_size)) { + free_chunkfile(cf); + goto corrupt; + } + free_chunkfile(cf); + + entry_size = store_index_entry_size(s->hash_algo); + if (index_size % entry_size) + goto corrupt; + s->num_entries = index_size / entry_size; + return s; + +corrupt: + free_store(s); + return NULL; +} + +static struct diff_hunks_store *diff_hunks_store_load(struct repository *r) +{ + struct diff_hunks_store *s; + char *fname; + + prepare_repo_settings(r); + if (!r->settings.core_diff_hunks) + return NULL; + + fname = diff_hunks_store_path(r); + s = load_store_at(r->hash_algo, fname); + free(fname); + return s; +} + +struct diff_hunks_store *repo_diff_hunks_store(struct repository *r) +{ + if (!r->objects) + return NULL; + if (r->objects->diff_hunks_store_attempted) + return r->objects->diff_hunks_store; + r->objects->diff_hunks_store_attempted = 1; + r->objects->diff_hunks_store = diff_hunks_store_load(r); + return r->objects->diff_hunks_store; +} + +void diff_hunks_read_stats(struct repository *r, + unsigned long *hits, unsigned long *misses) +{ + struct diff_hunks_store *s = repo_diff_hunks_store(r); + + *hits = s ? s->read_hits : 0; + *misses = s ? s->read_misses : 0; +} + +void close_diff_hunks_store(struct object_database *o) +{ + if (!o->diff_hunks_store) + return; + free_store(o->diff_hunks_store); + o->diff_hunks_store = NULL; +} + +/* + * Fill *out with the hunk record at offset in the data chunk, and return + * 1 if the record is in bounds, 0 otherwise. The read path does not + * re-verify the checksum, and a valid checksum would not bound the count + * anyway, so a read must call this and use *out only when it returns + * non-zero. + * + * A record is a be32 hunk count followed by that many DIFF_HUNKS_HUNK_SIZE + * hunks. "remaining" tracks the bytes from offset to the end of the data + * chunk: it must hold the count, and after the count is consumed it must + * hold every hunk. The bounds are written as subtraction and division + * (never addition or multiplication) so a crafted offset or count cannot + * overflow them. + */ +static int precomputed_entry_at(const struct diff_hunks_store *s, + uint32_t offset, struct precomputed_entry *out) +{ + size_t remaining; + uint32_t num_hunks; + + if (offset >= s->hdat_size) + return 0; + remaining = s->hdat_size - offset; + if (remaining < sizeof(uint32_t)) + return 0; + + num_hunks = get_be32(s->hdat + offset); + remaining -= sizeof(uint32_t); + if (num_hunks > remaining / DIFF_HUNKS_HUNK_SIZE) + return 0; + + out->num_hunks = num_hunks; + out->hunk_data = s->hdat + offset + sizeof(uint32_t); + return 1; +} + +struct lookup_key { + const struct object_id *old_oid; + const struct object_id *new_oid; + int xdl_opts; + unsigned int rawsz; +}; + +/* + * The store's total order over (old_oid, new_oid, xdl_opts), defined + * once so the write-side sort (writer_entry_cmp) and the read-side + * search (store_bsearch_cmp) order the keys identically. + */ +static int cmp_store_index_key(const unsigned char *old_a, const unsigned char *new_a, + uint32_t opts_a, + const unsigned char *old_b, const unsigned char *new_b, + uint32_t opts_b, unsigned int rawsz) +{ + int cmp = memcmp(old_a, old_b, rawsz); + if (!cmp) + cmp = memcmp(new_a, new_b, rawsz); + if (!cmp) + cmp = (opts_a > opts_b) - (opts_a < opts_b); + return cmp; +} + +static int store_bsearch_cmp(const void *key, const void *entry_ptr) +{ + const struct lookup_key *k = key; + const unsigned char *old_hash, *new_hash; + uint32_t xdl_opts; + + decode_store_index_key(entry_ptr, k->rawsz, &old_hash, &new_hash, + &xdl_opts); + return cmp_store_index_key(k->old_oid->hash, k->new_oid->hash, + (uint32_t)k->xdl_opts, + old_hash, new_hash, xdl_opts, k->rawsz); +} + +static int store_get_one(struct diff_hunks_store *s, const struct lookup_key *key, + struct precomputed_entry *out) +{ + size_t entry_size = store_index_entry_size(s->hash_algo); + const unsigned char *found; + + found = bsearch(key, s->index, s->num_entries, entry_size, + store_bsearch_cmp); + if (!found) + return 0; + return precomputed_entry_at(s, + index_entry_hdat_offset(found, store_index_key_size(s->hash_algo)), + out); +} + +static int diff_hunks_store_get(struct diff_hunks_store *s, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + struct precomputed_entry *out) +{ + struct lookup_key key; + + if (!s) + return 0; + /* The null OID names no blob and cannot key an entry. */ + if (is_null_oid(old_oid) || is_null_oid(new_oid)) + return 0; + + key.old_oid = old_oid; + key.new_oid = new_oid; + key.xdl_opts = xdl_opts; + key.rawsz = s->hash_algo->rawsz; + + return store_get_one(s, &key, out); +} + +/* + * A recorded hunk sequence must satisfy the provider interface's + * shared check (diff_provider_check_hunk()) before it may be replayed: + * coordinates decode from be32 into long, which is 32-bit on some + * platforms, so a crafted value can decode negative or out of order. + * An entry that fails reads as a miss, so the caller recomputes. + */ +static int replayable_hunks(const struct precomputed_entry *e) +{ + struct diff_provider_hunks_check c = { 0 }; + uint32_t i; + + /* + * Replaying a record with no hunks would assert the blob pair + * equivalent, a claim the store must never make (the writer + * refuses to record one), so such a record is invalid. + */ + if (!e->num_hunks) + return 0; + for (i = 0; i < e->num_hunks; i++) { + struct precomputed_hunk h; + nth_precomputed_hunk(e, i, &h); + if (diff_provider_check_hunk(&c, h.old_start, h.old_count, + h.new_start, h.new_count)) + return 0; + } + return 1; +} + +int diff_hunks_replay(struct diff_hunks_store *s, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + xdl_emit_hunk_consume_func_t hunk_func, void *cb_data) +{ + struct precomputed_entry e; + uint32_t i; + + if (!s) + return 0; + if (!diff_hunks_store_get(s, old_oid, new_oid, xdl_opts, &e) || + !replayable_hunks(&e)) { + s->read_misses++; + return 0; + } + for (i = 0; i < e.num_hunks; i++) { + struct precomputed_hunk h; + nth_precomputed_hunk(&e, i, &h); + hunk_func(h.old_start, h.old_count, + h.new_start, h.new_count, cb_data); + } + s->read_hits++; + return 1; +} + +/* + * The store's consult implementation. The store is not + * authoritative, so it serves a recorded pair or passes; what the + * recording key cannot express, it excludes here with the + * stop-no-record disposition. None of those legs reaches + * diff_hunks_replay(), so none of them counts as a miss. + */ +static enum diff_provider_disposition +diff_hunks_store_consult(struct diff_provider *provider UNUSED, + const struct diff_provider_request *req, + diff_provider_fill_fn fill UNUSED, + void *fill_data UNUSED, + xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data) +{ + /* + * xpparam_t is the consult's parameter input. Its flags are + * the store key's xdl_opts; ignore_regex (-I) and anchors + * (--anchored) shape the diff outside the key, so such a + * request is neither served nor recorded. + * + * Adding an xpparam_t field fires this assert (its size no + * longer matches the reference struct). To clear it: (1) add + * the field to the reference struct below; then (2) decide how + * it affects the key: make it part of the key, or exclude + * diffs that use it here with the disposition below. The + * assert only tracks size: a same-size reorder or a changed + * field meaning slips past, so re-read the fields when it + * fires. + */ + (void)BUILD_ASSERT_OR_ZERO(sizeof(xpparam_t) == sizeof(struct { + unsigned long flags; + regex_t **ignore_regex; + size_t ignore_regex_nr; + char **anchors; + size_t anchors_nr; + })); + if (req->xpp->ignore_regex_nr || req->xpp->anchors_nr) + return DIFF_PROVIDER_DISP_STOP_NO_RECORD; + /* + * Break detection (-B) rescores the pair outside xpparam_t, so + * it is outside the key for the same reason. + */ + if (req->diffopt && req->diffopt->break_opt != -1) + return DIFF_PROVIDER_DISP_STOP_NO_RECORD; + + if (!req->old_oid || !req->new_oid) + return DIFF_PROVIDER_DISP_PASS; + if (diff_hunks_replay(repo_diff_hunks_store(req->repo), + req->old_oid, req->new_oid, + req->xpp->flags, hunk_cb, cb_data)) + return DIFF_PROVIDER_DISP_ANSWERED; + return DIFF_PROVIDER_DISP_PASS; +} + +/* + * The provider borrows the repository's store through + * repo_diff_hunks_store() per request; the object database owns the + * file and tears it down, so there is nothing to release here. + */ +struct diff_provider *diff_hunks_store_provider_new(void) +{ + struct diff_provider *p = xcalloc(1, sizeof(*p)); + + p->consult = diff_hunks_store_consult; + return p; +} + +/* Validate one store file. Returns 0 if valid or absent, -1 on any error. */ +static int verify_store_at(struct repository *r, const char *fname) +{ + struct diff_hunks_store *s; + size_t entry_size; + uint32_t i; + int fd; + int ret = 0; + + /* + * A file that cannot be opened is not evidence of corruption: + * report the open error, and reserve the corruption diagnostics + * below for a file that was read and failed to parse. + */ + fd = git_open(fname); + if (fd < 0) { + if (errno == ENOENT) + return 0; /* absent is valid */ + return error_errno(_("unable to open diff-hunks store %s"), + fname); + } + close(fd); + s = load_store_at(r->hash_algo, fname); + if (!s) + return error(_("diff-hunks store failed to load (corrupt " + "header or hash mismatch): %s"), fname); + if (!hashfile_checksum_valid(r->hash_algo, s->data, s->data_len)) { + error(_("diff-hunks store has incorrect checksum and is " + "likely corrupt: %s"), fname); + free_store(s); + return -1; + } + + entry_size = store_index_entry_size(s->hash_algo); + for (i = 0; i < s->num_entries; i++) { + const unsigned char *ep = s->index + st_mult(entry_size, i); + size_t keysz = store_index_key_size(s->hash_algo); + uint32_t offset = index_entry_hdat_offset(ep, keysz); + struct precomputed_entry pe; + + /* + * Keyed by (old_oid, new_oid, xdl_opts), increasing. memcmp + * matches cmp_store_index_key's integer comparison of + * xdl_opts because it is non-negative, so its big-endian + * bytes order the same as its value. + */ + if (i > 0 && memcmp(ep - entry_size, ep, keysz) >= 0) { + error(_("diff-hunks entry %u not in sorted order"), i); + ret = -1; + } + if (!precomputed_entry_at(s, offset, &pe)) { + error(_("diff-hunks entry %u has out-of-bounds hunk " + "data"), i); + ret = -1; + } else if (!replayable_hunks(&pe)) { + error(_("diff-hunks entry %u holds an invalid hunk " + "sequence"), i); + ret = -1; + } + } + + free_store(s); + return ret; +} + +int diff_hunks_verify(struct repository *r) +{ + char *fname = diff_hunks_store_path(r); + int ret = 0; + + if (verify_store_at(r, fname)) + ret = -1; + free(fname); + return ret; +} + +int diff_hunks_clear(struct repository *r) +{ + char *fname = diff_hunks_store_path(r); + int ret = 0; + + if (unlink(fname) && errno != ENOENT) + ret = error_errno(_("unable to remove %s"), fname); + free(fname); + return ret; +} + +struct writer_entry { + struct object_id old_oid; + struct object_id new_oid; + int xdl_opts; + uint32_t hdat_offset; +}; + +struct diff_hunks_writer { + struct repository *r; + struct writer_entry *entries; + size_t nr, alloc; + size_t seed_nr; /* nr after seeding; finish skips a no-op flush */ + unsigned force_flush : 1; /* seed pruned: rewrite even a no-op warm */ + struct strbuf hdat; + struct hashmap dedup; /* hunk block content -> offset in hdat */ +}; + +/* A record of one distinct hunk block already present in hdat. */ +struct dedup_entry { + struct hashmap_entry ent; + uint32_t offset; + uint32_t len; +}; + +static int dedup_cmp(const void *cmp_data, + const struct hashmap_entry *a, + const struct hashmap_entry *b, + const void *keydata UNUSED) +{ + const struct diff_hunks_writer *writer = cmp_data; + const struct dedup_entry *ea = container_of(a, const struct dedup_entry, ent); + const struct dedup_entry *eb = container_of(b, const struct dedup_entry, ent); + + if (ea->len != eb->len) + return 1; + return memcmp(writer->hdat.buf + ea->offset, + writer->hdat.buf + eb->offset, ea->len); +} + +static struct diff_hunks_writer *diff_hunks_writer_new(struct repository *r) +{ + struct diff_hunks_writer *w; + + CALLOC_ARRAY(w, 1); + w->r = r; + strbuf_init(&w->hdat, 0); + hashmap_init(&w->dedup, dedup_cmp, w, 0); + return w; +} + +static void strbuf_put_be32(struct strbuf *sb, uint32_t val) +{ + unsigned char buf[4]; + put_be32(buf, val); + strbuf_add(sb, buf, 4); +} + +/* + * The hunk block just appended at `start` is deduplicated: if an + * identical block is already in hdat, this copy is dropped and the + * earlier offset returned; otherwise it is kept and remembered. + * Distinct keys that diff to the same hunks then share one block. + */ +static uint32_t intern_block(struct diff_hunks_writer *w, size_t start) +{ + size_t len = w->hdat.len - start; + struct dedup_entry key, *found, *added; + + hashmap_entry_init(&key.ent, memhash(w->hdat.buf + start, len)); + key.offset = (uint32_t)start; + key.len = (uint32_t)len; + + found = hashmap_get_entry(&w->dedup, &key, ent, NULL); + if (found) { + strbuf_setlen(&w->hdat, start); + return found->offset; + } + + added = xmalloc(sizeof(*added)); + hashmap_entry_init(&added->ent, key.ent.hash); + added->offset = key.offset; + added->len = key.len; + hashmap_add(&w->dedup, &added->ent); + return key.offset; +} + +int diff_hunks_writer_add(struct diff_hunks_writer *w, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + const struct precomputed_hunk *hunks, + size_t nr_hunks) +{ + struct writer_entry *e; + size_t i, block_start; + + if (!w) + return 0; + /* + * The block appended for this entry is sizeof(uint32_t) + + * nr_hunks * DIFF_HUNKS_HUNK_SIZE bytes. Bound nr_hunks so that + * length fits the uint32_t the dedup index records (and so the + * count itself fits the uint32_t written to the store). + */ + if (!nr_hunks || + nr_hunks > (UINT32_MAX - sizeof(uint32_t)) / DIFF_HUNKS_HUNK_SIZE || + is_null_oid(old_oid) || is_null_oid(new_oid)) + return 0; + if (w->hdat.len > UINT32_MAX) + return 0; + /* + * Coordinates are stored as 32-bit values; a result that cannot + * round-trip is dropped rather than silently truncated. + */ + for (i = 0; i < nr_hunks; i++) + if ((uintmax_t)hunks[i].old_start > (uintmax_t)INT32_MAX || + (uintmax_t)hunks[i].old_count > (uintmax_t)INT32_MAX || + (uintmax_t)hunks[i].new_start > (uintmax_t)INT32_MAX || + (uintmax_t)hunks[i].new_count > (uintmax_t)INT32_MAX) + return 0; + + ALLOC_GROW(w->entries, w->nr + 1, w->alloc); + e = &w->entries[w->nr++]; + oidcpy(&e->old_oid, old_oid); + oidcpy(&e->new_oid, new_oid); + e->xdl_opts = xdl_opts; + + block_start = w->hdat.len; + strbuf_put_be32(&w->hdat, (uint32_t)nr_hunks); + for (i = 0; i < nr_hunks; i++) { + strbuf_put_be32(&w->hdat, hunks[i].old_start); + strbuf_put_be32(&w->hdat, hunks[i].old_count); + strbuf_put_be32(&w->hdat, hunks[i].new_start); + strbuf_put_be32(&w->hdat, hunks[i].new_count); + } + e->hdat_offset = intern_block(w, block_start); + return 1; +} + +void diff_hunks_writer_record_stable(struct diff_hunks_writer *w, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + const struct precomputed_hunk *trimmed, + size_t nr_trimmed, + const struct precomputed_hunk *full, + size_t nr_full) +{ + size_t i; + + if (!w) + return; + /* + * Record only a trim-stable pair, one whose trimmed and + * untrimmed diffs are identical, so the single entry answers + * any consumer at any context (see the top of this file). A + * pair where the two diffs differ is never recorded and every + * consumer computes it. + */ + if (nr_trimmed != nr_full) + return; + for (i = 0; i < nr_trimmed; i++) + if (trimmed[i].old_start != full[i].old_start || + trimmed[i].old_count != full[i].old_count || + trimmed[i].new_start != full[i].new_start || + trimmed[i].new_count != full[i].new_count) + return; + diff_hunks_writer_add(w, old_oid, new_oid, xdl_opts, + trimmed, nr_trimmed); +} + +/* + * Seed the writer with fname's entries so a rewrite preserves them, + * setting *pruned when the rewrite will not carry the whole file + * forward: the file failed its checksum and was discarded outright, or + * individual entries were dropped because they failed the replayable + * check or the writer refused them (a key naming no blob). A + * rewrite re-checksums, so corruption must not be carried forward: + * that would launder it into a checksum-valid file that verify can no + * longer catch. This path already reads the whole file, so verify the + * checksum here (the reader keeps trusting committed files, without + * re-checksumming); an invalid + * entry reads as a miss anyway, so dropping it heals the store rather + * than losing anything a reader could use. + */ +static void diff_hunks_writer_seed(struct diff_hunks_writer *w, + const char *fname, int *pruned) +{ + struct diff_hunks_store *s = load_store_at(w->r->hash_algo, fname); + unsigned int rawsz; + size_t entry_size, keysz; + struct precomputed_hunk *hunks = NULL; + size_t hunks_alloc = 0; + uint32_t i, dropped = 0; + + if (!s) + return; + if (!hashfile_checksum_valid(w->r->hash_algo, s->data, s->data_len)) { + warning(_("diff-hunks store %s failed its checksum; " + "discarding it"), fname); + free_store(s); + *pruned = 1; + return; + } + rawsz = s->hash_algo->rawsz; + entry_size = store_index_entry_size(s->hash_algo); + keysz = store_index_key_size(s->hash_algo); + + for (i = 0; i < s->num_entries; i++) { + const unsigned char *ep = s->index + st_mult(entry_size, i); + const unsigned char *old_hash, *new_hash; + struct object_id old_oid, new_oid; + uint32_t xdl_opts, j; + struct precomputed_entry pe; + + decode_store_index_key(ep, rawsz, &old_hash, &new_hash, + &xdl_opts); + oidread(&old_oid, old_hash, s->hash_algo); + oidread(&new_oid, new_hash, s->hash_algo); + if (!precomputed_entry_at(s, index_entry_hdat_offset(ep, keysz), &pe) || + !replayable_hunks(&pe)) { + dropped++; + continue; + } + ALLOC_GROW(hunks, pe.num_hunks, hunks_alloc); + for (j = 0; j < pe.num_hunks; j++) + nth_precomputed_hunk(&pe, j, &hunks[j]); + if (!diff_hunks_writer_add(w, &old_oid, &new_oid, + (int)xdl_opts, hunks, pe.num_hunks)) + dropped++; + } + if (dropped) { + warning(Q_("diff-hunks store %s: dropping %u invalid entry", + "diff-hunks store %s: dropping %u invalid entries", + dropped), fname, dropped); + *pruned = 1; + } + free(hunks); + free_store(s); +} + +/* + * Writing is off by default. It is enabled per invocation by the + * GIT_DIFF_HUNKS_WRITE environment variable, or persistently by the + * diffHunks.write config, with the environment variable winning when + * set. Only a warming run (a diff or log the repository owner chooses + * to run with writing on) enables it, so ordinary reads never mutate + * the store. + */ +static int diff_hunks_write_enabled(struct repository *r) +{ + const char *env = getenv("GIT_DIFF_HUNKS_WRITE"); + int val; + + if (env) { + /* + * This is a warming opt-in, so an unparseable value must not + * abort an ordinary read command: treat it as disabled. + */ + val = git_parse_maybe_bool(env); + return val < 0 ? 0 : val; + } + if (!repo_config_get_bool(r, "diffhunks.write", &val)) + return val; + return 0; +} + +struct diff_hunks_writer *diff_hunks_writer_maybe_new(struct repository *r) +{ + struct diff_hunks_writer *w; + char *fname; + int pruned; + + if (!diff_hunks_write_enabled(r)) + return NULL; + /* + * Seed from the existing store so a flush merges with it rather + * than replacing it: a later warm adds newly computed pairs + * without discarding what earlier warms recorded. + */ + w = diff_hunks_writer_new(r); + fname = diff_hunks_store_path(r); + pruned = 0; + diff_hunks_writer_seed(w, fname, &pruned); + free(fname); + w->seed_nr = w->nr; + /* + * A pruning seed means the file on disk holds material the + * rewrite must not preserve; flush even if this warm computes + * nothing new, so the store on disk is repaired rather than + * left serving what the seed refused. + */ + w->force_flush = !!pruned; + return w; +} + +static int writer_entry_cmp(const void *va, const void *vb, void *ctx) +{ + const struct writer_entry *a = va, *b = vb; + unsigned int rawsz = *(const unsigned int *)ctx; + return cmp_store_index_key(a->old_oid.hash, a->new_oid.hash, + (uint32_t)a->xdl_opts, + b->old_oid.hash, b->new_oid.hash, + (uint32_t)b->xdl_opts, + rawsz); +} + +struct write_ctx { + struct diff_hunks_writer *w; + unsigned int rawsz; +}; + +static int write_index_chunk(struct hashfile *f, void *data) +{ + struct write_ctx *ctx = data; + size_t i; + + for (i = 0; i < ctx->w->nr; i++) { + hashwrite(f, ctx->w->entries[i].old_oid.hash, ctx->rawsz); + hashwrite(f, ctx->w->entries[i].new_oid.hash, ctx->rawsz); + hashwrite_be32(f, ctx->w->entries[i].xdl_opts); + hashwrite_be32(f, ctx->w->entries[i].hdat_offset); + } + return 0; +} + +static int write_data_chunk(struct hashfile *f, void *data) +{ + struct write_ctx *ctx = data; + hashwrite(f, ctx->w->hdat.buf, ctx->w->hdat.len); + return 0; +} + +/* Sort, dedup, and write the accumulated entries to the file at fname. */ +static int diff_hunks_writer_flush(struct diff_hunks_writer *w, char *fname) +{ + struct lock_file lk = LOCK_INIT; + struct hashfile *f; + struct chunkfile *cf; + unsigned int rawsz = w->r->hash_algo->rawsz; + struct write_ctx ctx = { w, rawsz }; + size_t entry_size; + + QSORT_S(w->entries, w->nr, writer_entry_cmp, &rawsz); + + /* + * The same blob pair recurs across history (reverts, cherry- + * picks); identical keys carry identical hunks, so keep one of + * each. The index must stay duplicate-free for binary search. + */ + if (w->nr > 1) { + size_t kept = 1, i; + for (i = 1; i < w->nr; i++) + if (writer_entry_cmp(&w->entries[kept - 1], + &w->entries[i], &rawsz)) + w->entries[kept++] = w->entries[i]; + w->nr = kept; + } + + if (safe_create_leading_directories(w->r, fname)) { + error(_("unable to create directory for %s"), fname); + return -1; + } + if (hold_lock_file_for_update(&lk, fname, 0) < 0) { + error_errno(_("unable to lock %s"), fname); + return -1; + } + adjust_shared_perm(w->r, get_lock_file_path(&lk)); + f = hashfd(w->r->hash_algo, get_lock_file_fd(&lk), + get_lock_file_path(&lk)); + + entry_size = store_index_entry_size(w->r->hash_algo); + cf = init_chunkfile(f); + add_chunk(cf, DIFF_HUNKS_CHUNKID_INDEX, w->nr * entry_size, + write_index_chunk); + add_chunk(cf, DIFF_HUNKS_CHUNKID_DATA, w->hdat.len, write_data_chunk); + + hashwrite_be32(f, DIFF_HUNKS_SIGNATURE); + hashwrite_u8(f, DIFF_HUNKS_VERSION); + hashwrite_u8(f, oid_version(w->r->hash_algo)); + hashwrite_u8(f, get_num_chunks(cf)); + hashwrite_u8(f, 0); /* reserved */ + + write_chunkfile(cf, &ctx); + free_chunkfile(cf); + + /* + * fsync per the user's configuration (like commit-graph and the + * multi-pack-index), then commit atomically. Readers trust the + * committed file rather than re-checksumming it; diff_hunks_verify() + * checks the checksum separately. + */ + finalize_hashfile(f, NULL, FSYNC_COMPONENT_DIFF_HUNKS, + CSUM_HASH_IN_STREAM | CSUM_FSYNC); + /* + * This same process may hold the current store mmapped (a warm + * that also reads); the commit below renames over it, which must + * never land on a live mapping (Windows refuses it). Close the + * store and clear the load-attempted flag first, so the next + * read loads the committed file. + */ + if (w->r->objects) { + close_diff_hunks_store(w->r->objects); + w->r->objects->diff_hunks_store_attempted = 0; + } + if (commit_lock_file(&lk)) { + error_errno(_("unable to write %s"), fname); + return -1; + } + return 0; +} + +static void diff_hunks_writer_free(struct diff_hunks_writer *w) +{ + if (!w) + return; + hashmap_clear_and_free(&w->dedup, struct dedup_entry, ent); + free(w->entries); + strbuf_release(&w->hdat); + free(w); +} + +void diff_hunks_writer_finish(struct diff_hunks_writer *w) +{ + if (!w) + return; + /* Skip the flush when the warm recorded nothing beyond its seed. */ + if (w->nr != w->seed_nr || w->force_flush) { + char *fname = diff_hunks_store_path(w->r); + diff_hunks_writer_flush(w, fname); + free(fname); + } + diff_hunks_writer_free(w); +} diff --git a/diff-hunks.h b/diff-hunks.h new file mode 100644 index 00000000000000..c58500d05c0de1 --- /dev/null +++ b/diff-hunks.h @@ -0,0 +1,141 @@ +#ifndef DIFF_HUNKS_H +#define DIFF_HUNKS_H + +#include "hash.h" +#include "xdiff-interface.h" /* xdl_emit_hunk_consume_func_t */ + +struct object_id; +struct repository; +struct object_database; + +/* + * A persistent store of precomputed diff hunk coordinates, at + * .git/objects/info/diff-hunks. Entries are keyed by the two blobs diffed + * and the xdl_opts they were diffed under, so a cached result is valid + * in any context that key recurs in, independent of path. The xdl_opts + * key component mirrors the (always non-negative) diff_options field it + * projects from, and is serialized and compared as a 4-byte big-endian + * integer. + * + * The hunks a pair produces are not unique. They vary with the xdiff + * algorithm and ignore flags (xdl_opts, part of the key), and with + * whether the diff was trimmed: a zero-context diff runs + * trim_common_tail, which can pick a different but equally valid set of + * hunks than an untrimmed diff. The store holds one entry per key, so a + * pair is recorded only when its trimmed and untrimmed diffs are + * identical (the recording caller checks); such an entry serves a + * consumer at any context. The rare pair where the two diffs differ is + * never recorded and is always computed. + * + * The store is a cache: ordinary commands read it and fall back to + * computing the diff when it is absent, stale, or corrupt. It is filled + * as a side effect of diff and log runs, but only when writing is + * enabled (such a write-enabled run is a warming run); writing is off + * by default, so an ordinary command reads the store without recording + * into it. + */ + +/* + * A hunk's coordinates. The type is long to match the xdiff emit + * callback; the values are a diff's line numbers and counts, always + * within the int32 range the on-disk format stores (see + * diff_hunks_writer_add()). + */ +struct precomputed_hunk { + long old_start; + long old_count; + long new_start; + long new_count; +}; + +/* + * The repository's store, loaded once on first use and cached on the + * object database. Returns NULL when reading is disabled + * (core.diffHunks=false), the store is absent, or it fails to parse + * (wrong signature, version, or object hash, or a corrupt structure). + * The lookup functions below accept a NULL store and treat it as + * empty (every lookup misses), so callers need not check for NULL. + * The object database owns the store; callers must not free it. + */ +struct diff_hunks_store *repo_diff_hunks_store(struct repository *r); + +/* Free the repository's cached store, at object-database teardown. */ +void close_diff_hunks_store(struct object_database *o); + +/* + * Consultation counters for the repository's store: pairs the store + * served (hits) and pairs it was consulted for but could not serve + * (misses). Both zero when reading is disabled or no store exists. + */ +void diff_hunks_read_stats(struct repository *r, + unsigned long *hits, unsigned long *misses); + +/* + * Replay the recorded hunks of an (old blob, new blob) pair diffed + * under xdl_opts through hunk_func. The sequence is validated before + * any callback runs: on a hit (return 1) every hunk is emitted, on a + * miss (return 0: absent pair, xdl_opts mismatch, or an entry that + * fails validation) nothing is emitted, so a caller may accumulate + * directly into its result. + */ +int diff_hunks_replay(struct diff_hunks_store *s, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + xdl_emit_hunk_consume_func_t hunk_func, void *cb_data); + +/* + * A warming run's writer: it accumulates the hunks it computes in memory + * and flushes them to the store in one pass at finish. + */ +struct diff_hunks_writer; + +/* + * Return a writer for a warming run, or NULL when writing is disabled + * (the default). diff_hunks_writer_add() tolerates a NULL writer, so a + * caller may attach the result unconditionally. Pair with + * diff_hunks_writer_finish(). + */ +struct diff_hunks_writer *diff_hunks_writer_maybe_new(struct repository *r); + +/* + * Record a blob pair's hunks as computed under xdl_opts; a later lookup + * with a matching key is served these hunks. The caller must have + * checked that the pair's trimmed and untrimmed diffs are identical + * (see the top of this file), so the entry answers at any context; + * diff_hunks_writer_record_stable() below performs that check. + * NULL-safe. Returns 1 when the entry was recorded, 0 when the writer + * refused it (no hunks, a null object id, or values the on-disk + * 32-bit fields cannot hold). + */ +int diff_hunks_writer_add(struct diff_hunks_writer *w, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + const struct precomputed_hunk *hunks, + size_t nr_hunks); + +/* + * Record the pair only if it is trim-stable: the recording caller + * hands over both the trimmed (xdi_diff) and untrimmed (xdl_diff) + * zero-context hunk sequences it computed, and the entry is added + * only when the two are identical. NULL-safe. + */ +void diff_hunks_writer_record_stable(struct diff_hunks_writer *w, + const struct object_id *old_oid, + const struct object_id *new_oid, + int xdl_opts, + const struct precomputed_hunk *trimmed, + size_t nr_trimmed, + const struct precomputed_hunk *full, + size_t nr_full); + +/* Flush the accumulated entries to the store and free the writer. NULL-safe. */ +void diff_hunks_writer_finish(struct diff_hunks_writer *w); + +/* Remove the store file. Returns 0 (incl. absent) or -1. */ +int diff_hunks_clear(struct repository *r); +/* Validate the store. Returns 0 if valid/absent, -1 if corrupt. */ +int diff_hunks_verify(struct repository *r); + +#endif /* DIFF_HUNKS_H */ diff --git a/diff-process.c b/diff-process.c new file mode 100644 index 00000000000000..d28f9cf973972a --- /dev/null +++ b/diff-process.c @@ -0,0 +1,669 @@ +/* + * The process provider of the hunk provider interface: consult a + * long-running external process via the pkt-line protocol for the + * hunks of a blob pair. The process answers from the pair's object + * names alone: it can serve a persistent cache keyed on the pair, or + * fetch the blobs from the repository itself (e.g. via "git cat-file + * --batch") and compute its own notion of which lines changed. The + * provider sits at the head of its repository's chain and gates + * itself per request; its state is the repository's pool of running + * processes, one per configured command, stopped when the provider + * is released. + * + * Protocol: pkt-line over stdin/stdout, following the pattern of + * the long-running filter process protocol (see convert.c). + * + * Handshake: + * git> git-diff-client / version=1 / flush + * process< git-diff-server / version=1 / flush + * git> capability=hunks-by-oid / flush + * process< capability=hunks-by-oid / flush + * + * Per-pair, when both sides are stored blobs: + * git> command=hunks-by-oid / pathname= + * git> old-oid= / new-oid= / flush + * process< hunk + * process< ... / flush + * process< status=success / flush + * + * No content is sent. Because Git holds no content for the exchange, + * the answer is used as the process sent it: the hunks are not re-run + * through xdiff's compaction, and a status=success response with zero + * hunks asserts that the blobs are equivalent, including their + * trailing newlines. A process that cannot answer from the object names + * (or cannot rule out a trailing-newline-only difference) responds + * status=need-content; the pair then gets the builtin answer, served + * from the diff-hunks store or computed. A later + * protocol extension can define a content-carrying request for such + * processes and for sides that are not stored blobs. + */ + +#include "git-compat-util.h" +#include "diff.h" +#include "diff-provider-internal.h" +#include "gettext.h" +#include "hex.h" +#include "odb.h" +#include "repository.h" +#include "sigchain.h" +#include "userdiff.h" +#include "sub-process.h" +#include "pkt-line.h" +#include "strbuf.h" + +#define CAP_OID_HUNKS (1u << 0) + +/* + * The provider's state: the repository's diff processes, keyed by + * their command string, so drivers that configure the same command + * share one process. An entry whose process failed stays in the + * pool with the failed bit set, so the command is not retried while + * the entry lives; the pool and its entries last until the provider + * is released. + */ +struct diff_process_state { + struct hashmap subprocesses; +}; + +struct diff_subprocess { + struct subprocess_entry subprocess; + /* + * Owns the string subprocess.cmd and the hashmap key borrow: the + * entry outlives the userdiff config a re-read may replace. + */ + char *cmd; + unsigned int supported_capabilities; + unsigned failed : 1; +}; + +static int start_diff_process_fn(struct subprocess_entry *subprocess) +{ + static int versions[] = { 1, 0 }; + static struct subprocess_capability capabilities[] = { + { "hunks-by-oid", CAP_OID_HUNKS }, + { NULL, 0 } + }; + struct diff_subprocess *entry = + container_of(subprocess, struct diff_subprocess, subprocess); + + return subprocess_handshake(subprocess, "git-diff", + versions, NULL, + capabilities, + &entry->supported_capabilities); +} + +/* + * The pool entry for a command, or NULL when its process fails to + * start here: the failure leaves a failed entry in the pool, so only + * the request that observed it maps it to an error and later + * requests pass the provider by. + */ +static struct diff_subprocess *get_or_launch_process( + struct diff_process_state *state, + struct userdiff_driver *drv) +{ + struct subprocess_entry *running; + struct diff_subprocess *entry; + + running = subprocess_find_entry(&state->subprocesses, drv->process); + if (running) { + entry = container_of(running, struct diff_subprocess, + subprocess); + return entry->failed ? NULL : entry; + } + + entry = xcalloc(1, sizeof(*entry)); + entry->cmd = xstrdup(drv->process); + if (subprocess_start_command(&entry->subprocess, entry->cmd, + start_diff_process_fn)) + entry->failed = 1; + hashmap_entry_init(&entry->subprocess.ent, strhash(entry->cmd)); + hashmap_add(&state->subprocesses, &entry->subprocess.ent); + if (entry->failed) { + warning(_("diff process '%s' failed to start;" + " using the builtin diff"), drv->process); + return NULL; + } + return entry; +} + +/* + * A hunk in the diff process's presentation coordinates: the line + * numbering it reports over the protocol. Kept distinct from struct + * xdl_hunk (xdiff's coordinates) so that only translated hunks ever + * reach a consumer; diff_process_hunk_to_xdl() is the single + * crossing point. + */ +struct diff_process_hunk { + long old_start, old_count; + long new_start, new_count; +}; + +/* + * Parse one non-negative decimal field of a hunk line into *out and + * advance *line past it. Fields must be plain decimal with no leading + * whitespace or sign (isdigit() takes an unsigned char to stay defined + * for high-bit bytes). The first three fields are followed by a single + * space; the last (is_last) is followed by end-of-string or a space. + * Trailing space-separated tokens after the last field are allowed and + * ignored, so a future protocol version can append fields (e.g. a + * "moved" marker) without an older Git rejecting the line, mirroring + * the request-side rule that processes ignore unknown keys. + * + * A value that overflows strtol() is not a parse failure: the line is + * well-formed, so the stream stays in protocol sync. It is reported + * through *out_of_range, and the caller skips the pair the same way + * it skips any other out-of-range coordinate. + */ +static int parse_hunk_field(const char **line, long *out, int is_last, + int *out_of_range) +{ + const char *p = *line; + char *end; + + if (!isdigit((unsigned char)*p)) + return -1; + errno = 0; + *out = strtol(p, &end, 10); + if (end == p) + return -1; + if (errno == ERANGE) + *out_of_range = 1; + else if (errno) + return -1; + if (is_last) { + if (*end != '\0' && *end != ' ') + return -1; + } else { + if (*end != ' ') + return -1; + end++; + } + *line = end; + return 0; +} + +static int parse_hunk_line(const char *line, + struct diff_process_hunk *presented, + int *out_of_range) +{ + *out_of_range = 0; + /* Format: "hunk " */ + if (!skip_prefix(line, "hunk ", &line)) + return -1; + if (parse_hunk_field(&line, &presented->old_start, 0, out_of_range) || + parse_hunk_field(&line, &presented->old_count, 0, out_of_range) || + parse_hunk_field(&line, &presented->new_start, 0, out_of_range) || + parse_hunk_field(&line, &presented->new_count, 1, out_of_range)) + return -1; + return 0; +} + +/* + * Translate a hunk from the diff process's presentation coordinates + * into xdiff's. + * + * Protocol starts are already 1-based positions (the line a change + * sits before), the same numbering xdiff uses, so the only adjustment + * is for an empty file side: "git diff" addresses it with a start of 0 + * and a count of 0 (e.g. "0 0 1 5" adds five lines to an empty old + * side), and since xdiff uses start-1 as an array index that 0 becomes + * 1 here. This is NOT the full inverse of xdl_emit_hunk_hdr() + * (xdiff/xutils.c): that emitter shifts a count-0 range to start-1 for + * the displayed "@@" header, but the protocol keeps the unshifted + * 1-based position for a mid-file insert or delete. This is the single + * point where presentation coordinates become xdiff coordinates, so + * any consumer of these coordinates may assume 1-based starts. + * + * Returns -1 for a start of 0 paired with a nonzero count, which names + * no line in either coordinate system. (parse_hunk_line() already + * guarantees non-negative starts and counts.) + */ +static int diff_process_hunk_to_xdl(const struct diff_process_hunk *presented, + struct xdl_hunk *xdl) +{ + long old_start = presented->old_start; + long new_start = presented->new_start; + + if ((!old_start && presented->old_count) || + (!new_start && presented->new_count)) + return -1; + if (!old_start) + old_start = 1; + if (!new_start) + new_start = 1; + + xdl->old_start = old_start; + xdl->old_count = presented->old_count; + xdl->new_start = new_start; + xdl->new_count = presented->new_count; + return 0; +} + +/* + * Validate the process's hunks (already in xdiff coordinates) before they + * bypass the diff algorithm. The content-independent rules (in-order, + * non-overlapping, lockstep-aligned, int32-bounded coordinates) are the + * provider interface's shared rule, diff_provider_check_hunk(); this + * function adds the two checks that need the blobs' line counts (a hunk + * past the end of a file, the run after the last hunk) and the + * per-rule diagnostics naming the process. On a bad response we warn + * and the caller falls back to the builtin diff. Returns 0 if valid, + * -1 (after warning) otherwise. + * + * old_lines/new_lines bound the line count of each side, or are + * negative when no bound is known. An oid-only answer arrives without + * content, so its caller passes upper bounds derived from the blobs' + * byte sizes, which caps coordinate magnitude but cannot support the + * run-after-the-last-hunk check: that one compares exact line counts, + * so it runs only when lines_exact is set, which no caller does today. + * It is kept for a content-carrying request, whose loaded buffers + * would provide exact counts. + */ +static int validate_external_hunks(const struct xdl_hunk *hunks, size_t nr, + long old_lines, long new_lines, + int lines_exact, + const char *process, const char *path) +{ + struct diff_provider_hunks_check c = { 0 }; + size_t i; + + for (i = 0; i < nr; i++) { + const struct xdl_hunk *h = &hunks[i]; + + if (old_lines >= 0 && + (h->old_count > old_lines - h->old_start + 1 || + h->new_count > new_lines - h->new_start + 1)) { + warning(_("diff process '%s' returned a hunk past the " + "end of '%s'; using the builtin diff"), + process, path); + return -1; + } + switch (diff_provider_check_hunk(&c, h->old_start, + h->old_count, h->new_start, + h->new_count)) { + case DIFF_PROVIDER_HUNKS_OK: + break; + case DIFF_PROVIDER_HUNKS_RANGE: + warning(_("diff process '%s' returned out-of-range " + "coordinates for '%s'; using the builtin diff"), + process, path); + return -1; + case DIFF_PROVIDER_HUNKS_OVERLAP: + warning(_("diff process '%s' returned overlapping hunks " + "for '%s'; using the builtin diff"), + process, path); + return -1; + case DIFF_PROVIDER_HUNKS_MISALIGNED: + warning(_("diff process '%s' returned hunks that leave " + "'%s' misaligned; using the builtin diff"), + process, path); + return -1; + } + } + if (lines_exact && + old_lines - c.prev_old_end != new_lines - c.prev_new_end) { + warning(_("diff process '%s' returned hunks that leave '%s' " + "misaligned; using the builtin diff"), + process, path); + return -1; + } + return 0; +} + +/* + * The most lines a blob can hold, from its size alone: every line, + * even an empty one, costs at least one byte, so a blob of N bytes + * holds at most N lines. Returns -1 when the size is unavailable, + * leaving the response bounded only by the shared int32 rule. A size + * beyond INT32_MAX clamps to it, which loses nothing: a coordinate + * that large fails the shared rule anyway. In a partial clone the + * size lookup must not fetch the blob from the promisor remote: + * validating an answer that exists to avoid loading content must not + * itself download that content, so a missing blob reads as size + * unavailable instead. + */ +static long blob_line_cap(struct repository *r, const struct object_id *oid) +{ + size_t size; + struct object_info oi = OBJECT_INFO_INIT; + + oi.sizep = &size; + if (odb_read_object_info_extended(r->objects, oid, &oi, + OBJECT_INFO_SKIP_FETCH_OBJECT) < 0) + return -1; + if (size > INT32_MAX) + return INT32_MAX; + return (long)size; +} + +/* + * The driver whose process a consultation for path would ask, or NULL + * when none applies (no driver, process not allowed, or xpp carries + * options the process is never told about). Needs no content, so + * the driver is picked before any blob is loaded. + */ +static struct userdiff_driver *diff_process_driver(struct diff_options *diffopt, + const char *path, + const xpparam_t *xpp) +{ + struct userdiff_driver *drv; + + if (!diffopt || !path) + return NULL; + if (!diffopt->flags.allow_diff_process || diffopt->ignore_driver_algorithm) + return NULL; + /* + * Whitespace-ignoring, regex-ignore (-I) and anchored options + * change which lines count as different, but the process is never + * told about them, so its hunks could not honor them. A forced + * diff algorithm (an option or configured algorithm setting) + * requests a specific builtin computation, which an + * authoritative answer would override. Rather than silently + * override the user's request, fall back to the builtin diff, + * which does honor these flags. Key this off xpp (the + * parameters this diff actually runs with) rather than diffopt, + * so a caller like blame, which keeps its algorithm and + * whitespace flags outside diffopt, is covered without a + * separate guard of its own. + */ + if ((xpp->flags & (XDF_WHITESPACE_FLAGS | XDF_IGNORE_BLANK_LINES | + XDF_DIFF_ALGORITHM_MASK)) || + xpp->ignore_regex_nr || xpp->anchors_nr) + return NULL; + + /* + * A path the protocol cannot carry never selects a process: an + * embedded newline would let the rest of the path forge further + * request keys, and the pathname must fit one packet. Passing + * here keeps the cost local to the path; a failed write would + * instead cost the whole command its process. + */ + if (strchr(path, '\n') || + strlen(path) > LARGE_PACKET_DATA_MAX - strlen("pathname=\n")) + return NULL; + + drv = userdiff_find_by_path(diffopt->repo->index, path); + if (!drv || !drv->process) + return NULL; + return drv; +} + +/* + * Without content there is no size-derived bound on a response, so cap + * accumulation at a constant instead. A response that exceeds the + * cap is a protocol error: the process is disabled for the rest of + * the command and the caller falls back to the builtin diff. + */ +#define OID_HUNKS_MAX (1 << 20) + +enum diff_process_result { + DIFF_PROCESS_ERROR = -1, /* failed; caller falls back to builtin */ + DIFF_PROCESS_OK = 0, /* the process supplied hunks */ + DIFF_PROCESS_SKIP, /* process did not apply: use builtin */ + DIFF_PROCESS_EQUIVALENT, /* process says files are equivalent */ +}; + +/* + * Ask drv's diff process to answer the request from the blob pair's + * object ids alone (the "hunks-by-oid" capability): no content is + * loaded or sent. On DIFF_PROCESS_OK the process's hunks are emitted + * through hunk_cb in 0-based emission coordinates, validated for order, + * overlap, and lockstep alignment first; because Git holds no content, + * the answer is used as the process sent it, without xdiff's compaction. + * DIFF_PROCESS_EQUIVALENT means the process asserts the pair equal. + * DIFF_PROCESS_SKIP covers everything that should fall through to the + * builtin computation: a missing capability, a missing object id, a + * status=need-content answer, or an invalid response. + */ +static enum diff_process_result diff_process_query_hunks( + struct diff_process_state *state, + struct userdiff_driver *drv, + const struct diff_provider_request *req, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data) +{ + const char *path = req->path; + struct diff_subprocess *entry; + struct child_process *process; + int fd_in, fd_out; + struct packet_reader reader; + struct strbuf status = STRBUF_INIT; + struct xdl_hunk *hunks = NULL; + struct diff_process_hunk presented; + struct xdl_hunk hunk; + size_t nr_hunks = 0, alloc_hunks = 0, i; + int bad_coords = 0; + long old_cap, new_cap; + enum diff_process_result res; + + if (!req->old_oid || !req->new_oid) + return DIFF_PROCESS_SKIP; + + entry = get_or_launch_process(state, drv); + if (!entry) + return DIFF_PROCESS_ERROR; + if (!(entry->supported_capabilities & CAP_OID_HUNKS)) + return DIFF_PROCESS_SKIP; + + process = subprocess_get_child_process(&entry->subprocess); + fd_in = process->in; + fd_out = process->out; + + sigchain_push(SIGPIPE, SIG_IGN); + + if (packet_write_fmt_gently(fd_in, "command=hunks-by-oid\n") || + packet_write_fmt_gently(fd_in, "pathname=%s\n", path) || + packet_write_fmt_gently(fd_in, "old-oid=%s\n", + oid_to_hex(req->old_oid)) || + packet_write_fmt_gently(fd_in, "new-oid=%s\n", + oid_to_hex(req->new_oid)) || + packet_flush_gently(fd_in)) + goto comm_error; + + packet_reader_init(&reader, fd_out, NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF | + PACKET_READ_GENTLE_ON_READ_ERROR); + for (;;) { + enum packet_read_status rs = packet_reader_read(&reader); + int out_of_range; + + if (rs == PACKET_READ_FLUSH) + break; + /* + * Only a hunk line may precede the flush. EOF and a + * malformed frame end the session; an empty packet, which + * a length-only read cannot tell from a flush, would + * truncate the hunk section here and leave the status + * section to poison the next request, so it is a protocol + * error too. + */ + if (rs != PACKET_READ_NORMAL || !reader.pktlen) + goto comm_error; + if (parse_hunk_line(reader.line, &presented, + &out_of_range) < 0) + goto comm_error; + if (bad_coords) + continue; + if (out_of_range || + diff_process_hunk_to_xdl(&presented, &hunk) < 0) { + /* + * Semantically invalid coordinates in a well-formed + * response: the stream stays in protocol sync, so + * drain the rest and fall back for this file while + * keeping the process alive, the same treatment + * validate_external_hunks() failures receive. + */ + bad_coords = 1; + continue; + } + if (nr_hunks >= OID_HUNKS_MAX) { + warning(_("diff process '%s' sent too many hunks" + " for '%s'; disabling it for the" + " remainder of this command"), + drv->process, path); + goto disable; + } + ALLOC_GROW(hunks, nr_hunks + 1, alloc_hunks); + hunks[nr_hunks++] = hunk; + } + + if (subprocess_read_status_gently(fd_out, &status)) + goto comm_error; + + if (!strcmp(status.buf, "success")) { + if (bad_coords) { + warning(_("diff process '%s' returned out-of-range " + "coordinates for '%s'; using the builtin diff"), + drv->process, path); + res = DIFF_PROCESS_SKIP; + goto out; + } + if (!nr_hunks) { + res = DIFF_PROCESS_EQUIVALENT; + goto out; + } + /* + * Bound the coordinates by the blobs' sizes, read from the + * object database without loading content. Either both + * bounds hold or neither is applied: a partial bound would + * misclassify a response that the other side's size would + * have caught. + */ + old_cap = blob_line_cap(req->repo, req->old_oid); + new_cap = blob_line_cap(req->repo, req->new_oid); + if (old_cap < 0 || new_cap < 0) + old_cap = new_cap = -1; + if (validate_external_hunks(hunks, nr_hunks, old_cap, new_cap, + 0, drv->process, path) < 0) { + res = DIFF_PROCESS_SKIP; + goto out; + } + /* + * Replay in the coordinates a hunk consumer receives from + * xdiff's emission: 0-based starts. The answer is used as + * the process sent it; with no content in hand it cannot be + * re-run through xdiff's compaction. + */ + for (i = 0; i < nr_hunks; i++) + hunk_cb(hunks[i].old_start - 1, hunks[i].old_count, + hunks[i].new_start - 1, hunks[i].new_count, + cb_data); + res = DIFF_PROCESS_OK; + goto out; + } + if (!strcmp(status.buf, "need-content")) { + /* + * The process cannot answer this pair from its object names; + * the caller computes the diff itself. + */ + res = DIFF_PROCESS_SKIP; + goto out; + } + if (!strcmp(status.buf, "abort")) { + /* The process withdrew: stop asking it for this session. */ + entry->supported_capabilities &= ~CAP_OID_HUNKS; + res = DIFF_PROCESS_SKIP; + goto out; + } + /* + * An unrecognized status is a protocol error, not a per-pair + * failure: this Git did not request anything it does not know, + * so the process is answering some other protocol, and asking + * it again would warn on every pair of the traversal. + */ + warning(_("diff process '%s' sent unrecognized status '%s' for " + "'%s'; disabling it for the remainder of this command"), + drv->process, status.buf, path); + goto disable; +out: + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return res; + +comm_error: + warning(_("diff process '%s' failed for '%s'; disabling it" + " for the remainder of this command"), + drv->process, path); +disable: + subprocess_stop_command(&entry->subprocess); + entry->failed = 1; + free(hunks); + strbuf_release(&status); + sigchain_pop(SIGPIPE); + return DIFF_PROCESS_ERROR; +} + +/* + * The process outranks every later provider through its chain + * position: when it answers, the walk ends, so no later provider + * serves the pair, and an answered pair is never recorded. When it + * does not answer (it defers with need-content, lacks the + * capability, or failed), the caller computes the builtin diff for + * that pair. The store holds builtin results and nothing else, so + * an identity answer for such a pair equals what the caller would + * compute. Every non-answer is therefore a pass: a refusal would + * suppress that equal answer, and would keep a warming run from + * recording the builtin result the caller computes anyway. + */ +static enum diff_provider_disposition +diff_process_consult(struct diff_provider *provider, + const struct diff_provider_request *req, + diff_provider_fill_fn fill UNUSED, void *fill_data UNUSED, + xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data) +{ + struct diff_process_state *state = provider->state; + struct userdiff_driver *drv; + struct subprocess_entry *running; + + drv = diff_process_driver(req->diffopt, req->path, req->xpp); + if (!drv) + return DIFF_PROVIDER_DISP_PASS; + running = subprocess_find_entry(&state->subprocesses, drv->process); + if (running && container_of(running, struct diff_subprocess, + subprocess)->failed) + return DIFF_PROVIDER_DISP_PASS; + + switch (diff_process_query_hunks(state, drv, req, + hunk_cb, cb_data)) { + case DIFF_PROCESS_OK: + case DIFF_PROCESS_EQUIVALENT: + return DIFF_PROVIDER_DISP_ANSWERED; + case DIFF_PROCESS_SKIP: + case DIFF_PROCESS_ERROR: + break; + } + return DIFF_PROVIDER_DISP_PASS; +} + +static void diff_process_release(struct diff_provider *provider) +{ + struct diff_process_state *state = provider->state; + struct hashmap_iter iter; + struct diff_subprocess *entry; + + /* A failed entry's process is already stopped or never ran. */ + hashmap_for_each_entry(&state->subprocesses, &iter, entry, + subprocess.ent) { + if (!entry->failed) + subprocess_stop_command(&entry->subprocess); + free(entry->cmd); + } + hashmap_clear_and_free(&state->subprocesses, + struct diff_subprocess, subprocess.ent); + free(state); +} + +struct diff_provider *diff_process_provider_new(void) +{ + struct diff_process_state *state = xcalloc(1, sizeof(*state)); + struct diff_provider *p = xcalloc(1, sizeof(*p)); + + hashmap_init(&state->subprocesses, cmd2process_cmp, NULL, 0); + p->consult = diff_process_consult; + p->release = diff_process_release; + p->state = state; + return p; +} diff --git a/diff-provider-internal.h b/diff-provider-internal.h new file mode 100644 index 00000000000000..cba1fa271adf6d --- /dev/null +++ b/diff-provider-internal.h @@ -0,0 +1,130 @@ +#ifndef DIFF_PROVIDER_INTERNAL_H +#define DIFF_PROVIDER_INTERNAL_H + +#include "diff-provider.h" + +/* + * The implementor-facing half of the hunk provider interface: the + * provider chain a repository owns, and the rules a provider applies + * to its own answer before any consumer sees it. Provider + * implementations include this header; consumers of the interface + * use only diff-provider.h. + */ + +/* + * A provider's verdict on one request. Only the chain walk + * (diff-provider.c) sees these; it maps the dispositions of a whole + * walk onto the public outcome set. + */ +enum diff_provider_disposition { + /* + * The provider failed to produce the answer it owns. Only + * the computing provider returns this: its compute leg is + * the one part of a consultation that can fail, and the walk + * ends with the public error outcome. + */ + DIFF_PROVIDER_DISP_ERROR = -1, + + /* + * Answered: every hunk of the pair has been emitted through + * the consumer's callback. + */ + DIFF_PROVIDER_DISP_ANSWERED = 0, + + /* Not this provider's request: the walk consults the next one. */ + DIFF_PROVIDER_DISP_PASS, + + /* + * The pair must not be answered from identity nor recorded: + * the request is shaped by parameters the provider's + * recording key cannot express, so a recorded answer would + * not match this request, and this request's result must not + * be recorded under that key. The walk goes on, but consults + * only the computing provider, and its fall-through outcome + * tells the consumer not to record. + */ + DIFF_PROVIDER_DISP_STOP_NO_RECORD, +}; + +/* + * One provider in a repository's chain (repository.h). The chain is + * assembled in diff-provider.c with a fixed composition; whether a + * provider applies to a request is decided by nobody but the + * provider, whose consult gates itself and passes. Chain position + * carries the authority resolution: an earlier provider's answer or + * refusal outranks every provider after it. + */ +struct diff_provider { + /* + * Consult this provider for one request. fill is NULL on a + * consult-only walk; only the computing provider reads it, + * and it must pass when fill is NULL. + */ + enum diff_provider_disposition + (*consult)(struct diff_provider *provider, + const struct diff_provider_request *req, + diff_provider_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data); + + /* + * Tear down the provider's state, or NULL when it owns none. + * Runs when the owning repository is cleared; the chain frees + * the provider itself afterwards. + */ + void (*release)(struct diff_provider *provider); + + void *state; + + /* + * Set on the provider that loads content and computes rather + * than answering from the request's identity. It alone is + * still consulted after a stop-no-record: an identity answer + * may no longer be served, but the computation must still + * run. + */ + unsigned computes:1; + + struct diff_provider *next; +}; + +/* + * The providers Git ships, besides the builtin computation that + * diff-provider.c holds itself. Each call returns a fresh provider + * for one repository's chain. + */ +struct diff_provider *diff_process_provider_new(void); +struct diff_provider *diff_hunks_store_provider_new(void); + +/* + * Incremental well-formedness check for a provider-supplied hunk + * sequence, shared by every provider. Each coordinate, and each + * hunk's end (its start plus count), must fit int32 (a consumer may + * truncate to int, and a provider may serialize as such); hunks must + * be in order and must not overlap; and the unchanged run between + * hunks must be the same length on both sides, or a consumer that + * walks the two files in lockstep desynchronizes. Every rule + * constrains differences between coordinates, so the check applies + * to 0-based and 1-based sequences alike. + * + * Feed the hunks in order to a zero-initialized struct; the first + * nonzero return names the violated rule, and the whole sequence must + * then be discarded unemitted. + */ +struct diff_provider_hunks_check { + int64_t prev_old_end, prev_new_end; +}; + +enum diff_provider_hunks_error { + DIFF_PROVIDER_HUNKS_OK = 0, + DIFF_PROVIDER_HUNKS_RANGE, /* negative or beyond int32 */ + DIFF_PROVIDER_HUNKS_OVERLAP, /* out of order or overlapping */ + DIFF_PROVIDER_HUNKS_MISALIGNED, /* unchanged runs differ in length */ +}; + +enum diff_provider_hunks_error +diff_provider_check_hunk(struct diff_provider_hunks_check *c, + long old_start, long old_count, + long new_start, long new_count); + +#endif /* DIFF_PROVIDER_INTERNAL_H */ diff --git a/diff-provider.c b/diff-provider.c new file mode 100644 index 00000000000000..c8aaf8e857a8c0 --- /dev/null +++ b/diff-provider.c @@ -0,0 +1,190 @@ +#include "git-compat-util.h" +#include "diff.h" +#include "diff-provider-internal.h" +#include "replace-object.h" +#include "repository.h" + +/* + * The terminal provider: the builtin computation. A request that + * carries a fill callback is answered by loading the pair's content + * and running xdiff, so a walk that reaches it never falls through + * to the consumer. On a consult-only walk it passes, and the walk's + * fall-through outcome tells the consumer to compute. + */ +static enum diff_provider_disposition +builtin_consult(struct diff_provider *provider UNUSED, + const struct diff_provider_request *req, + diff_provider_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data) +{ + xdemitconf_t xecfg = { .hunk_func = hunk_cb }; + xdemitcb_t ecb = { .priv = cb_data }; + mmfile_t old_file, new_file; + + if (!fill) + return DIFF_PROVIDER_DISP_PASS; + if (fill(fill_data, &old_file, &new_file) < 0) + return DIFF_PROVIDER_DISP_ERROR; + if (xdi_diff(&old_file, &new_file, req->xpp, &xecfg, &ecb) < 0) + return DIFF_PROVIDER_DISP_ERROR; + return DIFF_PROVIDER_DISP_ANSWERED; +} + +static struct diff_provider *builtin_provider_new(void) +{ + struct diff_provider *p = xcalloc(1, sizeof(*p)); + + p->consult = builtin_consult; + p->computes = 1; + return p; +} + +/* + * The repository's chain, assembled on first walk. The composition + * is fixed, and the order is the authority resolution: the process + * outranks the store, and the builtin computation is the terminal + * provider, so the chain always ends in an implementor that can + * answer. Nothing is decided per repository here; each provider + * gates itself per request. + */ +static struct diff_provider *provider_chain(struct repository *r) +{ + struct diff_provider **tail = &r->diff_providers; + + if (*tail) + return *tail; + *tail = diff_process_provider_new(); + tail = &(*tail)->next; + *tail = diff_hunks_store_provider_new(); + tail = &(*tail)->next; + *tail = builtin_provider_new(); + return r->diff_providers; +} + +void diff_providers_clear(struct repository *r) +{ + struct diff_provider *p = r->diff_providers; + + while (p) { + struct diff_provider *next = p->next; + + if (p->release) + p->release(p); + free(p); + p = next; + } + r->diff_providers = NULL; +} + +/* + * The walk shared by diff_provider_consult() and + * diff_provider_emit_hunks(): consult the chain in order and map its + * dispositions onto the outcome set. The first answer ends the + * walk. A stop-no-record disposition (diff-provider-internal.h) + * is a refusal, not a pass: the provider does not answer, but rules + * the pair out of identity service and out of recording, so from + * then on the walk consults only the computing provider, and a walk + * that ends unanswered carries the no-record verdict. With a fill + * callback the terminal provider computes instead of passing, so an + * emit walk returns only answered or error. + */ +static enum diff_provider_outcome +walk_providers(const struct diff_provider_request *req, + diff_provider_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data) +{ + struct diff_provider *p; + int no_record = 0; + + if (req->diffopt && req->diffopt->repo != req->repo) + BUG("diff provider request walks one repository's chain " + "with another repository's diff options"); + + /* + * An object replacement redirects a blob's content + * (OBJECT_INFO_LOOKUP_REPLACE) while leaving the id that names it + * unchanged, so an answer keyed on the raw id would be the + * pre-replacement diff. A replacement is therefore a parameter + * outside the recording key: no provider may serve a replaced pair + * from its identity, and a result computed for it must not be + * recorded under the raw id. Mark the walk no-record so the + * identity providers step aside and the builtin computes from the + * replaced content. The check is a no-op when the repository has + * no replace refs. + */ + if ((req->old_oid && + lookup_replace_object(req->repo, req->old_oid) != req->old_oid) || + (req->new_oid && + lookup_replace_object(req->repo, req->new_oid) != req->new_oid)) + no_record = 1; + + for (p = provider_chain(req->repo); p; p = p->next) { + enum diff_provider_disposition disp; + + if (no_record && !p->computes) + continue; + disp = p->consult(p, req, fill, fill_data, + hunk_cb, cb_data); + if (disp == DIFF_PROVIDER_DISP_ERROR && !p->computes) + BUG("only the computing provider may return the " + "error disposition"); + if (p->computes && !fill && disp != DIFF_PROVIDER_DISP_PASS) + BUG("the computing provider must pass on a " + "fill-less walk"); + switch (disp) { + case DIFF_PROVIDER_DISP_ANSWERED: + return DIFF_PROVIDER_ANSWERED; + case DIFF_PROVIDER_DISP_PASS: + continue; + case DIFF_PROVIDER_DISP_STOP_NO_RECORD: + no_record = 1; + continue; + case DIFF_PROVIDER_DISP_ERROR: + return DIFF_PROVIDER_ERROR; + } + } + return no_record ? DIFF_PROVIDER_UNANSWERED_NO_RECORD : + DIFF_PROVIDER_UNANSWERED; +} + +enum diff_provider_outcome +diff_provider_consult(const struct diff_provider_request *req, + xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data) +{ + return walk_providers(req, NULL, NULL, hunk_cb, cb_data); +} + +enum diff_provider_hunks_error +diff_provider_check_hunk(struct diff_provider_hunks_check *c, + long old_start, long old_count, + long new_start, long new_count) +{ + if (old_start < 0 || old_count < 0 || + new_start < 0 || new_count < 0 || + old_start > INT32_MAX || old_count > INT32_MAX || + new_start > INT32_MAX || new_count > INT32_MAX || + (int64_t)old_start + old_count > INT32_MAX || + (int64_t)new_start + new_count > INT32_MAX) + return DIFF_PROVIDER_HUNKS_RANGE; + if (old_start < c->prev_old_end || new_start < c->prev_new_end) + return DIFF_PROVIDER_HUNKS_OVERLAP; + if (old_start - c->prev_old_end != new_start - c->prev_new_end) + return DIFF_PROVIDER_HUNKS_MISALIGNED; + /* + * With each field bounded to int32 above, the int64 sums cannot + * overflow even where long is 32-bit, and the range rule has + * already capped them at INT32_MAX. + */ + c->prev_old_end = (int64_t)old_start + old_count; + c->prev_new_end = (int64_t)new_start + new_count; + return DIFF_PROVIDER_HUNKS_OK; +} + +enum diff_provider_outcome +diff_provider_emit_hunks(const struct diff_provider_request *req, + diff_provider_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data) +{ + return walk_providers(req, fill, fill_data, hunk_cb, cb_data); +} diff --git a/diff-provider.h b/diff-provider.h new file mode 100644 index 00000000000000..061e1c2f5c2772 --- /dev/null +++ b/diff-provider.h @@ -0,0 +1,159 @@ +#ifndef DIFF_PROVIDER_H +#define DIFF_PROVIDER_H + +#include "xdiff-interface.h" + +/* + * The hunk provider interface sits between naming a pair of file + * versions to diff and computing their changed line ranges. + * Consumers that operate on hunk coordinates route their diff + * through here, so that a provider can answer for the pair before + * its content is loaded. + * + * A hunk provider answers a consumer's request from the pair's + * identity, its blob object ids and the settings that determine the + * diff, before any content is loaded; a request no provider answers + * falls through to the consumer's own computation. Two providers implement this + * interface with different authority. The diff-hunks store + * (diff-hunks.h) is in-process and not authoritative: it may only + * reproduce the builtin result, so it never asserts a pair + * equivalent, and it stands aside wherever a process outranks it. A + * process configured in diff..process (diff-process.c) is + * authoritative for its paths: its answer may deliberately differ + * from the builtin diff, including asserting a pair equivalent. The + * interface resolves that authority through a provider chain owned + * by the repository, built on first consultation and released by + * repo_clear(): chain order is the resolution, and the builtin + * computation itself is the chain's terminal provider. A consumer + * never names a provider; it reads the outcome below. Every answer a + * provider serves from identity passes the shared coordinate check + * (diff-provider-internal.h) before any consumer sees it. + */ + +struct diff_options; +struct object_id; +struct repository; + +/* + * The result of a consultation: two dependent axes flattened into + * their four valid points. The first axis is the state of the + * response: the pair was answered, no provider answered, or (from + * diff_provider_emit_hunks() alone) the attempt failed. The second + * axis exists only in the unanswered state: whether what the caller + * computes for this request may be recorded, the one rule the + * interface imposes on an otherwise free caller. The rule travels + * in the outcome because the knowledge is a provider's while the + * recording is the caller's, and it shares the enum with the state, + * rather than riding a separate flag, so that no meaningless + * combination is representable and -Wswitch forces every consumer + * that switches to place the no-record arm. + * + * These values describe consultations, not providers: the set does + * not grow when a provider is added; a new provider maps onto these + * values inside the interface, so consumer code is written once. + * Each entry point returns a subrange of the set (stated at its + * declaration); a switch over this enum should list every value and + * omit "default:" so -Wswitch keeps it exhaustive, and a caller for + * whom only one value is actionable may compare against that value + * alone. + */ +enum diff_provider_outcome { + /* + * Loading or diffing the pair failed. Returned only by + * diff_provider_emit_hunks(), whose compute leg is the only + * part of a consultation that can fail. + */ + DIFF_PROVIDER_ERROR = -1, + + /* + * The request is answered: every hunk of the pair has been + * emitted through the callback. An authoritative provider + * that finds the pair equivalent answers with no hunks at + * all, so a callback that never fired is an answer, not an + * accident. + */ + DIFF_PROVIDER_ANSWERED = 0, + + /* + * No provider answered. What happens next is the caller's + * business, typically computing the diff itself; a result it + * computes for this request may be recorded. + */ + DIFF_PROVIDER_UNANSWERED, + + /* + * No provider answered, and what the caller computes for + * this request must not be recorded: either an authoritative + * provider owns the pair and declined this request, or the + * request is shaped by parameters outside the recording key, + * the key a recorded result is later served by. + */ + DIFF_PROVIDER_UNANSWERED_NO_RECORD, +}; + +/* + * A consultation request. The interface consults providers from + * these fields alone; no content is loaded before an answer. + * + * repo owns the provider chain the request walks. old_oid/new_oid + * name the blobs whose bytes are diffed; pass NULL for a side whose + * bytes are not a stored blob (a working-tree file, textconv output, + * a gitlink), so no provider answers from an id it cannot look up. + * path names the file the pair is diffed as; a provider selected by + * path applies only where it is set. diffopt carries the diff + * settings that live outside xpp; xpp carries the parameters the + * diff runs with. Each provider gates itself on the fields that + * concern it. + */ +struct diff_provider_request { + struct repository *repo; + const struct object_id *old_oid; + const struct object_id *new_oid; + const char *path; + struct diff_options *diffopt; + const xpparam_t *xpp; +}; + +/* + * Consult the providers for the request's pair without computing. + * On DIFF_PROVIDER_ANSWERED the hunks were emitted through hunk_cb + * (0-based emission coordinates, context 0) and were validated + * before the first callback ran, so a consumer may accumulate + * directly into its result. Never returns DIFF_PROVIDER_ERROR. + * The callback's return value is not consulted: emission of a + * validated answer has no error leg, so the callback must return 0. + */ +enum diff_provider_outcome +diff_provider_consult(const struct diff_provider_request *req, + xdl_emit_hunk_consume_func_t hunk_cb, void *cb_data); + +/* + * Load the pair's content. Called at most once per request, only + * when the ranges are computed rather than provided. The buffers + * borrow storage owned by the callback's owner. + */ +typedef int (*diff_provider_fill_fn)(void *data, mmfile_t *old_file, + mmfile_t *new_file); + +/* + * Consult the providers and, when no identity answer serves the + * request, load the pair's content through fill and compute its + * exact changed ranges (context 0). Emits to hunk_cb either way and + * returns DIFF_PROVIDER_ANSWERED, or DIFF_PROVIDER_ERROR when fill + * or the diff fails. The unanswered outcomes are never returned: a + * pair no provider answers is computed here instead of in the caller. + */ +enum diff_provider_outcome +diff_provider_emit_hunks(const struct diff_provider_request *req, + diff_provider_fill_fn fill, void *fill_data, + xdl_emit_hunk_consume_func_t hunk_cb, + void *cb_data); + +/* + * Release the repository's provider chain: stop any provider-owned + * processes and free the providers. Called by repo_clear(); the + * chain builds again on the next consultation. + */ +void diff_providers_clear(struct repository *r); + +#endif /* DIFF_PROVIDER_H */ diff --git a/diff.c b/diff.c index cfe515af4e1759..3b97a14f26ebfd 100644 --- a/diff.c +++ b/diff.c @@ -16,6 +16,8 @@ #include "revision.h" #include "quote.h" #include "diff.h" +#include "diff-hunks.h" +#include "diff-provider.h" #include "diffcore.h" #include "delta.h" #include "hex.h" @@ -34,6 +36,7 @@ #include "tmp-objdir.h" #include "graph.h" #include "oid-array.h" +#include "trace2.h" #include "packfile.h" #include "pager.h" #include "parse-options.h" @@ -610,28 +613,74 @@ struct emit_callback { }; /* - * State for the line-range callback wrappers that sit between - * xdi_diff_outf() and fn_out_consume(). xdiff produces a normal, - * unfiltered diff; the wrappers intercept each hunk header and line, - * track post-image position, and forward only lines that fall within - * the requested ranges. Contiguous in-range lines are collected into - * range hunks and flushed with a synthetic @@ header so that - * fn_out_consume() sees well-formed unified-diff fragments. + * Line-range filter: scopes "git log -L" output to the tracked ranges. * - * Removal lines ('-') cannot be classified by post-image position, so - * they are buffered in pending_rm until the next '+' or ' ' line - * reveals whether they precede an in-range line (flush into range hunk) or - * an out-of-range line (discard). + * It sits between xdi_diff_outf() and an output callback (fn_out_consume, + * diffstat_consume, checkdiff_consume). xdiff produces a normal diff; the + * filter forwards only the lines inside the requested ranges, collecting + * contiguous in-range lines into a "range hunk" emitted with a synthetic + * @@ header so the callback sees well-formed unified-diff fragments. + * + * A diff describes the change from a pre-image to a post-image. Each + * line is context (' ', in both), a removal ('-', pre-image only), or + * an addition ('+', post-image only). -L tracks ranges in the + * post-image, so a line is in range by its post-image position. + * + * Two 1-based cursors track the next line in each image, named as in + * struct emit_callback and seeded from the xdiff hunk header: + * + * lno_in_postimage advances on '+' and ' ' (lines in the post-image) + * lno_in_preimage advances on '-' and ' ' (lines in the pre-image) + * + * Ranges are 0-based half-open [start, end), so a line is tested at the + * 0-based index idx_in_postimage = lno_in_postimage - 1. + * + * A '-' is not present in the post-image, so it has no post-image line + * number of its own. Since it does not advance lno_in_postimage, it is + * classified at the idx_in_postimage that the following '+'/' ' will + * occupy. xdiff emits a change's removals before its additions, so that + * index is already known when the '-' arrives. + * + * The synthetic "@@ - + @@" header has two sides, old (the + * pre-image) and new (the post-image), matching the xdiff_emit_hunk_fn + * callback; the hunk.old_begin / hunk.new_begin fields below hold those + * begins, and flush_range_hunk() derives the counts from the buffered + * lines. + * + * Example, tracking post-image line 2 (range [1, 2)) of: + * + * pre-image post-image + * 1 a 1 a + * 2 b 2 X (b -> X) + * 3 c 3 c + * + * classify each line by idx_in_postimage. The pre and post columns + * are each cursor's value while that line is classified, i.e. before + * the line advances them (pre = lno_in_preimage, + * post = lno_in_postimage, idx = idx_in_postimage): + * ' a' pre 1 post 1 idx 0 -> before start, skip + * '-b' pre 2 post 2 idx 1 -> keep (removal) + * '+X' pre 3 post 2 idx 1 -> keep (addition) + * ' c' pre 3 post 3 idx 2 -> past end, flush + * + * -b and +X share idx = 1 because -b did not advance lno_in_postimage; + * both land in the range hunk, flushed when ' c' crosses the range end. */ -struct line_range_callback { +struct line_range_filter { xdiff_emit_line_fn orig_line_fn; + /* + * Optional; consumers that report file line numbers (e.g. + * checkdiff) need the synthetic hunk header to set their + * post-image position before in-range lines are replayed. + */ + xdiff_emit_hunk_fn orig_hunk_fn; void *orig_cb_data; const struct range_set *ranges; /* 0-based [start, end) */ unsigned int cur_range; /* index into the range_set */ /* Post/pre-image line counters (1-based, set from hunk headers) */ - long lno_post; - long lno_pre; + long lno_in_postimage; + long lno_in_preimage; /* * Function name from most recent xdiff hunk header; @@ -640,17 +689,17 @@ struct line_range_callback { char func[80]; long funclen; - /* Range hunk being accumulated for the current range */ - struct strbuf rhunk; - long rhunk_old_begin, rhunk_old_count; - long rhunk_new_begin, rhunk_new_count; - int rhunk_active; - int rhunk_has_changes; /* any '+' or '-' lines? */ - - /* Removal lines not yet known to be in-range */ - struct strbuf pending_rm; - int pending_rm_count; - long pending_rm_pre_begin; /* pre-image line of first pending */ + /* + * The range hunk being accumulated. At most one is live at a time: + * it is flushed and reset as the cursor leaves each range (and once + * more at end of diff), then reused for the next range. + */ + struct { + struct strbuf lines; /* buffered in-range diff lines */ + long old_begin; + long new_begin; + int active; + } hunk; int ret; /* latched error from orig_line_fn */ }; @@ -2540,26 +2589,60 @@ static int quick_consume(void *priv, char *line UNUSED, unsigned long len UNUSED return 1; } -static void discard_pending_rm(struct line_range_callback *s) +static void line_range_filter_init(struct line_range_filter *filter, + const struct range_set *ranges, + xdiff_emit_line_fn line_fn, + void *cb_data) { - strbuf_reset(&s->pending_rm); - s->pending_rm_count = 0; + memset(filter, 0, sizeof(*filter)); + filter->orig_line_fn = line_fn; + filter->orig_cb_data = cb_data; + filter->ranges = ranges; + strbuf_init(&filter->hunk.lines, 0); } -static void flush_rhunk(struct line_range_callback *s) +/* + * Begin a range hunk at the first in-range line. Its position fixes the + * hunk's begins, taken from the two image cursors before they advance: + * new_begin from the post-image, old_begin from the pre-image. The line + * counts are not tracked here; flush_range_hunk() derives them from the + * buffered lines. + */ +static void begin_range_hunk(struct line_range_filter *filter) +{ + filter->hunk.active = 1; + filter->hunk.new_begin = filter->lno_in_postimage; + filter->hunk.old_begin = filter->lno_in_preimage; + strbuf_reset(&filter->hunk.lines); +} + +static void flush_range_hunk(struct line_range_filter *filter) { struct strbuf hdr = STRBUF_INIT; const char *p, *end; + long old_count = 0, new_count = 0; + int has_changes = 0; - if (!s->rhunk_active || s->ret) + if (!filter->hunk.active || filter->ret) return; - /* Drain any pending removal lines into the range hunk */ - if (s->pending_rm_count) { - strbuf_addbuf(&s->rhunk, &s->pending_rm); - s->rhunk_old_count += s->pending_rm_count; - s->rhunk_has_changes = 1; - discard_pending_rm(s); + /* + * Derive the hunk's geometry from the buffered lines: a ' ' + * counts on both sides, a '-' on the old side, a '+' on the new. + * A '-' or '+' marks a real change; the "\ No newline at end of + * file" marker (line[0] == '\\') counts on neither side. + */ + p = filter->hunk.lines.buf; + end = p + filter->hunk.lines.len; + while (p < end) { + const char *eol = memchr(p, '\n', end - p); + if (*p == ' ' || *p == '-') + old_count++; + if (*p == ' ' || *p == '+') + new_count++; + if (*p == '-' || *p == '+') + has_changes = 1; + p = eol ? eol + 1 : end; } /* @@ -2568,22 +2651,28 @@ static void flush_rhunk(struct line_range_callback *s) * ctxlen causes xdiff to emit context covering a range that * has no changes in this commit. */ - if (!s->rhunk_has_changes) { - s->rhunk_active = 0; - strbuf_reset(&s->rhunk); + if (!has_changes) { + filter->hunk.active = 0; + strbuf_reset(&filter->hunk.lines); return; } - strbuf_addf(&hdr, "@@ -%ld,%ld +%ld,%ld @@", - s->rhunk_old_begin, s->rhunk_old_count, - s->rhunk_new_begin, s->rhunk_new_count); - if (s->funclen > 0) { - strbuf_addch(&hdr, ' '); - strbuf_add(&hdr, s->func, s->funclen); - } - strbuf_addch(&hdr, '\n'); + xdiff_emit_hunk_header(&hdr, filter->hunk.old_begin, old_count, + filter->hunk.new_begin, new_count, + filter->func, filter->funclen); + + /* + * Inform a line-numbering consumer of the post-image position + * before replaying lines, mirroring the hunk callback xdiff + * would have issued for a non-scoped diff. + */ + if (filter->orig_hunk_fn) + filter->orig_hunk_fn(filter->orig_cb_data, + filter->hunk.old_begin, old_count, + filter->hunk.new_begin, new_count, + filter->func, filter->funclen); - s->ret = s->orig_line_fn(s->orig_cb_data, hdr.buf, hdr.len); + filter->ret = filter->orig_line_fn(filter->orig_cb_data, hdr.buf, hdr.len); strbuf_release(&hdr); /* @@ -2591,135 +2680,159 @@ static void flush_rhunk(struct line_range_callback *s) * The cast discards const because xdiff_emit_line_fn takes * char *, though fn_out_consume does not modify the buffer. */ - p = s->rhunk.buf; - end = p + s->rhunk.len; - while (!s->ret && p < end) { + p = filter->hunk.lines.buf; + end = p + filter->hunk.lines.len; + while (!filter->ret && p < end) { const char *eol = memchr(p, '\n', end - p); unsigned long line_len = eol ? (unsigned long)(eol - p + 1) : (unsigned long)(end - p); - s->ret = s->orig_line_fn(s->orig_cb_data, (char *)p, line_len); + filter->ret = filter->orig_line_fn(filter->orig_cb_data, (char *)p, line_len); p += line_len; } - s->rhunk_active = 0; - strbuf_reset(&s->rhunk); + filter->hunk.active = 0; + strbuf_reset(&filter->hunk.lines); } static void line_range_hunk_fn(void *data, - long old_begin, long old_nr UNUSED, - long new_begin, long new_nr UNUSED, + long old_begin, long old_nr, + long new_begin, long new_nr, const char *func, long funclen) { - struct line_range_callback *s = data; + struct line_range_filter *filter = data; /* - * When count > 0, begin is 1-based. When count == 0, begin is - * adjusted down by 1 by xdl_emit_hunk_hdr(), but no lines of - * that type will arrive, so the value is unused. - * - * Any pending removal lines from the previous xdiff hunk are - * intentionally left in pending_rm: the line callback will - * flush or discard them when the next content line reveals - * whether the removals precede in-range content. + * Seed the per-image line cursors from the hunk header's begins. For + * a side with no lines (count 0), xdiff's callback has already moved + * its begin to the line before the change, so add one back to recover + * the true 1-based start. xdiff_emit_hunk_header() reapplies that -1 + * when the clipped hunk is emitted. */ - s->lno_post = new_begin; - s->lno_pre = old_begin; + filter->lno_in_postimage = new_nr ? new_begin : new_begin + 1; + filter->lno_in_preimage = old_nr ? old_begin : old_begin + 1; if (funclen > 0) { - if (funclen > (long)sizeof(s->func)) - funclen = sizeof(s->func); - memcpy(s->func, func, funclen); + if (funclen > (long)sizeof(filter->func)) + funclen = sizeof(filter->func); + memcpy(filter->func, func, funclen); } - s->funclen = funclen; + filter->funclen = funclen; } static int line_range_line_fn(void *priv, char *line, unsigned long len) { - struct line_range_callback *s = priv; - const struct range *cur; - long lno_0, cur_pre; + struct line_range_filter *filter = priv; + long idx_in_postimage; + int in_range; - if (s->ret) - return s->ret; - - if (line[0] == '-') { - if (!s->pending_rm_count) - s->pending_rm_pre_begin = s->lno_pre; - s->lno_pre++; - strbuf_add(&s->pending_rm, line, len); - s->pending_rm_count++; - return s->ret; - } + if (filter->ret) + return filter->ret; if (line[0] == '\\') { - if (s->pending_rm_count) - strbuf_add(&s->pending_rm, line, len); - else if (s->rhunk_active) - strbuf_add(&s->rhunk, line, len); - /* otherwise outside tracked range; drop silently */ - return s->ret; + if (filter->hunk.active) + strbuf_add(&filter->hunk.lines, line, len); + return filter->ret; } - if (line[0] != '+' && line[0] != ' ') + if (line[0] != '+' && line[0] != ' ' && line[0] != '-') BUG("unexpected diff line type '%c'", line[0]); - lno_0 = s->lno_post - 1; - cur_pre = s->lno_pre; /* save before advancing for context lines */ - s->lno_post++; - if (line[0] == ' ') - s->lno_pre++; + /* + * idx_in_postimage is this line's 0-based post-image index (see the model on + * struct line_range_filter). The cursors are advanced only after + * the line is classified, so a '-' is tested at the same idx_in_postimage as + * the '+'/' ' that follows it. + */ + idx_in_postimage = filter->lno_in_postimage - 1; - /* Advance past ranges we've passed */ - while (s->cur_range < s->ranges->nr && - lno_0 >= s->ranges->ranges[s->cur_range].end) { - if (s->rhunk_active) - flush_rhunk(s); - discard_pending_rm(s); - s->cur_range++; + /* Retire ranges we have passed, flushing the one we leave. */ + while (filter->cur_range < filter->ranges->nr && + idx_in_postimage >= filter->ranges->ranges[filter->cur_range].end) { + if (filter->hunk.active) + flush_range_hunk(filter); + filter->cur_range++; } - /* Past all ranges */ - if (s->cur_range >= s->ranges->nr) { - discard_pending_rm(s); - return s->ret; - } + in_range = filter->cur_range < filter->ranges->nr && + idx_in_postimage >= filter->ranges->ranges[filter->cur_range].start && + idx_in_postimage < filter->ranges->ranges[filter->cur_range].end; - cur = &s->ranges->ranges[s->cur_range]; + if (in_range) { + if (!filter->hunk.active) + begin_range_hunk(filter); - /* Before current range */ - if (lno_0 < cur->start) { - discard_pending_rm(s); - return s->ret; + strbuf_add(&filter->hunk.lines, line, len); } - /* In range so start a new range hunk if needed */ - if (!s->rhunk_active) { - s->rhunk_active = 1; - s->rhunk_has_changes = 0; - s->rhunk_new_begin = lno_0 + 1; - s->rhunk_old_begin = s->pending_rm_count - ? s->pending_rm_pre_begin : cur_pre; - s->rhunk_old_count = 0; - s->rhunk_new_count = 0; - strbuf_reset(&s->rhunk); + /* + * Advance each image's cursor: a line present in that image (see + * the model) consumes one of its line numbers. + */ + if (line[0] != '-') + filter->lno_in_postimage++; + if (line[0] != '+') + filter->lno_in_preimage++; + + return filter->ret; +} + +/* + * Run an xdiff pass through an initialized line-range filter, flush the + * final range hunk, and release the filter. Inflates ctxlen to the largest + * range span first, so that every change within a single range lands in one + * xdiff hunk and the inter-change context is emitted; the filter then clips + * back to range boundaries. The optimal ctxlen depends on where changes fall + * within the range, which is only known after xdiff runs, so the max span is + * the upper bound that guarantees correctness in a single pass. Every + * consumer (patch, diffstat, check) relies on one xdiff hunk per range, so + * this lives here rather than at each call site. Also clears + * XDL_EMIT_NO_HUNK_HDR: the filter seeds its per-image position from the hunk + * headers, so a consumer that otherwise suppresses them (diffstat) still gets + * them here. Returns non-zero if xdiff or any forwarded callback failed. + */ +static int line_range_filter_diff(struct line_range_filter *filter, + mmfile_t *mf1, mmfile_t *mf2, + xpparam_t *xpp, xdemitconf_t *xecfg) +{ + const struct range_set *ranges = filter->ranges; + long max_span = 0; + unsigned int i; + int ret; + + for (i = 0; i < ranges->nr; i++) { + long span = ranges->ranges[i].end - ranges->ranges[i].start; + if (span > max_span) + max_span = span; } + if (max_span > xecfg->ctxlen) + xecfg->ctxlen = max_span; - /* Flush pending removals into range hunk */ - if (s->pending_rm_count) { - strbuf_addbuf(&s->rhunk, &s->pending_rm); - s->rhunk_old_count += s->pending_rm_count; - s->rhunk_has_changes = 1; - discard_pending_rm(s); + /* the filter seeds its per-image position from hunk headers */ + xecfg->flags &= ~XDL_EMIT_NO_HUNK_HDR; + + ret = xdi_diff_outf(mf1, mf2, line_range_hunk_fn, + line_range_line_fn, filter, xpp, xecfg); + if (!ret) { + flush_range_hunk(filter); + ret = filter->ret; } + strbuf_release(&filter->hunk.lines); + return ret; +} - strbuf_add(&s->rhunk, line, len); - s->rhunk_new_count++; - if (line[0] == '+') - s->rhunk_has_changes = 1; - else - s->rhunk_old_count++; +/* + * Expose the in-file line-range filter to callers outside diff.c (e.g. + * pickaxe -G); see xdiff-interface.h for the contract. + */ +int diff_emit_line_ranges(mmfile_t *one, mmfile_t *two, + const struct range_set *ranges, + xdiff_emit_line_fn line_fn, void *cb_data, + xpparam_t *xpp, xdemitconf_t *xecfg) +{ + struct line_range_filter filter; - return s->ret; + line_range_filter_init(&filter, ranges, line_fn, cb_data); + return line_range_filter_diff(&filter, one, two, xpp, xecfg); } static void pprint_rename(struct strbuf *name, const char *a, const char *b) @@ -2819,6 +2932,77 @@ static struct diffstat_file *diffstat_add(struct diffstat_t *diffstat, return x; } +struct diffstat_hunk_cb_data { + struct precomputed_hunk **h; + size_t *nr, *alloc; +}; + +/* + * Hunk callback that appends each hunk's coordinates to a growable + * array, so one xdiff pass can both sum a diffstat and record hunks for + * the store. + */ +static int diffstat_hunk_cb(long start_a, long count_a, + long start_b, long count_b, + void *cb_data) +{ + struct diffstat_hunk_cb_data *d = cb_data; + + ALLOC_GROW(*d->h, *d->nr + 1, *d->alloc); + (*d->h)[*d->nr].old_start = start_a; + (*d->h)[*d->nr].old_count = count_a; + (*d->h)[*d->nr].new_start = start_b; + (*d->h)[*d->nr].new_count = count_b; + (*d->nr)++; + return 0; +} + +/* + * Collect the hunks of the two files at zero context. diff_fn chooses + * whether trimming runs: xdi_diff applies trim_common_tail, yielding the + * zero-context hunks blame reads; xdl_diff does not, yielding the + * untrimmed hunks. Both run at zero context, so the untrimmed hunks are + * not grouped the way a nonzero context would group them; diffstat only + * sums their counts, which grouping does not change. Sets *ph (caller + * frees) and *ph_nr. + */ +typedef int (*xdiff_fn)(mmfile_t *, mmfile_t *, xpparam_t const *, + xdemitconf_t const *, xdemitcb_t *); +static int collect_hunks(xdiff_fn diff_fn, mmfile_t *mf1, mmfile_t *mf2, + xpparam_t *xpp, struct precomputed_hunk **ph, + size_t *ph_nr) +{ + size_t ph_alloc = 0; + xdemitcb_t ecb = { 0 }; + xdemitconf_t xecfg = { 0 }; + struct diffstat_hunk_cb_data cd = { ph, ph_nr, &ph_alloc }; + + *ph = NULL; + *ph_nr = 0; + xecfg.hunk_func = diffstat_hunk_cb; + ecb.priv = &cd; + return diff_fn(mf1, mf2, xpp, &xecfg, &ecb); +} + +void diff_hunks_attach(struct diff_options *o) +{ + if (!(o->output_format & + (DIFF_FORMAT_DIFFSTAT | DIFF_FORMAT_SHORTSTAT | DIFF_FORMAT_NUMSTAT))) + return; + o->hunks_writer = diff_hunks_writer_maybe_new(o->repo); +} + +void diff_hunks_detach(struct diff_options *o) +{ + unsigned long hits, misses; + + diff_hunks_read_stats(o->repo, &hits, &misses); + if (hits) + trace2_data_intmax("diff-hunks", o->repo, "read-hits", hits); + diff_hunks_writer_finish(o->hunks_writer); + o->hunks_writer = NULL; +} + static int diffstat_consume(void *priv, char *line, unsigned long len) { struct diffstat_t *diffstat = priv; @@ -3586,7 +3770,7 @@ static unsigned char *deflate_it(char *data, unsigned long size, unsigned long *result_size) { - int bound; + size_t bound; unsigned char *deflated; git_zstream stream; struct repo_config_values *cfg = repo_config_values(the_repository); @@ -3624,9 +3808,11 @@ static void emit_binary_diff_body(struct diff_options *o, delta = NULL; deflated = deflate_it(two->ptr, two->size, &deflate_size); if (one->size && two->size) { + size_t delta_size_st = 0; delta = diff_delta(one->ptr, one->size, two->ptr, two->size, - &delta_size, deflate_size); + &delta_size_st, deflate_size); + delta_size = cast_size_t_to_ulong(delta_size_st); if (delta) { void *to_free = delta; orig_size = delta_size; @@ -4064,51 +4250,15 @@ static void builtin_diff(const char *name_a, xdi_diff_outf(&mf1, &mf2, NULL, quick_consume, &ecbdata, &xpp, &xecfg); } else if (line_ranges) { - struct line_range_callback lr_state; - unsigned int i; - long max_span = 0; + struct line_range_filter lr_filter; - memset(&lr_state, 0, sizeof(lr_state)); - lr_state.orig_line_fn = fn_out_consume; - lr_state.orig_cb_data = &ecbdata; - lr_state.ranges = line_ranges; - strbuf_init(&lr_state.rhunk, 0); - strbuf_init(&lr_state.pending_rm, 0); + line_range_filter_init(&lr_filter, line_ranges, + fn_out_consume, &ecbdata); - /* - * Inflate ctxlen so that all changes within - * any single range are merged into one xdiff - * hunk and the inter-change context is emitted. - * The callback clips back to range boundaries. - * - * The optimal ctxlen depends on where changes - * fall within the range, which is only known - * after xdiff runs; the max range span is the - * upper bound that guarantees correctness in a - * single pass. - */ - for (i = 0; i < line_ranges->nr; i++) { - long span = line_ranges->ranges[i].end - - line_ranges->ranges[i].start; - if (span > max_span) - max_span = span; - } - if (max_span > xecfg.ctxlen) - xecfg.ctxlen = max_span; - - if (xdi_diff_outf(&mf1, &mf2, - line_range_hunk_fn, - line_range_line_fn, - &lr_state, &xpp, &xecfg)) + if (line_range_filter_diff(&lr_filter, &mf1, &mf2, + &xpp, &xecfg)) die("unable to generate diff for %s", one->path); - - flush_rhunk(&lr_state); - if (lr_state.ret) - die("unable to generate diff for %s", - one->path); - strbuf_release(&lr_state.rhunk); - strbuf_release(&lr_state.pending_rm); } else if (xdi_diff_outf(&mf1, &mf2, NULL, fn_out_consume, &ecbdata, &xpp, &xecfg)) die("unable to generate diff for %s", one->path); @@ -4156,6 +4306,125 @@ static const char *get_compact_summary(const struct diff_filepair *p, int is_ren return NULL; } +/* + * Hunk callback for the provider interface: sum counts into a + * diffstat entry. + */ +static int diffstat_sum_hunk_cb(long start_a UNUSED, long count_a, + long start_b UNUSED, long count_b, + void *cb_data) +{ + struct diffstat_file *data = cb_data; + + data->added += count_b; + data->deleted += count_a; + return 0; +} + +/* + * Fill data->added/deleted for a modified pair through the hunk provider + * interface: on an answer, sum the provided counts; on a warming run, + * compute and record them. Returns 1 when it produced the counts, 0 when + * the caller must compute the diffstat itself. + * + * The providers own the exclusions the request can express (-B, -I, + * and --anchored are outside the store key). This consumer additionally + * excludes --ignore-blank-lines before consulting: that flag is part of + * the key, but it coalesces hunks differently between the emit and + * hunk-callback paths, so a served answer would not match a store-less + * run's --stat output. (--inter-hunk-context is not excluded: it only + * groups hunks, and diffstat sums their counts, which grouping does not + * change.) Recording requires both sides to be valid regular files whose + * blobs the key can name. + */ +static int diffstat_from_hunks(struct diff_options *o, + struct diff_filespec *one, + struct diff_filespec *two, + struct diffstat_file *data) +{ + struct precomputed_hunk *ph_trim, *ph_full, *counts; + size_t n_trim, n_full, n_counts, k; + mmfile_t mf1, mf2; + xpparam_t xpp = { .flags = o->xdl_opts, + .ignore_regex = o->ignore_regex, + .ignore_regex_nr = o->ignore_regex_nr, + .anchors = o->anchors, + .anchors_nr = o->anchors_nr }; + struct diff_provider_request req = { + .repo = o->repo, + .old_oid = (one->oid_valid && !S_ISGITLINK(one->mode)) ? + &one->oid : NULL, + .new_oid = (two->oid_valid && !S_ISGITLINK(two->mode)) ? + &two->oid : NULL, + /* + * Attribute lookup and the process protocol need the + * repo-relative path; the display name a caller passes + * around may be stripped of o->prefix and would miss a + * driver scoped to a directory. + */ + .path = one->path, + .diffopt = o, + .xpp = &xpp, + }; + + if (o->xdl_opts & XDF_IGNORE_BLANK_LINES) + return 0; + /* format-patch keeps its diffstat off the store (see the flag). */ + if (o->flags.no_precomputed_hunks) + return 0; + + switch (diff_provider_consult(&req, diffstat_sum_hunk_cb, data)) { + case DIFF_PROVIDER_ANSWERED: + return 1; + case DIFF_PROVIDER_UNANSWERED: + break; + case DIFF_PROVIDER_ERROR: /* not returned by a consult */ + case DIFF_PROVIDER_UNANSWERED_NO_RECORD: + return 0; + } + + /* A miss on a read-only run: let the caller compute the diffstat. */ + if (!o->hunks_writer) + return 0; + /* Recording needs blobs the key can name, on both sides. */ + if (!req.old_oid || !req.new_oid || + !DIFF_FILE_VALID(one) || !DIFF_FILE_VALID(two) || + !S_ISREG(one->mode) || !S_ISREG(two->mode)) + return 0; + + if (fill_mmfile(o->repo, &mf1, one) < 0 || + fill_mmfile(o->repo, &mf2, two) < 0) + die("unable to read files to diff"); + + /* + * Compute the zero-context trimmed diff (what blame reads) and the + * untrimmed diff (whose counts a nonzero-context stat matches). + * xdi_diff runs first: it enforces the size limit, so the xdl_diff + * call is already bounded. + */ + if (collect_hunks(xdi_diff, &mf1, &mf2, &xpp, &ph_trim, &n_trim) || + collect_hunks(xdl_diff, &mf1, &mf2, &xpp, &ph_full, &n_full)) + die("unable to generate diffstat for %s", one->path); + + /* + * Match a store-less run: at zero context xdi_diff trims, so sum the + * trimmed diff; otherwise sum the untrimmed one. + */ + counts = o->context ? ph_full : ph_trim; + n_counts = o->context ? n_full : n_trim; + for (k = 0; k < n_counts; k++) { + data->added += counts[k].new_count; + data->deleted += counts[k].old_count; + } + + diff_hunks_writer_record_stable(o->hunks_writer, &one->oid, &two->oid, + o->xdl_opts, ph_trim, n_trim, + ph_full, n_full); + free(ph_trim); + free(ph_full); + return 1; +} + static void builtin_diffstat(const char *name_a, const char *name_b, struct diff_filespec *one, struct diff_filespec *two, @@ -4207,27 +4476,52 @@ static void builtin_diffstat(const char *name_a, const char *name_b, } else if (may_differ) { - /* Crazy xdl interfaces.. */ - xpparam_t xpp; - xdemitconf_t xecfg; - - if (fill_mmfile(o->repo, &mf1, one) < 0 || - fill_mmfile(o->repo, &mf2, two) < 0) - die("unable to read files to diff"); + /* + * Serve from a hunk provider (the process, then the store), + * or record into the store on a warming run. A "log -L" + * range-scoped stat is not the whole-pair diff the store + * keys, so it neither reads nor records. Otherwise diff + * normally. + */ + if (p->line_ranges || + !diffstat_from_hunks(o, one, two, data)) { + /* Crazy xdl interfaces.. */ + xpparam_t xpp; + xdemitconf_t xecfg; + + if (fill_mmfile(o->repo, &mf1, one) < 0 || + fill_mmfile(o->repo, &mf2, two) < 0) + die("unable to read files to diff"); + + memset(&xpp, 0, sizeof(xpp)); + memset(&xecfg, 0, sizeof(xecfg)); + xpp.flags = o->xdl_opts; + xpp.ignore_regex = o->ignore_regex; + xpp.ignore_regex_nr = o->ignore_regex_nr; + xpp.anchors = o->anchors; + xpp.anchors_nr = o->anchors_nr; + xecfg.ctxlen = o->context; + xecfg.interhunkctxlen = o->interhunkcontext; + xecfg.flags = XDL_EMIT_NO_HUNK_HDR; - memset(&xpp, 0, sizeof(xpp)); - memset(&xecfg, 0, sizeof(xecfg)); - xpp.flags = o->xdl_opts; - xpp.ignore_regex = o->ignore_regex; - xpp.ignore_regex_nr = o->ignore_regex_nr; - xpp.anchors = o->anchors; - xpp.anchors_nr = o->anchors_nr; - xecfg.ctxlen = o->context; - xecfg.interhunkctxlen = o->interhunkcontext; - xecfg.flags = XDL_EMIT_NO_HUNK_HDR; - if (xdi_diff_outf(&mf1, &mf2, NULL, - diffstat_consume, diffstat, &xpp, &xecfg)) - die("unable to generate diffstat for %s", one->path); + if (p->line_ranges) { + struct line_range_filter lr_filter; + + line_range_filter_init(&lr_filter, + p->line_ranges, + diffstat_consume, + diffstat); + + if (line_range_filter_diff(&lr_filter, &mf1, + &mf2, &xpp, &xecfg)) + die("unable to generate diffstat for %s", + one->path); + } else if (xdi_diff_outf(&mf1, &mf2, NULL, + diffstat_consume, diffstat, + &xpp, &xecfg)) + die("unable to generate diffstat for %s", + one->path); + } if (DIFF_FILE_VALID(one) && DIFF_FILE_VALID(two)) { struct diffstat_file *file = @@ -4255,11 +4549,29 @@ static void builtin_diffstat(const char *name_a, const char *name_b, diff_free_filespec_data(two); } +/* + * Is the 0-based line index within any of the tracked ranges? + * (range_set ranges are 0-based, half-open [start, end).) This is a + * one-shot query for a single line and scans; the streaming filter + * (line_range_line_fn) uses a forward cursor instead. + */ +static int idx_in_ranges(const struct range_set *ranges, long idx) +{ + unsigned int i; + + for (i = 0; i < ranges->nr; i++) + if (idx >= ranges->ranges[i].start && + idx < ranges->ranges[i].end) + return 1; + return 0; +} + static void builtin_checkdiff(const char *name_a, const char *name_b, const char *attr_path, struct diff_filespec *one, struct diff_filespec *two, - struct diff_options *o) + struct diff_options *o, + const struct range_set *line_ranges) { mmfile_t mf1, mf2; struct checkdiff_t data; @@ -4299,7 +4611,19 @@ static void builtin_checkdiff(const char *name_a, const char *name_b, memset(&xecfg, 0, sizeof(xecfg)); xecfg.ctxlen = 1; /* at least one context line */ xpp.flags = 0; - if (xdi_diff_outf(&mf1, &mf2, checkdiff_consume_hunk, + + if (line_ranges) { + struct line_range_filter lr_filter; + + line_range_filter_init(&lr_filter, line_ranges, + checkdiff_consume, &data); + lr_filter.orig_hunk_fn = checkdiff_consume_hunk; + + if (line_range_filter_diff(&lr_filter, &mf1, &mf2, + &xpp, &xecfg)) + die("unable to generate checkdiff for %s", + one->path); + } else if (xdi_diff_outf(&mf1, &mf2, checkdiff_consume_hunk, checkdiff_consume, &data, &xpp, &xecfg)) die("unable to generate checkdiff for %s", one->path); @@ -4312,6 +4636,17 @@ static void builtin_checkdiff(const char *name_a, const char *name_b, check_blank_at_eof(&mf1, &mf2, &ecbdata); blank_at_eof = ecbdata.blank_at_eof_in_postimage; + /* + * check_blank_at_eof() scans the whole file; with -L, + * keep the report only when its line is in a tracked + * range. The error's location is the first trailing + * blank line (blank_at_eof, 1-based; ranges 0-based), so + * we scope by that line. + */ + if (blank_at_eof && line_ranges && + !idx_in_ranges(line_ranges, blank_at_eof - 1)) + blank_at_eof = 0; + if (blank_at_eof) { static char *err; if (!err) @@ -5107,7 +5442,8 @@ static void run_checkdiff(struct diff_filepair *p, struct diff_options *o) diff_fill_oid_info(p->one, o->repo->index); diff_fill_oid_info(p->two, o->repo->index); - builtin_checkdiff(name, other, attr_path, p->one, p->two, o); + builtin_checkdiff(name, other, attr_path, p->one, p->two, o, + p->line_ranges); } void repo_diff_setup(struct repository *r, struct diff_options *options) @@ -5904,6 +6240,27 @@ static int diff_opt_submodule(const struct option *opt, return 0; } +static int diff_opt_ext_diff(const struct option *opt, + const char *arg, int unset) +{ + struct diff_options *options = opt->value; + + BUG_ON_OPT_ARG(arg); + options->flags.allow_external = !unset; + options->flags.allow_diff_process = !unset; + return 0; +} + +static int diff_opt_diff_process(const struct option *opt, + const char *arg, int unset) +{ + struct diff_options *options = opt->value; + + BUG_ON_OPT_ARG(arg); + options->flags.allow_diff_process = !unset; + return 0; +} + static int diff_opt_textconv(const struct option *opt, const char *arg, int unset) { @@ -6163,7 +6520,7 @@ struct option *add_diff_options(const struct option *opts, N_("continue listing the history of a file beyond renames"), PARSE_OPT_NOARG, diff_opt_follow), OPT_INTEGER('l', NULL, &options->rename_limit, - N_("prevent rename/copy detection if the number of rename/copy targets exceeds given limit")), + N_("limit to cheap rename/copy detection if the number of rename/copy targets exceeds this value")), OPT_GROUP(N_("Diff algorithm options")), OPT_CALLBACK_F(0, "minimal", options, NULL, @@ -6234,8 +6591,12 @@ struct option *add_diff_options(const struct option *opts, N_("exit with 1 if there were differences, 0 otherwise")), OPT_BOOL(0, "quiet", &options->flags.quick, N_("disable all output of the program")), - OPT_BOOL(0, "ext-diff", &options->flags.allow_external, - N_("allow an external diff helper to be executed")), + OPT_CALLBACK_F(0, "ext-diff", options, NULL, + N_("allow an external diff helper to be executed"), + PARSE_OPT_NOARG, diff_opt_ext_diff), + OPT_CALLBACK_F(0, "diff-process", options, NULL, + N_("allow a configured diff process to be consulted"), + PARSE_OPT_NOARG, diff_opt_diff_process), OPT_CALLBACK_F(0, "textconv", options, NULL, N_("run external text conversion filters when comparing binary files"), PARSE_OPT_NOARG, diff_opt_textconv), diff --git a/diff.h b/diff.h index bb5cddaf3499e9..e0b58a8105fe8f 100644 --- a/diff.h +++ b/diff.h @@ -173,6 +173,15 @@ struct diff_flags { */ unsigned allow_external; + /** + * Allows diff..process to be consulted. Set by the + * porcelain commands whose output may reflect a diff process + * (diff, log, show, blame) and by --ext-diff or --diff-process; + * plumbing does not set it by default, so its output stays + * builtin. Cleared by --no-ext-diff or --no-diff-process. + */ + unsigned allow_diff_process; + /** * For communication between the calling program and the options parser; * tell the calling program to signal the presence of difference using @@ -206,6 +215,13 @@ struct diff_flags { unsigned suppress_diff_headers; unsigned dual_color_diffed_diffs; unsigned suppress_hunk_header_line_count; + + /* + * Do not serve the diffstat from the precomputed-hunks store. + * Set by format-patch so a generated patch carries the builtin + * counts and does not depend on the sender's local store state. + */ + unsigned no_precomputed_hunks; }; static inline void diff_flags_or(struct diff_flags *a, @@ -224,6 +240,17 @@ static inline void diff_flags_or(struct diff_flags *a, #define DIFF_WITH_ALG(opts, flag) (((opts)->xdl_opts & ~XDF_DIFF_ALGORITHM_MASK) | XDF_##flag) +/* + * The xdl_opts bits git turns on by default that a from-scratch xdl_opts + * (git blame's own option parsing) does not set, and so must OR in to match + * a store warmed at the default diff settings; a diff_options-based consumer + * (diffstat) already has them in o->xdl_opts. Today this is only the indent + * heuristic. It does NOT cover a non-default diff.algorithm: a repo that + * configures one records under that algorithm, and a consumer keying without + * it misses (a lost hit, not wrong output). + */ +#define DIFF_HUNKS_DEFAULT_XDL_OPTS XDF_INDENT_HEURISTIC + enum diff_words_type { DIFF_WORDS_NONE = 0, DIFF_WORDS_PORCELAIN, @@ -420,6 +447,15 @@ struct diff_options { */ int max_depth; int max_depth_valid; + + /* + * Precomputed diff hunks (see diff-hunks.h). diffstat consults the + * hunk provider interface before running xdiff, keyed by each file + * pair's blob object IDs. When hunks_writer is set (a warming run), + * diffstat also records the hunks it computes; the writer is + * attached only for the stat output formats. + */ + struct diff_hunks_writer *hunks_writer; }; unsigned diff_filter_bit(char status); @@ -668,6 +704,17 @@ void diffcore_fix_diff_index(void); int diff_queue_is_empty(struct diff_options *o); void diff_flush(struct diff_options*); void diff_free(struct diff_options*); + +/* + * Attach a diff-hunks writer to a diff producing a stat format, so a + * warming run records the hunks it computes; a no-op when writing is off + * or for other formats. (Reading is separate: consumers consult the + * providers through diff_provider_consult(); see diff-provider.h.) Pair + * with diff_hunks_detach() once the diff is done. + */ +void diff_hunks_attach(struct diff_options *o); +void diff_hunks_detach(struct diff_options *o); + void diff_warn_rename_limit(const char *varname, int needed, int degraded_cc); /* diff-raw status letters */ diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index b0915be86fc475..bad3a88c0f7ea3 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -16,7 +16,8 @@ typedef int (*pickaxe_fn)(mmfile_t *one, mmfile_t *two, struct diff_options *o, - regex_t *regexp, kwset_t kws); + regex_t *regexp, kwset_t kws, + const struct range_set *ranges); struct diffgrep_cb { regex_t *regexp; @@ -42,7 +43,8 @@ static int diffgrep_consume(void *priv, char *line, unsigned long len) static int diff_grep(mmfile_t *one, mmfile_t *two, struct diff_options *o, - regex_t *regexp, kwset_t kws UNUSED) + regex_t *regexp, kwset_t kws UNUSED, + const struct range_set *ranges) { struct diffgrep_cb ecbdata; xpparam_t xpp; @@ -50,8 +52,11 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, int ret; /* - * We have both sides; need to run textual diff and see if - * the pattern appears on added/deleted lines. + * We have both sides; need to run textual diff and see if the + * pattern appears on added/deleted lines. Under -L (ranges set), + * forward only the tracked range's lines so the match is scoped. + * -G needs only a hit/no-hit answer, so the line-number bookkeeping + * the filter does for -L patch and check output is irrelevant here. */ memset(&xpp, 0, sizeof(xpp)); memset(&xecfg, 0, sizeof(xecfg)); @@ -65,8 +70,12 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, * An xdiff error might be our "data->hit" from above. See the * comment for xdiff_emit_line_fn in xdiff-interface.h */ - ret = xdi_diff_outf(one, two, NULL, diffgrep_consume, - &ecbdata, &xpp, &xecfg); + if (ranges) + ret = diff_emit_line_ranges(one, two, ranges, diffgrep_consume, + &ecbdata, &xpp, &xecfg); + else + ret = xdi_diff_outf(one, two, NULL, diffgrep_consume, + &ecbdata, &xpp, &xecfg); if (ecbdata.hit) return 1; if (ret) @@ -119,8 +128,13 @@ static unsigned int contains(mmfile_t *mf, regex_t *regexp, kwset_t kws, static int has_changes(mmfile_t *one, mmfile_t *two, struct diff_options *o UNUSED, - regex_t *regexp, kwset_t kws) + regex_t *regexp, kwset_t kws, + const struct range_set *ranges UNUSED) { + /* + * -S counts needle occurrences in each whole blob. Scoping this to + * a -L range is left to a follow-up; for now -S ignores the range. + */ unsigned int c1 = one ? contains(one, regexp, kws, 0) : 0; unsigned int c2 = two ? contains(two, regexp, kws, c1 + 1) : 0; return c1 != c2; @@ -132,6 +146,7 @@ static int pickaxe_match(struct diff_filepair *p, struct diff_options *o, struct userdiff_driver *textconv_one = NULL; struct userdiff_driver *textconv_two = NULL; mmfile_t mf1, mf2; + const struct range_set *ranges; int ret; /* ignore unmerged */ @@ -169,7 +184,13 @@ static int pickaxe_match(struct diff_filepair *p, struct diff_options *o, mf1.size = fill_textconv(o->repo, textconv_one, p->one, &mf1.ptr); mf2.size = fill_textconv(o->repo, textconv_two, p->two, &mf2.ptr); - ret = fn(&mf1, &mf2, o, regexp, kws); + /* + * -L scopes the search to the tracked range, but the range is in + * original-file line coordinates that do not map onto textconv + * output, so search the whole file when textconv is in play. + */ + ranges = (textconv_one || textconv_two) ? NULL : p->line_ranges; + ret = fn(&mf1, &mf2, o, regexp, kws, ranges); if (textconv_one) free(mf1.ptr); diff --git a/environment.c b/environment.c index 76ee65e62b4823..2933b258964946 100644 --- a/environment.c +++ b/environment.c @@ -119,23 +119,23 @@ int is_bare_repository(struct repository *repo) int repo_protect_ntfs(struct repository *repo) { - return (repo && repo->initialized) ? - repo_config_values(repo)->protect_ntfs : - PROTECT_NTFS_DEFAULT; + return repo->initialized + ? repo_config_values(repo)->protect_ntfs + : PROTECT_NTFS_DEFAULT; } int repo_protect_hfs(struct repository *repo) { - return (repo && repo->initialized) ? - repo_config_values(repo)->protect_hfs : - PROTECT_HFS_DEFAULT; + return repo->initialized + ? repo_config_values(repo)->protect_hfs + : PROTECT_HFS_DEFAULT; } int repo_ignore_case(struct repository *repo) { - return (repo && repo->initialized) ? - repo_config_values(repo)->ignore_case : - 0; + return repo->initialized + ? repo_config_values(repo)->ignore_case + : 0; } int repo_trust_executable_bit(struct repository *repo) @@ -253,6 +253,7 @@ static const struct fsync_component_name { { "pack", FSYNC_COMPONENT_PACK }, { "pack-metadata", FSYNC_COMPONENT_PACK_METADATA }, { "commit-graph", FSYNC_COMPONENT_COMMIT_GRAPH }, + { "diff-hunks", FSYNC_COMPONENT_DIFF_HUNKS }, { "index", FSYNC_COMPONENT_INDEX }, { "objects", FSYNC_COMPONENTS_OBJECTS }, { "reference", FSYNC_COMPONENT_REFERENCE }, @@ -571,6 +572,11 @@ int git_default_core_config(const char *var, const char *value, return 0; } + if (!strcmp(var, "core.usenanosec")) { + cfg->use_nanosec = git_config_bool(var, value); + return 0; + } + /* Add other config variables here and to Documentation/config.adoc. */ return platform_core_config(var, value, ctx, cb); } @@ -745,6 +751,7 @@ int git_default_config(const char *var, const char *value, void repo_config_values_init(struct repo_config_values *cfg) { + /* section "core" config values */ cfg->attributes_file = NULL; cfg->excludes_file = NULL; cfg->editor_program = NULL; @@ -756,20 +763,25 @@ void repo_config_values_init(struct repo_config_values *cfg) cfg->autorebase = AUTOREBASE_NEVER; cfg->object_creation_mode = OBJECT_CREATION_MODE; cfg->apply_sparse_checkout = 0; - cfg->protect_hfs = PROTECT_HFS_DEFAULT; - cfg->protect_ntfs = PROTECT_NTFS_DEFAULT; - cfg->ignore_case = 0; - cfg->trust_executable_bit = 1; - cfg->has_symlinks = platform_has_symlinks(); - cfg->branch_track = BRANCH_TRACK_REMOTE; cfg->trust_ctime = 1; cfg->check_stat = 1; cfg->zlib_compression_level = Z_BEST_SPEED; cfg->pack_compression_level = Z_DEFAULT_COMPRESSION; cfg->precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ cfg->core_sparse_checkout_cone = 0; - cfg->sparse_expect_files_outside_of_patterns = 0; cfg->warn_on_object_refname_ambiguity = 1; + cfg->protect_hfs = PROTECT_HFS_DEFAULT; + cfg->protect_ntfs = PROTECT_NTFS_DEFAULT; + cfg->ignore_case = 0; + cfg->trust_executable_bit = 1; + cfg->has_symlinks = platform_has_symlinks(); + cfg->use_nanosec = 0; + + /* section "sparse" config values */ + cfg->sparse_expect_files_outside_of_patterns = 0; + + /* section "branch" config values */ + cfg->branch_track = BRANCH_TRACK_REMOTE; } void repo_config_values_clear(struct repo_config_values *cfg) diff --git a/environment.h b/environment.h index e7ec5b0437342d..59cb06f0a0d559 100644 --- a/environment.h +++ b/environment.h @@ -115,7 +115,6 @@ enum object_creation_mode { }; struct repo_config_values { - /* section "core" config values */ char *attributes_file; char *excludes_file; char *editor_program; @@ -139,11 +138,8 @@ struct repo_config_values { int ignore_case; int trust_executable_bit; int has_symlinks; - - /* section "sparse" config values */ + int use_nanosec; int sparse_expect_files_outside_of_patterns; - - /* section "branch" config values */ enum branch_track branch_track; }; @@ -175,22 +171,15 @@ int git_default_core_config(const char *var, const char *value, const struct config_context *ctx, void *cb); /* - * Getters for the `protect_hfs` and `protect_ntfs` fields of `struct repo_config_values`. - * They check `repo->initialized` to prevent calling `repo_config_values()` - * before the repository setup is fully complete or in non-git environments. + * Getters for configuration variables in `struct repo_config_values`. + * These functions require a non-NULL repository pointer and handle + * repositories that are not fully initialized by returning appropriate + * default values. */ int repo_protect_hfs(struct repository *repo); int repo_protect_ntfs(struct repository *repo); - -/* - * Getter for the `ignore_case` field of `struct repo_config_values`. - * It checks `repo->initialized` to prevent calling repo_config_values()` - * before the repository setup is fully complete or in non-git environments. - */ int repo_ignore_case(struct repository *repo); - int repo_trust_executable_bit(struct repository *repo); - int repo_has_symlinks(struct repository *repo); const char *repo_excludes_file(struct repository *repo); diff --git a/fetch-object-info.c b/fetch-object-info.c index ba7e179c44ee54..0a58308f9b2559 100644 --- a/fetch-object-info.c +++ b/fetch-object-info.c @@ -1,6 +1,7 @@ #include "git-compat-util.h" #include "gettext.h" #include "hex.h" +#include "object.h" #include "pkt-line.h" #include "connect.h" #include "oid-array.h" @@ -9,20 +10,26 @@ #include "string-list.h" /* Sends object-info command and its arguments into the request buffer. */ -static void send_object_info_request(const int fd_out, struct object_info_args *args) +static void send_object_info_request(const int fd_out, + const struct string_list *server_options, + const struct oid_array *oids, + unsigned ask_size, + unsigned ask_type) { struct strbuf req_buf = STRBUF_INIT; - write_command_and_capabilities(&req_buf, "object-info", args->server_options); + write_command_and_capabilities(&req_buf, "object-info", server_options); - if (unsorted_string_list_has_string(args->object_info_options, "size")) + if (ask_size) packet_buf_write(&req_buf, "size"); - else if (args->object_info_options->nr) - BUG("only size should be in object_info_options"); - if (args->oids) - for (size_t i = 0; i < args->oids->nr; i++) - packet_buf_write(&req_buf, "oid %s", oid_to_hex(&args->oids->oid[i])); + if (ask_type) + packet_buf_write(&req_buf, "type"); + + if (oids) + for (size_t i = 0; i < oids->nr; i++) + packet_buf_write(&req_buf, "oid %s", + oid_to_hex(&oids->oid[i])); packet_buf_flush(&req_buf); if (write_in_full(fd_out, req_buf.buf, req_buf.len) < 0) @@ -45,35 +52,42 @@ static int parse_object_size(const char *s, size_t *res) return 0; } -int fetch_object_info(const enum protocol_version version, struct object_info_args *args, - struct packet_reader *reader, struct object_info *object_info_data, - const int stateless_rpc, const int fd_out) +void fetch_object_info(const enum protocol_version version, + const struct string_list *server_options, + const struct oid_array *oids, + struct packet_reader *reader, + struct fetch_object_info_results *results, + const int stateless_rpc, + const int fd_out) { + unsigned ask_size = 0; + unsigned ask_type = 0; int size_index = -1; + int type_index = -1; + size_t wanted; + + results->nr = oids->nr; + CALLOC_ARRAY(results->unrecognized, results->nr); switch (version) { case protocol_v2: if (!server_supports_v2("object-info")) die(_("object-info capability is not enabled on the server")); - /* - * When removing an element from the list it gets swapped by the - * last element, iterate backwards to prevent elements skipping - * evaluation. - * - * object_info_options->nr can be safely casted without overflow - * because the number of options is a small known number (the - * supported placeholders which currently are size and type). - */ - for (int i = (int)args->object_info_options->nr - 1; i >= 0; i--) - if (!server_supports_feature("object-info", - args->object_info_options->items[i].string, 0)) - unsorted_string_list_delete_item(args->object_info_options, i, 0); + + if (results->wants_size && + server_supports_feature("object-info", "size", 0)) + ask_size = 1; + + if (results->wants_type && + server_supports_feature("object-info", "type", 0)) + ask_type = 1; /* * Even if no options are left, we still send the oid so we get * at least an existence check. */ - send_object_info_request(fd_out, args); + send_object_info_request(fd_out, server_options, oids, ask_size, + ask_type); break; case protocol_v1: case protocol_v0: @@ -81,43 +95,49 @@ int fetch_object_info(const enum protocol_version version, struct object_info_ar case protocol_unknown_version: BUG("unknown protocol version"); } + wanted = ask_size + ask_type; - for (size_t i = 0; i < args->object_info_options->nr; i++) { + for (size_t i = 0; i < wanted; i++) { if (packet_reader_read(reader) != PACKET_READ_NORMAL) { check_stateless_delimiter(stateless_rpc, reader, "stateless delimiter expected"); - return -1; + die(_("object-info: expected %" PRIuMAX " attributes, got %" PRIuMAX), + (uintmax_t)wanted, (uintmax_t)i); } - if (!unsorted_string_list_has_string(args->object_info_options, reader->line)) - return -1; - if (!strcmp(reader->line, "size")) { - /* - * i is the number of supported options which currently - * is only size. No risk of overflow. - */ + if (!ask_size) + die(_("object-info: unrequested 'size' attribute")); + if (results->sizes) + die(_("object-info: duplicate 'size' attribute")); size_index = (int)i; - for (size_t j = 0; j < args->oids->nr; j++) - object_info_data[j].sizep = - xcalloc(1, sizeof(*object_info_data[j].sizep)); + CALLOC_ARRAY(results->sizes, results->nr); + } else if (!strcmp(reader->line, "type")) { + if (!ask_type) + die(_("object-info: unrequested 'type' attribute")); + if (results->types) + die(_("object-info: duplicate 'type' attribute")); + type_index = (int)i; + CALLOC_ARRAY(results->types, results->nr); } else { - BUG("only size is supported"); + die(_("object-info: unknown attribute '%s'"), + reader->line); } } - for (size_t i = 0; - packet_reader_read(reader) == PACKET_READ_NORMAL && - i < args->oids->nr; - i++) { + for (size_t i = 0; i < oids->nr; i++) { struct string_list object_info_values = STRING_LIST_INIT_DUP; + if (packet_reader_read(reader) != PACKET_READ_NORMAL) + die(_("object-info: expected %" PRIuMAX " objects, got %" PRIuMAX), + (uintmax_t)oids->nr, (uintmax_t)i); + string_list_split(&object_info_values, reader->line, " ", -1); if (strcmp(object_info_values.items[0].string, - oid_to_hex(&args->oids->oid[i]))) + oid_to_hex(&oids->oid[i]))) die(_("object-info: expected OID: %s, got %s"), - oid_to_hex(&args->oids->oid[i]), + oid_to_hex(&oids->oid[i]), object_info_values.items[0].string); /* @@ -127,30 +147,53 @@ int fetch_object_info(const enum protocol_version version, struct object_info_ar */ if (object_info_values.nr >= 2 && !strcmp(object_info_values.items[1].string, "")) { - object_info_data[i].unrecognized = 1; + results->unrecognized[i] = 1; string_list_clear(&object_info_values, 0); continue; } /* - * Because we filter the options to be only the supported by - * the server we expect the server to answer with the same - * number of attributes requested. + * Because we only ask for attributes the server said it + * supports, we expect the answer to have one value per + * requested attribute, plus the OID. */ - if (args->object_info_options->nr + 1 != object_info_values.nr) + if (wanted + 1 != object_info_values.nr) die("object-info: unexpected number of attributes: %s", reader->line); - if (size_index >= 0 && + if (results->sizes && parse_object_size(object_info_values.items[size_index + 1].string, - object_info_data[i].sizep)) - die("object-info: ref %s has invalid size %s", + &results->sizes[i])) + die("object-info: object %s has invalid size %s", object_info_values.items[0].string, object_info_values.items[size_index + 1].string); + if (results->types) { + const char *type_str = + object_info_values.items[type_index + 1].string; + int type = type_from_string_gently(type_str, -1, 1); + + if (type < 0) + die(_("object-info: object %s has invalid type '%s'"), + object_info_values.items[0].string, type_str); + + results->types[i] = type; + } + string_list_clear(&object_info_values, 0); } + + if (packet_reader_read(reader) != PACKET_READ_FLUSH) + die(_("object-info: expected flush after %" PRIuMAX " objects"), + (uintmax_t)oids->nr); + check_stateless_delimiter(stateless_rpc, reader, "stateless delimiter expected"); +} - return 0; +void free_fetch_object_info_results(struct fetch_object_info_results *results) +{ + free(results->sizes); + free(results->types); + free(results->unrecognized); + memset(results, 0, sizeof(*results)); } diff --git a/fetch-object-info.h b/fetch-object-info.h index 269cebb3f7df48..2fba96c6f7de52 100644 --- a/fetch-object-info.h +++ b/fetch-object-info.h @@ -1,25 +1,40 @@ #ifndef FETCH_OBJECT_INFO_H #define FETCH_OBJECT_INFO_H +#include "object.h" #include "pkt-line.h" #include "protocol.h" -struct object_info_args { - struct string_list *object_info_options; - const struct string_list *server_options; - struct oid_array *oids; +struct fetch_object_info_results { + size_t *sizes; + enum object_type *types; + uint8_t *unrecognized; + size_t nr; + unsigned wants_size:1; + unsigned wants_type:1; }; -struct object_info; +#define FETCH_OBJECT_INFO_RESULTS_INIT { 0 } + +struct oid_array; /* - * Sends git-cat-file object-info command into the request buf and read the + * Sends git-cat-file object-info command into the request buf and reads the * results from packets. * - * Modifies args->object_info_options, on return it contains only the supported - * options by the server. + * The caller sets the wants_* flags in "results" to indicate which attributes + * it is interested in. On return, "results" holds one array per attribute that + * the server both advertised and answered with. An array left NULL means the + * attribute is not available. + * Release them with free_fetch_object_info_results(). */ -int fetch_object_info(enum protocol_version version, struct object_info_args *args, - struct packet_reader *reader, struct object_info *object_info_data, - int stateless_rpc, int fd_out); +void fetch_object_info(enum protocol_version version, + const struct string_list *server_options, + const struct oid_array *oids, + struct packet_reader *reader, + struct fetch_object_info_results *results, + int stateless_rpc, + int fd_out); + +void free_fetch_object_info_results(struct fetch_object_info_results *results); #endif /* FETCH_OBJECT_INFO_H */ diff --git a/fetch-pack.c b/fetch-pack.c index 626f799712ec2e..5d0296f970e464 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1075,7 +1075,7 @@ static int get_pack(struct fetch_pack_args *args, die(_("fetch-pack: unable to fork off %s"), cmd_name); if (do_keep && (pack_lockfiles || fsck_objects)) { int is_well_formed; - char *pack_lockfile = index_pack_lockfile(the_repository, + char *pack_lockfile = index_pack_lockfile(the_repository->objects->sources, cmd.out, &is_well_formed); diff --git a/git-compat-util.h b/git-compat-util.h index a0f901ce793ed6..a2d8651ed748cd 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -162,6 +162,9 @@ static inline int is_xplatform_dir_sep(int c) #include "compat/win32/path-utils.h" #include "compat/msvc.h" #endif +#ifdef DARWIN_REGEXEC +#include "compat/darwin.h" +#endif /* used on Mac OS X */ #ifdef PRECOMPOSE_UNICODE @@ -996,6 +999,7 @@ static inline int strtol_i(char const *s, int base, int *result) #error "Git requires REG_STARTEND support. Compile with NO_REGEX=NeedsStartEnd" #endif +#ifndef regexec_buf static inline int regexec_buf(const regex_t *preg, const char *buf, size_t size, size_t nmatch, regmatch_t pmatch[], int eflags) { @@ -1004,6 +1008,7 @@ static inline int regexec_buf(const regex_t *preg, const char *buf, size_t size, pmatch[0].rm_eo = size; return regexec(preg, buf, nmatch, pmatch, eflags | REG_STARTEND); } +#endif #ifdef USE_ENHANCED_BASIC_REGULAR_EXPRESSIONS int git_regcomp(regex_t *preg, const char *pattern, int cflags); diff --git a/git-send-email.perl b/git-send-email.perl index bb8ddd1eef2c25..2071cff6ae3f0a 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -863,7 +863,7 @@ sub get_patch_subject { return "GIT: $1\n"; } close $fh; - die sprintf(__("No subject line in %s?"), $fn); + die sprintf(__("No 'Subject:' line in '%s'\n"), $fn); } if ($compose) { diff --git a/git-zlib.c b/git-zlib.c index d21adb3bf5b15e..1c94f90497ee47 100644 --- a/git-zlib.c +++ b/git-zlib.c @@ -33,7 +33,7 @@ static const char *zerr_to_string(int status) /* uLong is 32-bit on Windows, even on 64-bit systems */ #define ULONG_MAX_VALUE maximum_unsigned_value_of_type(uLong) -static inline uInt zlib_buf_cap(unsigned long len) +static inline uInt zlib_buf_cap(size_t len) { return (ZLIB_BUF_MAX < len) ? ZLIB_BUF_MAX : len; } @@ -167,9 +167,21 @@ int git_inflate(git_zstream *strm, int flush) return status; } -unsigned long git_deflate_bound(git_zstream *strm, unsigned long size) +size_t git_deflate_bound(git_zstream *strm, size_t size) { - return deflateBound(&strm->z, size); +#if SIZE_MAX > ULONG_MAX + if (size > maximum_unsigned_value_of_type(uLong)) + /* + * deflateBound() takes uLong, which is 32-bit on + * Windows. For inputs above that range, return zlib's + * stored-block formula (the conservative path it would + * itself use for an unknown stream state) plus the + * worst-case wrapper overhead. + */ + return size + (size >> 5) + (size >> 7) + (size >> 11) + + 7 + 18; +#endif + return deflateBound(&strm->z, (uLong)size); } void git_deflate_init(git_zstream *strm, int level) diff --git a/git-zlib.h b/git-zlib.h index 44380e8ad38305..9248d11ca9622c 100644 --- a/git-zlib.h +++ b/git-zlib.h @@ -5,8 +5,8 @@ typedef struct git_zstream { struct z_stream_s z; - unsigned long avail_in; - unsigned long avail_out; + size_t avail_in; + size_t avail_out; size_t total_in; size_t total_out; unsigned char *next_in; @@ -25,6 +25,6 @@ void git_deflate_end(git_zstream *); int git_deflate_abort(git_zstream *); int git_deflate_end_gently(git_zstream *); int git_deflate(git_zstream *, int flush); -unsigned long git_deflate_bound(git_zstream *, unsigned long); +size_t git_deflate_bound(git_zstream *, size_t); #endif /* GIT_ZLIB_H */ diff --git a/git.c b/git.c index e5f1811b6bb762..c8df287d70228c 100644 --- a/git.c +++ b/git.c @@ -202,7 +202,7 @@ static int handle_options(const char ***argv, int *argc, int *envchanged) if (envchanged) *envchanged = 1; } else if (!strcmp(cmd, "--no-lazy-fetch")) { - fetch_if_missing = 0; + the_repository->fetch_if_missing = 0; setenv(NO_LAZY_FETCH_ENVIRONMENT, "1", 1); if (envchanged) *envchanged = 1; @@ -304,11 +304,15 @@ static int handle_options(const char ***argv, int *argc, int *envchanged) if (envchanged) *envchanged = 1; } else if (!strcmp(cmd, "--shallow-file")) { - (*argv)++; - (*argc)--; - setenv(GIT_SHALLOW_FILE_ENVIRONMENT, (*argv)[0], 1); + if (*argc < 2) { + fprintf(stderr, _("no file given for '%s' option\n" ), "--shallow-file"); + usage(git_usage_string); + } + setenv(GIT_SHALLOW_FILE_ENVIRONMENT, (*argv)[1], 1); if (envchanged) *envchanged = 1; + (*argv)++; + (*argc)--; } else if (!strcmp(cmd, "-C")) { if (*argc < 2) { fprintf(stderr, _("no directory given for '%s' option\n" ), "-C"); @@ -566,6 +570,7 @@ static struct cmd_struct commands[] = { { "diagnose", cmd_diagnose, RUN_SETUP_GENTLY }, { "diff", cmd_diff, NO_PARSEOPT }, { "diff-files", cmd_diff_files, RUN_SETUP | NEED_WORK_TREE | NO_PARSEOPT }, + { "diff-hunks", cmd_diff_hunks, RUN_SETUP }, { "diff-index", cmd_diff_index, RUN_SETUP | NO_PARSEOPT }, { "diff-pairs", cmd_diff_pairs, RUN_SETUP | NO_PARSEOPT }, { "diff-tree", cmd_diff_tree, RUN_SETUP | NO_PARSEOPT }, diff --git a/hex-ll.c b/hex-ll.c index 4d7ece1de5ed7c..b2e9684693057e 100644 --- a/hex-ll.c +++ b/hex-ll.c @@ -36,10 +36,45 @@ const signed char hexval_table[256] = { -1, -1, -1, -1, -1, -1, -1, -1, /* f8-ff */ }; -int hex_to_bytes(unsigned char *binary, const char *hex, size_t len) +const signed char hexval_lc_table[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, /* 00-07 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 08-0f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 10-17 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 18-1f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 20-27 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 28-2f */ + 0, 1, 2, 3, 4, 5, 6, 7, /* 30-37 */ + 8, 9, -1, -1, -1, -1, -1, -1, /* 38-3f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 40-47 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 48-4f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 50-57 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 58-5f */ + -1, 10, 11, 12, 13, 14, 15, -1, /* 60-67 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 68-67 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 70-77 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 78-7f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 80-87 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 88-8f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 90-97 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 98-9f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* a0-a7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* a8-af */ + -1, -1, -1, -1, -1, -1, -1, -1, /* b0-b7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* b8-bf */ + -1, -1, -1, -1, -1, -1, -1, -1, /* c0-c7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* c8-cf */ + -1, -1, -1, -1, -1, -1, -1, -1, /* d0-d7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* d8-df */ + -1, -1, -1, -1, -1, -1, -1, -1, /* e0-e7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* e8-ef */ + -1, -1, -1, -1, -1, -1, -1, -1, /* f0-f7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* f8-ff */ +}; + +int hex_to_bytes(unsigned char *binary, const char *hex, size_t len, enum hexkind kind) { for (; len; len--, hex += 2) { - unsigned int val = (hexval(hex[0]) << 4) | hexval(hex[1]); + unsigned int val = (hexval(hex[0], kind) << 4) | hexval(hex[1], kind); if (val & ~0xff) return -1; diff --git a/hex-ll.h b/hex-ll.h index a381fa85562234..2f9c8d7c25fb73 100644 --- a/hex-ll.h +++ b/hex-ll.h @@ -1,20 +1,32 @@ #ifndef HEX_LL_H #define HEX_LL_H +enum hexkind { + HEX_KIND_MIXED = 0, + HEX_KIND_LOWER = 1, +}; + +#ifdef WITH_BREAKING_CHANGES +#define HEX_KIND_OID HEX_KIND_LOWER +#else +#define HEX_KIND_OID HEX_KIND_MIXED +#endif + extern const signed char hexval_table[256]; -static inline unsigned int hexval(unsigned char c) +extern const signed char hexval_lc_table[256]; +static inline unsigned int hexval(unsigned char c, enum hexkind kind) { - return hexval_table[c]; + return kind == HEX_KIND_MIXED ? hexval_table[c] : hexval_lc_table[c]; } /* * Convert two consecutive hexadecimal digits into a char. Return a * negative value on error. Don't run over the end of short strings. */ -static inline int hex2chr(const char *s) +static inline int hex2chr(const char *s, enum hexkind kind) { - unsigned int val = hexval(s[0]); - return (val & ~0xf) ? val : (val << 4) | hexval(s[1]); + unsigned int val = hexval(s[0], kind); + return (val & ~0xf) ? val : (val << 4) | hexval(s[1], kind); } /* @@ -22,6 +34,6 @@ static inline int hex2chr(const char *s) * values to `binary` as `len` bytes. Return 0 on success, or -1 if * the input does not consist of hex digits). */ -int hex_to_bytes(unsigned char *binary, const char *hex, size_t len); +int hex_to_bytes(unsigned char *binary, const char *hex, size_t len, enum hexkind kind); #endif diff --git a/hex.c b/hex.c index f02832140d2d43..4e1e81af3f476b 100644 --- a/hex.c +++ b/hex.c @@ -9,7 +9,7 @@ static int get_hash_hex_algop(const char *hex, unsigned char *hash, const struct git_hash_algo *algop) { for (size_t i = 0; i < algop->rawsz; i++) { - int val = hex2chr(hex); + int val = hex2chr(hex, HEX_KIND_OID); if (val < 0) return -1; *hash++ = val; diff --git a/http-push.c b/http-push.c index 786a2e9c0d0546..43b4b61c701b2f 100644 --- a/http-push.c +++ b/http-push.c @@ -367,7 +367,7 @@ static void start_put(struct transfer_request *request) void *unpacked; size_t len; int hdrlen; - ssize_t size; + size_t size; git_zstream stream; struct repo_config_values *cfg = repo_config_values(the_repository); @@ -1031,12 +1031,13 @@ static int get_oid_hex_from_objpath(const char *path, struct object_id *oid) if (strlen(path) != the_hash_algo->hexsz + 1) return -1; - if (hex_to_bytes(oid->hash, path, 1)) + if (hex_to_bytes(oid->hash, path, 1, HEX_KIND_OID)) return -1; path += 2; path++; /* skip '/' */ - return hex_to_bytes(oid->hash + 1, path, the_hash_algo->rawsz - 1); + return hex_to_bytes(oid->hash + 1, path, the_hash_algo->rawsz - 1, + HEX_KIND_OID); } static void process_ls_object(struct remote_ls_ctx *ls) diff --git a/http.c b/http.c index a0d399b2745ae6..26bdd028ea1410 100644 --- a/http.c +++ b/http.c @@ -44,6 +44,7 @@ static CURL *curl_default; char curl_errorstr[CURL_ERROR_SIZE]; static int curl_ssl_verify = -1; +static int curl_ssl_verify_status; static int curl_ssl_try; static char *curl_http_version; static char *ssl_cert; @@ -400,6 +401,10 @@ static int http_options(const char *var, const char *value, curl_ssl_verify = git_config_bool(var, value); return 0; } + if (!strcmp("http.sslverifystatus", var)) { + curl_ssl_verify_status = git_config_bool(var, value); + return 0; + } if (!strcmp("http.sslcipherlist", var)) return git_config_string(&ssl_cipherlist, var, value); if (!strcmp("http.sslversion", var)) @@ -1133,6 +1138,11 @@ static CURL *get_curl_handle(void) curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2L); } + if (curl_ssl_verify_status && + curl_easy_setopt(result, CURLOPT_SSL_VERIFYSTATUS, 1L) != CURLE_OK) + die(_("http.sslVerifyStatus is set, but the TLS backend of " + "this libcurl cannot verify certificate status")); + if (curl_http_version) { long opt; if (!get_curl_http_version_opt(curl_http_version, &opt)) { @@ -1608,6 +1618,8 @@ struct active_request_slot *get_active_slot(void) if (!slot->curl) { slot->curl = curl_easy_duphandle(curl_default); + if (!slot->curl) + die("curl_easy_duphandle failed"); curl_session_count++; } diff --git a/list-objects-filter.c b/list-objects-filter.c index c912ff3079a7d7..10e44f135780ba 100644 --- a/list-objects-filter.c +++ b/list-objects-filter.c @@ -828,3 +828,48 @@ void list_objects_filter__free(struct filter *filter) filter->free_fn(filter->filter_data); free(filter); } + +/* + * NEEDSWORK: this reimplements the blob:limit size check rather than + * reusing the existing filter machinery in + * list_objects_filter__filter_object(). That machinery is currently + * tied to the object-walk path and cannot easily be driven from a + * plain oidset. It would be nice to refactor the filter code so this + * helper can reuse it instead of duplicating the size check. + */ +int list_objects_filter__filter_oidset(struct repository *r, + const struct list_objects_filter_options *opts, + const struct oidset *in, + struct oidset *omitted) +{ + struct oidset_iter iter; + const struct object_id *oid; + + if (opts->choice != LOFC_BLOB_LIMIT) + return error(_("filter_oidset: only blob:limit filters are supported")); + + oidset_iter_init(in, &iter); + while ((oid = oidset_iter_next(&iter))) { + struct object_info info = OBJECT_INFO_INIT; + enum object_type type; + size_t size; + + info.typep = &type; + info.sizep = &size; + + /* + * Use OBJECT_INFO_SKIP_FETCH_OBJECT to avoid triggering + * a lazy fetch while inspecting candidates for removal. + */ + if (odb_read_object_info_extended(r->objects, oid, &info, + OBJECT_INFO_SKIP_FETCH_OBJECT) < 0) + continue; + + if (type != OBJ_BLOB) + continue; + + if (size >= opts->blob_limit_value) + oidset_insert(omitted, oid); + } + return 0; +} diff --git a/list-objects-filter.h b/list-objects-filter.h index 9e98814111cee4..5207ab70a9188e 100644 --- a/list-objects-filter.h +++ b/list-objects-filter.h @@ -94,4 +94,20 @@ enum list_objects_filter_result list_objects_filter__filter_object( */ void list_objects_filter__free(struct filter *filter); +/* + * Given a set of OIDs in 'in', populate 'omitted' with those that + * would be filtered by 'opts'. Currently only blob:limit=N is + * supported. Objects that cannot be read are silently skipped. + * + * NEEDSWORK: this reimplements the blob:limit size check rather than + * reusing the existing filter machinery. See the matching comment in + * list-objects-filter.c. + * + * Return 0 on success, -1 if the filter is not supported. + */ +int list_objects_filter__filter_oidset(struct repository *r, + const struct list_objects_filter_options *opts, + const struct oidset *in, + struct oidset *omitted); + #endif /* LIST_OBJECTS_FILTER_H */ diff --git a/loose.c b/loose.c index e97c3bb50af867..c159d29d2d2035 100644 --- a/loose.c +++ b/loose.c @@ -61,12 +61,17 @@ static int insert_loose_map(struct odb_source_loose *loose, return inserted; } -static int load_one_loose_object_map(struct repository *repo, struct odb_source_loose *loose) +int loose_object_map_load(struct odb_source_loose *loose) { - struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT; + struct repository *repo = loose->base.odb->repo; + struct strbuf buf = STRBUF_INIT; + char *path; FILE *fp; int ret = -1; + if (!should_use_loose_object_map(repo)) + return 0; + if (!loose->map) loose_object_map_init(&loose->map); if (!loose->cache) { @@ -78,10 +83,10 @@ static int load_one_loose_object_map(struct repository *repo, struct odb_source_ insert_loose_map(loose, repo->hash_algo->empty_blob, repo->compat_hash_algo->empty_blob); insert_loose_map(loose, repo->hash_algo->null_oid, repo->compat_hash_algo->null_oid); - repo_common_path_replace(repo, &path, "objects/loose-object-idx"); - fp = fopen(path.buf, "rb"); + path = xstrfmt("%s/loose-object-idx", loose->base.path); + fp = fopen(path, "rb"); if (!fp) { - strbuf_release(&path); + free(path); return 0; } @@ -102,7 +107,7 @@ static int load_one_loose_object_map(struct repository *repo, struct odb_source_ err: fclose(fp); strbuf_release(&buf); - strbuf_release(&path); + free(path); return ret; } @@ -110,17 +115,12 @@ int repo_read_loose_object_map(struct repository *repo) { struct odb_source *source; - if (!should_use_loose_object_map(repo)) - return 0; - - odb_prepare_alternates(repo->objects); - for (source = repo->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - if (load_one_loose_object_map(repo, files->loose) < 0) { + if (loose_object_map_load(files->loose) < 0) return -1; - } } + return 0; } diff --git a/loose.h b/loose.h index 6c9b3f4571602f..ed663ac550fbb7 100644 --- a/loose.h +++ b/loose.h @@ -13,6 +13,7 @@ struct loose_object_map { void loose_object_map_init(struct loose_object_map **map); void loose_object_map_clear(struct loose_object_map **map); +int loose_object_map_load(struct odb_source_loose *loose); int repo_loose_object_map_oid(struct repository *repo, const struct object_id *src, const struct git_hash_algo *dest_algo, diff --git a/mailinfo.c b/mailinfo.c index 13949ff31e1769..85c3119048fe5e 100644 --- a/mailinfo.c +++ b/mailinfo.c @@ -396,7 +396,7 @@ static int decode_q_segment(struct strbuf *out, const struct strbuf *q_seg, int ch, d = *in; if (d == '\n' || !d) break; /* drop trailing newline */ - ch = hex2chr(in); + ch = hex2chr(in, HEX_KIND_MIXED); if (ch >= 0) { strbuf_addch(out, ch); in += 2; diff --git a/meson.build b/meson.build index 7073d5844d2531..e0b64fa0790cbb 100644 --- a/meson.build +++ b/meson.build @@ -356,6 +356,8 @@ libgit_sources = [ 'diff-merges.c', 'diff-lib.c', 'diff-no-index.c', + 'diff-process.c', + 'diff-provider.c', 'diff.c', 'diffcore-break.c', 'diffcore-delta.c', @@ -363,6 +365,7 @@ libgit_sources = [ 'diffcore-pickaxe.c', 'diffcore-rename.c', 'diffcore-rotate.c', + 'diff-hunks.c', 'dir-iterator.c', 'dir.c', 'editor.c', @@ -633,6 +636,7 @@ builtin_sources = [ 'builtin/describe.c', 'builtin/diagnose.c', 'builtin/diff-files.c', + 'builtin/diff-hunks.c', 'builtin/diff-index.c', 'builtin/diff-pairs.c', 'builtin/diff-tree.c', @@ -1318,7 +1322,6 @@ elif host_machine.system() == 'windows' libgit_c_args += [ '-DDETECT_MSYS_TTY', - '-DENSURE_MSYSTEM_IS_SET', '-DNATIVE_CRLF', '-DNOGDI', '-DNO_POSIX_GOODIES', @@ -1328,6 +1331,20 @@ elif host_machine.system() == 'windows' '-D__USE_MINGW_ANSI_STDIO=0', ] + msystem = get_option('msystem') + if msystem != '' + mingw_prefix = get_option('mingw_prefix') + if mingw_prefix == '' + mingw_prefix = msystem.to_lower() + elif mingw_prefix.startswith('/') + mingw_prefix = mingw_prefix.substring(1) + endif + libgit_c_args += [ + '-DENSURE_MSYSTEM_IS_SET="' + msystem + '"', + '-DMINGW_PREFIX="' + mingw_prefix + '"' + ] + endif + libgit_dependencies += compiler.find_library('ntdll') libgit_include_directories += 'compat/win32' if compiler.get_id() == 'msvc' @@ -1407,6 +1424,11 @@ if not get_option('b_sanitize').contains('address') and get_option('regex').allo libgit_c_args += '-DUSE_ENHANCED_BASIC_REGULAR_EXPRESSIONS' compat_sources += 'compat/regcomp_enhanced.c' endif + + if host_machine.system() == 'darwin' + libgit_c_args += '-DDARWIN_REGEXEC' + compat_sources += 'compat/darwin/regexec.c' + endif elif not get_option('regex').enabled() libgit_c_args += [ '-DNO_REGEX', @@ -1464,6 +1486,7 @@ checkfuncs = { 'initgroups' : [], 'strtoumax' : ['strtoumax.c', 'strtoimax.c'], 'pread' : ['pread.c'], + 'writev' : ['writev.c'], } if host_machine.system() == 'windows' diff --git a/meson_options.txt b/meson_options.txt index dc88f130d754a1..5fecb97dafc49b 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -21,6 +21,10 @@ option('runtime_prefix', type: 'boolean', value: false, description: 'Resolve ancillary tooling and support files relative to the location of the runtime binary instead of hard-coding them into the binary.') option('sane_tool_path', type: 'array', value: [], description: 'An array of paths to pick up tools from in case the normal tools are broken or lacking.') +option('msystem', type: 'string', value: '', + description: 'Fall-back on Windows when MSYSTEM is not set.') +option('mingw_prefix', type: 'string', value: '', + description: 'Fall-back on Windows when MINGW_PREFIX is not set.') # Build information compiled into Git and other parts like documentation. option('build_date', type: 'string', value: '', @@ -111,6 +115,8 @@ option('default_help_format', type: 'combo', choices: ['man', 'html', 'platform' description: 'Default format used when executing git-help(1).') option('docs_backend', type: 'combo', choices: ['asciidoc', 'asciidoctor', 'auto'], value: 'auto', description: 'Which backend to use to generate documentation.') +option('man_base_url', type: 'string', value: '', + description: 'The base URL to use for relative links in manuals') # Testing. option('benchmarks', type: 'feature', value: 'auto', diff --git a/midx-write.c b/midx-write.c index 580724d21a677f..fa7254e8244eb2 100644 --- a/midx-write.c +++ b/midx-write.c @@ -133,8 +133,17 @@ static uint32_t midx_pack_perm(struct write_midx_context *ctx, static int should_include_pack(const struct write_midx_context *ctx, const char *file_name) { + struct multi_pack_index *m = ctx->m; /* - * Note that at most one of ctx->m and ctx->to_include are set, + * When writing incrementally, ctx->m may contain layers above + * the selected base MIDX, which must be included in the new + * layer. + */ + if (ctx->incremental) + m = ctx->base_midx; + + /* + * Note that at most one of m and ctx->to_include are set, * so we are testing midx_contains_pack() and * string_list_has_string() independently (guarded by the * appropriate NULL checks). @@ -148,10 +157,7 @@ static int should_include_pack(const struct write_midx_context *ctx, * should be performed independently (likely checking * to_include before the existing MIDX). */ - if (ctx->m && midx_contains_pack(ctx->m, file_name)) - return 0; - else if (ctx->base_midx && midx_contains_pack(ctx->base_midx, - file_name)) + if (m && midx_contains_pack(m, file_name)) return 0; else if (ctx->to_include && !string_list_has_string(ctx->to_include, file_name)) @@ -865,7 +871,7 @@ static void find_commits_for_midx_bitmap(struct commit_stack *commits, * complain later that we don't have reachability closure (and fail * appropriately). */ - fetch_if_missing = 0; + ctx->repo->fetch_if_missing = 0; revs.exclude_promisor_objects = 1; if (prepare_revision_walk(&revs)) @@ -1851,12 +1857,14 @@ static int write_midx_internal(struct write_midx_opts *opts) int write_midx_file(struct odb_source_packed *source, const char *preferred_pack_name, const char *refs_snapshot, + const char *incremental_base, unsigned flags) { struct write_midx_opts opts = { .source = source, .preferred_pack_name = preferred_pack_name, .refs_snapshot = refs_snapshot, + .incremental_base = incremental_base, .flags = flags, }; diff --git a/midx.c b/midx.c index 76c3f92cc374b8..05fe99f8cad4bd 100644 --- a/midx.c +++ b/midx.c @@ -906,6 +906,48 @@ static int compare_pair_pos_vs_id(const void *_a, const void *_b) return b->pack_int_id - a->pack_int_id; } +/* + * Return whether the pack index contains an entry with both "oid" and + * "offset". A pack index may contain duplicate OIDs, so an arbitrary + * OID lookup is not enough to validate a particular offset. + * + * Do not use offset_to_pack_pos() here: it may consult an optional '.rev' + * file, which is verified separately, or build an in-memory reverse index + * that remains attached to the pack. Verify the MIDX directly against its + * source pack index instead. + */ +static int pack_index_has_oid_at_offset(struct packed_git *p, + const struct object_id *oid, + off_t offset) +{ + struct object_id candidate; + uint32_t pos, i; + + if (!bsearch_pack(oid, p, &pos)) + return 0; + + if (nth_packed_object_offset(p, pos) == offset) + return 1; + + for (i = pos; i > 0; i--) { + if (nth_packed_object_id(&candidate, p, i - 1) || + !oideq(&candidate, oid)) + break; + if (nth_packed_object_offset(p, i - 1) == offset) + return 1; + } + + for (i = pos + 1; i < p->num_objects; i++) { + if (nth_packed_object_id(&candidate, p, i) || + !oideq(&candidate, oid)) + break; + if (nth_packed_object_offset(p, i) == offset) + return 1; + } + + return 0; +} + /* * Limit calls to display_progress() for performance reasons. * The interval here was arbitrarily chosen. @@ -1015,7 +1057,6 @@ int verify_midx_file(struct odb_source_packed *source, unsigned flags) for (i = 0; i < m->num_objects + m->num_objects_in_base; i++) { struct object_id oid; struct pack_entry e; - off_t m_offset, p_offset; if (i > 0 && pairs[i-1].pack_int_id != pairs[i].pack_int_id && nth_midxed_pack(m, pairs[i-1].pack_int_id)) { @@ -1040,12 +1081,16 @@ int verify_midx_file(struct odb_source_packed *source, unsigned flags) break; } - m_offset = e.offset; - p_offset = find_pack_entry_one(&oid, e.p); - - if (m_offset != p_offset) - midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64), - pairs[i].pos, oid_to_hex(&oid), m_offset, p_offset); + /* + * Check that the exact offset recorded in the MIDX + * belongs to this OID. A pack index may contain + * duplicate OIDs, in which case an arbitrary OID lookup + * can return a different, equally valid copy than the + * one selected by the MIDX writer. + */ + if (!pack_index_has_oid_at_offset(e.p, &oid, e.offset)) + midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64), + pairs[i].pos, oid_to_hex(&oid), e.offset); midx_display_sparse_progress(progress, i + 1); } diff --git a/midx.h b/midx.h index 939c18e5885e6f..e35db74905a5da 100644 --- a/midx.h +++ b/midx.h @@ -131,7 +131,7 @@ int prepare_multi_pack_index_one(struct odb_source_packed *source); */ int write_midx_file(struct odb_source_packed *source, const char *preferred_pack_name, const char *refs_snapshot, - unsigned flags); + const char *incremental_base, unsigned flags); int write_midx_file_only(struct odb_source_packed *source, struct string_list *packs_to_include, const char *preferred_pack_name, diff --git a/notes.c b/notes.c index ec9c2cb150d4e3..7e9e3eb2d2cb27 100644 --- a/notes.c +++ b/notes.c @@ -428,7 +428,7 @@ static void load_subtree(struct notes_tree *t, struct leaf_node *subtree, goto handle_non_note; if (hex_to_bytes(object_oid.hash + prefix_len, entry.path, - hashsz - prefix_len)) + hashsz - prefix_len, HEX_KIND_MIXED)) goto handle_non_note; /* entry.path is not a SHA1 */ memset(object_oid.hash + hashsz, 0, GIT_MAX_RAWSZ - hashsz); @@ -442,7 +442,8 @@ static void load_subtree(struct notes_tree *t, struct leaf_node *subtree, /* internal nodes must be trees */ goto handle_non_note; - if (hex_to_bytes(object_oid.hash + len++, entry.path, 1)) + if (hex_to_bytes(object_oid.hash + len++, entry.path, 1, + HEX_KIND_OID)) goto handle_non_note; /* entry.path is not a SHA1 */ /* diff --git a/object-file.c b/object-file.c index ec35c318bc9fe7..08fa8086e3baf8 100644 --- a/object-file.c +++ b/object-file.c @@ -10,6 +10,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "config.h" #include "convert.h" #include "dir.h" #include "environment.h" @@ -26,6 +27,7 @@ #include "packfile.h" #include "path.h" #include "read-cache-ll.h" +#include "run-command.h" #include "setup.h" #include "strvec.h" #include "tempfile.h" @@ -122,7 +124,7 @@ int check_object_signature(struct repository *r, const struct object_id *oid, } int stream_object_signature(struct repository *r, - struct odb_read_stream *st, + struct odb_stream *st, const struct object_id *oid) { struct object_id real_oid; @@ -138,7 +140,7 @@ int stream_object_signature(struct repository *r, git_hash_update(&c, hdr, hdrlen); for (;;) { char buf[1024 * 16]; - ssize_t readlen = odb_read_stream_read(st, buf, sizeof(buf)); + ssize_t readlen = odb_stream_read(st, buf, sizeof(buf)); if (readlen < 0) return -1; if (!readlen) @@ -485,8 +487,13 @@ struct odb_transaction_files { struct odb_transaction base; struct tmp_objdir *objdir; + struct odb_source *quarantine; struct transaction_packfile packfile; const char *prefix; + + struct tempfile **pack_lockfiles; + size_t pack_lockfiles_nr; + size_t pack_lockfiles_alloc; }; int odb_transaction_files_prepare(struct odb_transaction *base) @@ -507,7 +514,7 @@ int odb_transaction_files_prepare(struct odb_transaction *base) if (!transaction->objdir) return error(_("unable to create temporary object directory")); - tmp_objdir_replace_primary_odb(transaction->objdir, 0); + transaction->quarantine = tmp_objdir_replace_primary_odb(transaction->objdir, 0); return 0; } @@ -702,9 +709,9 @@ static void prepare_packfile_transaction(struct odb_transaction_files *transacti die_errno("unable to write pack header"); } -static int hash_blob_stream(struct odb_write_stream *stream, - const struct git_hash_algo *hash_algo, - struct object_id *result_oid, size_t size) +static int hash_stream(struct odb_stream *stream, + const struct git_hash_algo *hash_algo, + struct object_id *result_oid) { unsigned char buf[16384]; struct git_hash_ctx ctx; @@ -712,22 +719,23 @@ static int hash_blob_stream(struct odb_write_stream *stream, size_t bytes_hashed = 0; header_len = format_object_header((char *)buf, sizeof(buf), - OBJ_BLOB, size); + stream->type, stream->size); git_hash_init(&ctx, hash_algo); git_hash_update(&ctx, buf, header_len); - while (!stream->is_finished) { - ssize_t read_result = odb_write_stream_read(stream, buf, - sizeof(buf)); - + while (1) { + ssize_t read_result = odb_stream_read(stream, buf, + sizeof(buf)); if (read_result < 0) return -1; + if (!read_result) + break; git_hash_update(&ctx, buf, read_result); bytes_hashed += read_result; } - if (bytes_hashed != size) + if (bytes_hashed != stream->size) return -1; git_hash_final_oid(result_oid, &ctx); @@ -739,9 +747,9 @@ static int hash_blob_stream(struct odb_write_stream *stream, * Read the contents from the stream provided, streaming it to the * packfile in state while updating the hash in ctx. */ -static void stream_blob_to_pack(struct transaction_packfile *state, - struct git_hash_ctx *ctx, size_t size, - struct odb_write_stream *stream) +static void stream_to_pack(struct transaction_packfile *state, + struct git_hash_ctx *ctx, + struct odb_stream *stream) { git_zstream s; unsigned char ibuf[16384]; @@ -749,21 +757,23 @@ static void stream_blob_to_pack(struct transaction_packfile *state, unsigned hdrlen; int status = Z_OK; struct repo_config_values *cfg = repo_config_values(the_repository); + bool is_finished = false; size_t bytes_read = 0; git_deflate_init(&s, cfg->pack_compression_level); - hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), OBJ_BLOB, size); + hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), stream->type, stream->size); s.next_out = obuf + hdrlen; s.avail_out = sizeof(obuf) - hdrlen; while (status != Z_STREAM_END) { - if (!stream->is_finished && !s.avail_in) { - ssize_t rsize = odb_write_stream_read(stream, ibuf, - sizeof(ibuf)); - + if (!is_finished && !s.avail_in) { + ssize_t rsize = odb_stream_read(stream, ibuf, + sizeof(ibuf)); if (rsize < 0) - die("failed to read blob data"); + die("failed to read object data"); + if (!rsize) + is_finished = true; git_hash_update(ctx, ibuf, rsize); @@ -772,7 +782,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state, bytes_read += rsize; } - status = git_deflate(&s, stream->is_finished ? Z_FINISH : 0); + status = git_deflate(&s, is_finished ? Z_FINISH : 0); if (!s.avail_out || status == Z_STREAM_END) { size_t written = s.next_out - obuf; @@ -793,9 +803,9 @@ static void stream_blob_to_pack(struct transaction_packfile *state, } } - if (bytes_read != size) - die("read %" PRIuMAX " bytes of blob data, but expected %" PRIuMAX " bytes", - (uintmax_t)bytes_read, (uintmax_t)size); + if (bytes_read != stream->size) + die("read %" PRIuMAX " bytes of object data, but expected %" PRIuMAX " bytes", + (uintmax_t)bytes_read, (uintmax_t)stream->size); git_deflate_end(&s); } @@ -865,12 +875,11 @@ static void flush_packfile_transaction(struct odb_transaction_files *transaction * result, which we need to know beforehand when writing a git object. * Since the primary motivation for trying to stream from the working * tree file and to avoid mmaping it in core is to deal with large - * binary blobs, they generally do not want to get any conversion, and + * objects, they generally do not want to get any conversion, and * callers should avoid this code path when filters are requested. */ static int odb_transaction_files_write_object_stream(struct odb_transaction *base, - struct odb_write_stream *stream, - size_t size, + struct odb_stream *stream, struct object_id *result_oid) { struct odb_transaction_files *transaction = container_of(base, @@ -884,7 +893,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas struct pack_idx_entry *idx; header_len = format_object_header((char *)obuf, sizeof(obuf), - OBJ_BLOB, size); + stream->type, stream->size); git_hash_init(&ctx, transaction->base.source->odb->repo->hash_algo); git_hash_update(&ctx, obuf, header_len); @@ -899,7 +908,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas * to zlib compression and is sufficient for this check. */ if (state->nr_written && pack_size_limit_cfg && - pack_size_limit_cfg < state->offset + size) + pack_size_limit_cfg < state->offset + stream->size) flush_packfile_transaction(transaction); CALLOC_ARRAY(idx, 1); @@ -909,7 +918,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas hashfile_checkpoint(state->f, &checkpoint); idx->offset = state->offset; crc32_begin(state->f); - stream_blob_to_pack(state, &ctx, size, stream); + stream_to_pack(state, &ctx, stream); git_hash_final_oid(result_oid, &ctx); idx->crc32 = crc32_end(state->f); @@ -950,8 +959,8 @@ int index_fd(struct index_state *istate, struct object_id *oid, ret = index_core(istate, oid, fd, xsize_t(st->st_size), type, path, flags); } else { - struct odb_write_stream stream; - odb_write_stream_from_fd(&stream, fd, xsize_t(st->st_size)); + struct odb_stream *stream = odb_stream_from_fd(fd, xsize_t(st->st_size), + OBJ_BLOB); if (flags & INDEX_WRITE_OBJECT) { struct object_database *odb = the_repository->objects; @@ -961,18 +970,14 @@ int index_fd(struct index_state *istate, struct object_id *oid, if (!inflight) odb_transaction_begin_or_die(odb, &transaction, 0); ret = odb_transaction_write_object_stream(transaction, - &stream, - xsize_t(st->st_size), - oid); + stream, oid); if (!inflight) - odb_transaction_commit(transaction); + odb_transaction_commit_and_finalize_or_die(transaction); } else { - ret = hash_blob_stream(&stream, - the_repository->hash_algo, oid, - xsize_t(st->st_size)); + ret = hash_stream(stream, the_repository->hash_algo, oid); } - odb_write_stream_release(&stream); + odb_stream_close(stream); } close(fd); @@ -1073,7 +1078,7 @@ int for_each_file_in_obj_subdir(unsigned int subdir_nr, strbuf_add(path, de->d_name, namelen); if (namelen == algop->hexsz - 2 && !hex_to_bytes(oid.hash + 1, de->d_name, - algop->rawsz - 1)) { + algop->rawsz - 1, HEX_KIND_OID)) { oid_set_algo(&oid, algop); memset(oid.hash + algop->rawsz, 0, GIT_MAX_RAWSZ - algop->rawsz); @@ -1290,6 +1295,170 @@ static int odb_transaction_files_commit(struct odb_transaction *base) return 0; } +static const char *parse_pack_header(struct pack_header *hdr, int pack_fd) +{ + switch (read_pack_header(pack_fd, hdr)) { + case PH_ERROR_EOF: + return "eof before pack header was fully read"; + + case PH_ERROR_PACK_SIGNATURE: + return "protocol error (pack signature mismatch detected)"; + + case PH_ERROR_PROTOCOL: + return "protocol error (pack version unsupported)"; + + default: + return "unknown error in parse_pack_header"; + + case 0: + return NULL; + } +} + +static void push_header_arg(struct strvec *args, struct pack_header *hdr) +{ + strvec_pushf(args, "--pack_header=%"PRIu32",%"PRIu32, + ntohl(hdr->hdr_version), ntohl(hdr->hdr_entries)); +} + +static unsigned int get_unpack_limit(struct repository *repo) +{ + unsigned int limit = 100; + + repo_config_get_uint(repo, "transfer.unpacklimit", &limit); + repo_config_get_uint(repo, "receive.unpacklimit", &limit); + + return limit; +} + +static int odb_transaction_files_write_pack(struct odb_transaction *base, + int pack_fd, struct strbuf *err_msg, + const struct odb_transaction_write_pack_opts *opts) +{ + struct odb_transaction_files *transaction = + container_of(base, struct odb_transaction_files, base); + struct repository *repo = base->source->odb->repo; + struct child_process child = CHILD_PROCESS_INIT; + struct pack_header hdr; + const char *hdr_err; + int err_fd = opts->err_fd; + int status; + + hdr_err = parse_pack_header(&hdr, pack_fd); + if (hdr_err) { + if (err_fd > 0) + close(err_fd); + strbuf_addstr(err_msg, hdr_err); + return -1; + } + + if (opts->shallow_file) { + strvec_push(&child.args, "--shallow-file"); + strvec_push(&child.args, opts->shallow_file); + } + + odb_transaction_env(base, &child.env); + + if (ntohl(hdr.hdr_entries) < get_unpack_limit(repo)) { + strvec_push(&child.args, "unpack-objects"); + push_header_arg(&child.args, &hdr); + if (opts->quiet) + strvec_push(&child.args, "-q"); + if (opts->fsck_objects) + strvec_pushf(&child.args, "--strict%s", + opts->fsck_msg_types); + if (opts->max_input_size) + strvec_pushf(&child.args, "--max-input-size=%"PRIuMAX, + (uintmax_t)opts->max_input_size); + child.no_stdout = 1; + child.in = pack_fd; + child.err = err_fd; + child.git_cmd = 1; + status = run_command(&child); + if (status) { + strbuf_addstr(err_msg, "unpack-objects abnormal exit"); + return -1; + } + } else { + char hostname[HOST_NAME_MAX + 1]; + char *lockfile; + + strvec_pushl(&child.args, "index-pack", "--stdin", NULL); + push_header_arg(&child.args, &hdr); + + if (xgethostname(hostname, sizeof(hostname))) + xsnprintf(hostname, sizeof(hostname), "localhost"); + strvec_pushf(&child.args, + "--keep=receive-pack %"PRIuMAX" on %s", + (uintmax_t)getpid(), + hostname); + + if (!opts->quiet && err_fd) + strvec_push(&child.args, "--show-resolving-progress"); + if (err_fd) + strvec_push(&child.args, "--report-end-of-input"); + if (opts->fsck_objects) + strvec_pushf(&child.args, "--strict%s", + opts->fsck_msg_types); + if (!opts->reject_thin) + strvec_push(&child.args, "--fix-thin"); + if (opts->max_input_size) + strvec_pushf(&child.args, "--max-input-size=%"PRIuMAX, + (uintmax_t)opts->max_input_size); + child.out = -1; + child.in = pack_fd; + child.err = err_fd; + child.git_cmd = 1; + status = start_command(&child); + if (status) { + strbuf_addstr(err_msg, "index-pack fork failed"); + return -1; + } + + /* + * The lockfile filepath is expected to be the final location of + * the ".keep" file after being migrated to the main ODB source. + * This ensures the lockfile can be found and removed later + * after the ODB transaction has been committed. + */ + lockfile = index_pack_lockfile(base->source, child.out, NULL); + if (lockfile) { + ALLOC_GROW(transaction->pack_lockfiles, + transaction->pack_lockfiles_nr + 1, + transaction->pack_lockfiles_alloc); + transaction->pack_lockfiles[transaction->pack_lockfiles_nr++] = + register_tempfile(lockfile); + free(lockfile); + } + close(child.out); + + status = finish_command(&child); + if (status) { + strbuf_addstr(err_msg, "index-pack abnormal exit"); + return -1; + } + + odb_source_prepare(transaction->quarantine, + ODB_PREPARE_FLUSH_CACHES); + } + + return 0; +} + +static int odb_transaction_files_finalize(struct odb_transaction *base) +{ + struct odb_transaction_files *transaction = + container_of(base, struct odb_transaction_files, base); + int ret = 0; + + for (size_t i = 0; i < transaction->pack_lockfiles_nr; i++) + ret |= delete_tempfile(&transaction->pack_lockfiles[i]); + + free(transaction->pack_lockfiles); + + return ret; +} + static int odb_transaction_files_env(struct odb_transaction *base, struct strvec *env) { @@ -1313,7 +1482,9 @@ int odb_transaction_files_begin(struct odb_source *source, transaction = xcalloc(1, sizeof(*transaction)); transaction->base.source = source; transaction->base.commit = odb_transaction_files_commit; + transaction->base.finalize = odb_transaction_files_finalize; transaction->base.write_object_stream = odb_transaction_files_write_object_stream; + transaction->base.write_pack = odb_transaction_files_write_pack; transaction->base.env = odb_transaction_files_env; transaction->prefix = "bulk-fsync"; @@ -1340,13 +1511,3 @@ int odb_transaction_files_begin(struct odb_source *source, return 0; } - -void free_object_info_contents(struct object_info *object_info) -{ - if (!object_info) - return; - free(object_info->typep); - free(object_info->sizep); - free(object_info->disk_sizep); - free(object_info->delta_base_oid); -} diff --git a/object-file.h b/object-file.h index 805f2cfa289661..f44758c4f8ba01 100644 --- a/object-file.h +++ b/object-file.h @@ -101,7 +101,7 @@ int check_object_signature(struct repository *r, const struct object_id *oid, * the streaming interface and rehash it to do the same. */ int stream_object_signature(struct repository *r, - struct odb_read_stream *stream, + struct odb_stream *stream, const struct object_id *oid); enum finalize_object_file_flags { diff --git a/object-name.c b/object-name.c index 83efba0ba668e5..d6da559e0e433f 100644 --- a/object-name.c +++ b/object-name.c @@ -236,17 +236,10 @@ static int parse_oid_prefix(const char *name, int len, { for (int i = 0; i < len; i++) { unsigned char c = name[i]; - unsigned char val; - if (c >= '0' && c <= '9') { - val = c - '0'; - } else if (c >= 'a' && c <= 'f') { - val = c - 'a' + 10; - } else if (c >= 'A' && c <='F') { - val = c - 'A' + 10; - c -= 'A' - 'a'; - } else { + int val = hexval(c, HEX_KIND_OID); + + if (val < 0) return -1; - } if (hex_out) hex_out[i] = c; @@ -280,7 +273,6 @@ static int init_object_disambiguation(struct repository *r, ds->len = len; ds->repo = r; - odb_prepare_alternates(r->objects); return 0; } @@ -1803,13 +1795,15 @@ static enum get_oid_result get_oid_with_context_1(struct repository *repo, memcmp(ce->name, cp, namelen)) break; if (ce_stage(ce) == stage) { + int ret = reject_tree_in_index(repo, only_to_die, ce, + stage, prefix, cp); + + if (!ret) { + oidcpy(oid, &ce->oid); + oc->mode = ce->ce_mode; + } free(new_path); - if (reject_tree_in_index(repo, only_to_die, ce, - stage, prefix, cp)) - return -1; - oidcpy(oid, &ce->oid); - oc->mode = ce->ce_mode; - return 0; + return ret; } pos++; } diff --git a/object.c b/object.c index 23b84aa7e29531..97f7fc0e87a1db 100644 --- a/object.c +++ b/object.c @@ -345,7 +345,7 @@ struct object *parse_object_with_flags(struct repository *r, if ((!obj || obj->type == OBJ_NONE || obj->type == OBJ_BLOB) && odb_read_object_info(r->objects, oid, NULL) == OBJ_BLOB) { if (!skip_hash) { - struct odb_read_stream *stream = odb_read_stream_open(r->objects, oid, NULL); + struct odb_stream *stream = odb_stream_from_object(r->objects, oid, NULL); if (!stream) { error(_("unable to open object stream for %s"), oid_to_hex(oid)); @@ -354,11 +354,11 @@ struct object *parse_object_with_flags(struct repository *r, if (stream_object_signature(r, stream, repl) < 0) { error(_("hash mismatch %s"), oid_to_hex(oid)); - odb_read_stream_close(stream); + odb_stream_close(stream); return NULL; } - odb_read_stream_close(stream); + odb_stream_close(stream); } parse_blob_buffer(lookup_blob(r, oid)); return lookup_object(r, oid); diff --git a/object.h b/object.h index 8fb03ff90a3e56..46cade33fb18fb 100644 --- a/object.h +++ b/object.h @@ -74,6 +74,7 @@ void object_array_init(struct object_array *array); * bisect.c: 16 * bundle.c: 16 * http-push.c: 11-----14 + * builtin/history.c: 11---13 * commit-graph.c: 15 * commit-reach.c: 16-------20 * builtin/last-modified.c: 1617 diff --git a/odb.c b/odb.c index caf1d0f5424d29..b2ddd06f9e1dfb 100644 --- a/odb.c +++ b/odb.c @@ -2,11 +2,11 @@ #include "abspath.h" #include "commit-graph.h" #include "config.h" -#include "dir.h" +#include "diff-hunks.h" #include "environment.h" #include "gettext.h" +#include "hashmap.h" #include "hex.h" -#include "khash.h" #include "lockfile.h" #include "loose.h" #include "midx.h" @@ -29,8 +29,47 @@ #include "trace2.h" #include "write-or-die.h" -KHASH_INIT(odb_path_map, const char * /* key: odb_path */, - struct odb_source *, 1, fspathhash, fspatheq) +/* + * NEEDSWORK: we're using "core.ignoreCase" to deduplicate alternates that + * _may_ be the same. This requires quite a bit of boilerplate for dubious + * benefit: + * + * - Duplicating alternates should really only lead to regressed performance. + * + * - We don't properly resolve symlinks or mointpoints, so we may still end + * up duplicating alternates. + * + * - The value may be lying, in which case we might deduplicate alternates + * that are in fact not mapping to the same directory. + * + * We should investigate whether we can remove this whole mechanism outright. + */ +static int odb_source_paths_cmp(struct object_database *o, + const char *a, const char *b) +{ + if (o->source_paths_icase < 0) { + int icase = 0; + repo_config_get_bool(o->repo, "core.ignorecase", &icase); + o->source_paths_icase = icase; + } + + return o->source_paths_icase ? strcasecmp(a, b) : strcmp(a, b); +} + +static int odb_source_by_path_cmp(const void *cb_data, + const struct hashmap_entry *entry, + const struct hashmap_entry *entry_or_key, + const void *keydata) +{ + struct object_database *o = (struct object_database *)cb_data; + const struct odb_source *source = container_of(entry, const struct odb_source, by_path_entry); + const char *path = keydata; + + if (!path) + path = container_of(entry_or_key, const struct odb_source, by_path_entry)->path; + + return odb_source_paths_cmp(o, source->path, path); +} int odb_mkstemp(struct object_database *odb, struct strbuf *temp_filename, const char *pattern) @@ -58,8 +97,8 @@ int odb_mkstemp(struct object_database *odb, */ static bool odb_is_source_usable(struct object_database *o, const char *path) { - int r; struct strbuf normalized_objdir = STRBUF_INIT; + struct hashmap_entry key; bool usable = false; strbuf_realpath(&normalized_objdir, o->sources->path, 1); @@ -76,20 +115,18 @@ static bool odb_is_source_usable(struct object_database *o, const char *path) * Prevent the common mistake of listing the same * thing twice, or object directory itself. */ - if (!o->source_by_path) { - khiter_t p; - - o->source_by_path = kh_init_odb_path_map(); + if (!hashmap_get_size(&o->source_by_path)) { assert(!o->sources->next); - p = kh_put_odb_path_map(o->source_by_path, o->sources->path, &r); - assert(r == 1); /* never used */ - kh_value(o->source_by_path, p) = o->sources; + hashmap_entry_init(&o->sources->by_path_entry, + strihash(o->sources->path)); + hashmap_add(&o->source_by_path, &o->sources->by_path_entry); } - if (fspatheq(path, normalized_objdir.buf)) + if (!odb_source_paths_cmp(o, path, normalized_objdir.buf)) goto out; - if (kh_get_odb_path_map(o->source_by_path, path) < kh_end(o->source_by_path)) + hashmap_entry_init(&key, strihash(path)); + if (hashmap_get(&o->source_by_path, &key, path)) goto out; usable = true; @@ -172,8 +209,6 @@ static struct odb_source *odb_add_alternate_recursively(struct object_database * { struct odb_source *alternate = NULL; struct strvec sources = STRVEC_INIT; - khiter_t pos; - int ret; if (!odb_is_source_usable(odb, source)) goto error; @@ -184,10 +219,11 @@ static struct odb_source *odb_add_alternate_recursively(struct object_database * *odb->sources_tail = alternate; odb->sources_tail = &(alternate->next); - pos = kh_put_odb_path_map(odb->source_by_path, alternate->path, &ret); - if (!ret) + hashmap_entry_init(&alternate->by_path_entry, strihash(alternate->path)); + if (hashmap_get(&odb->source_by_path, &alternate->by_path_entry, + alternate->path)) BUG("source must not yet exist"); - kh_value(odb->source_by_path, pos) = alternate; + hashmap_add(&odb->source_by_path, &alternate->by_path_entry); /* recursively add alternates */ odb_source_read_alternates(alternate, &sources); @@ -210,32 +246,21 @@ void odb_add_to_alternates_file(struct object_database *odb, int ret = odb_source_write_alternate(odb->sources, dir); if (ret < 0) die(NULL); - if (odb->loaded_alternates) - odb_add_alternate_recursively(odb, dir, 0); + odb_add_alternate_recursively(odb, dir, 0); } struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, const char *dir) { - /* - * Make sure alternates are initialized, or else our entry may be - * overwritten when they are. - */ - odb_prepare_alternates(odb); return odb_add_alternate_recursively(odb, dir, 0); } struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, - const char *dir, int will_destroy) + const char *dir, int will_destroy, + struct odb_source **prev_source) { struct odb_source *source; - /* - * Make sure alternates are initialized, or else our entry may be - * overwritten when they are. - */ - odb_prepare_alternates(odb); - /* * Make a new primary odb and link the old primary ODB in as an * alternate @@ -250,7 +275,11 @@ struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, source->will_destroy = will_destroy; source->next = odb->sources; odb->sources = source; - return source->next; + + if (prev_source) + *prev_source = source->next; + + return source; } void odb_restore_primary_source(struct object_database *odb, @@ -341,7 +370,6 @@ struct odb_source *odb_find_source(struct object_database *odb, const char *obj_ char *obj_dir_real = real_pathdup(obj_dir, 1); struct strbuf odb_path_real = STRBUF_INIT; - odb_prepare_alternates(odb); for (source = odb->sources; source; source = source->next) { strbuf_realpath(&odb_path_real, source->path, 1); if (!strcmp(obj_dir_real, odb_path_real.buf)) @@ -475,7 +503,6 @@ int odb_for_each_alternate(struct object_database *odb, struct odb_source *alternate; int r = 0; - odb_prepare_alternates(odb); for (alternate = odb->sources->next; alternate; alternate = alternate->next) { r = cb(alternate, payload); if (r) @@ -484,26 +511,22 @@ int odb_for_each_alternate(struct object_database *odb, return r; } -void odb_prepare_alternates(struct object_database *odb) +static void odb_prepare_alternates(struct object_database *odb, + const char *alternate_db) { struct strvec sources = STRVEC_INIT; - if (odb->loaded_alternates) - return; - - parse_alternates(odb->alternate_db, PATH_SEP, NULL, &sources); + parse_alternates(alternate_db, PATH_SEP, NULL, &sources); odb_source_read_alternates(odb->sources, &sources); + for (size_t i = 0; i < sources.nr; i++) odb_add_alternate_recursively(odb, sources.v[i], 0); - odb->loaded_alternates = 1; - strvec_clear(&sources); } int odb_has_alternates(struct object_database *odb) { - odb_prepare_alternates(odb); return !!odb->sources->next; } @@ -528,8 +551,6 @@ void disable_obj_read_lock(void) pthread_mutex_destroy(&obj_read_mutex); } -int fetch_if_missing = 1; - static int register_all_submodule_sources(struct object_database *odb) { int ret = odb->submodule_source_paths.nr; @@ -563,8 +584,6 @@ static int do_oid_object_info_extended(struct object_database *odb, if (!odb_source_read_object_info(odb->inmemory_objects, oid, oi, flags)) return 0; - odb_prepare_alternates(odb); - while (1) { struct odb_source *source; @@ -595,7 +614,7 @@ static int do_oid_object_info_extended(struct object_database *odb, continue; /* Check if it is a missing object */ - if (fetch_if_missing && repo_has_promisor_remote(odb->repo) && + if (odb->repo->fetch_if_missing && repo_has_promisor_remote(odb->repo) && !already_retried && !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { promisor_remote_get_direct(odb->repo, real, 1); @@ -827,7 +846,6 @@ int odb_freshen_object(struct object_database *odb, const struct object_id *oid) { struct odb_source *source; - odb_prepare_alternates(odb); for (source = odb->sources; source; source = source->next) if (odb_source_freshen_object(source, oid, NULL)) return 1; @@ -842,7 +860,6 @@ int odb_for_each_object_ext(struct object_database *odb, { int ret; - odb_prepare_alternates(odb); for (struct odb_source *source = odb->sources; source; source = source->next) { if (opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !source->local) continue; @@ -880,7 +897,6 @@ int odb_count_objects(struct object_database *odb, return 0; } - odb_prepare_alternates(odb); for (source = odb->sources; source; source = source->next) { unsigned long c; @@ -960,7 +976,6 @@ int odb_find_abbrev_len(struct object_database *odb, goto out; } - odb_prepare_alternates(odb); for (struct odb_source *source = odb->sources; source; source = source->next) { ret = odb_source_find_abbrev_len(source, oid, len, &len); if (ret) @@ -1028,10 +1043,10 @@ int odb_write_object_ext(struct object_database *odb, } int odb_write_object_stream(struct object_database *odb, - struct odb_write_stream *stream, size_t len, + struct odb_stream *stream, struct object_id *oid) { - return odb_source_write_object_stream(odb->sources, stream, len, oid); + return odb_source_write_object_stream(odb->sources, stream, oid); } int odb_optimize(struct object_database *odb, @@ -1046,27 +1061,55 @@ bool odb_optimize_required(struct object_database *odb, return odb_source_optimize_required(odb->sources, opts); } +void odb_generate_pack_options_release(struct odb_generate_pack_options *opts) +{ + oid_array_clear(&opts->wants); + oid_array_clear(&opts->haves); + oid_array_clear(&opts->shallows); +} + +int odb_generate_pack(struct object_database *odb, + struct odb_pack_generator **out, + const struct odb_generate_pack_options *opts) +{ + if (!odb->sources->generate_pack) + return error(_("primary object source does not support generating packfiles")); + return odb_source_generate_pack(odb->sources, out, opts); +} + +int odb_pack_generator_finish(struct odb_pack_generator *generator) +{ + return generator->finish(generator); +} + struct object_database *odb_new(struct repository *repo, - const char *primary_source, - const char *secondary_sources) + enum odb_new_flags flags) { - struct object_database *o = xmalloc(sizeof(*o)); - char *to_free = NULL; + char *primary_source = NULL, *secondary_sources = NULL; + struct object_database *o; - memset(o, 0, sizeof(*o)); + CALLOC_ARRAY(o, 1); o->repo = repo; pthread_mutex_init(&o->replace_mutex, NULL); string_list_init_dup(&o->submodule_source_paths); + hashmap_init(&o->source_by_path, odb_source_by_path_cmp, o, 0); + o->source_paths_icase = -1; + if (flags & ODB_NEW_HONOR_ENV) { + primary_source = xstrdup_or_null(getenv(DB_ENVIRONMENT)); + secondary_sources = xstrdup_or_null(getenv(ALTERNATE_DB_ENVIRONMENT)); + } if (!primary_source) - primary_source = to_free = xstrfmt("%s/objects", repo->commondir); + primary_source = xstrfmt("%s/objects", repo->commondir); + o->sources = odb_source_new(o, primary_source, true); o->sources_tail = &o->sources->next; - o->alternate_db = xstrdup_or_null(secondary_sources); o->inmemory_objects = &odb_source_inmemory_new(o)->base; - free(to_free); + odb_prepare_alternates(o, secondary_sources); + free(secondary_sources); + free(primary_source); return o; } @@ -1076,6 +1119,7 @@ void odb_close(struct object_database *o) for (source = o->sources; source; source = source->next) odb_source_close(source); close_commit_graph(o); + close_diff_hunks_store(o); } static void odb_free_sources(struct object_database *o) @@ -1091,8 +1135,7 @@ static void odb_free_sources(struct object_database *o) odb_source_free(o->inmemory_objects); o->inmemory_objects = NULL; - kh_destroy_odb_path_map(o->source_by_path); - o->source_by_path = NULL; + hashmap_clear(&o->source_by_path); } void odb_free(struct object_database *o) @@ -1100,8 +1143,6 @@ void odb_free(struct object_database *o) if (!o) return; - free(o->alternate_db); - oidmap_clear(&o->replace_map, 1); pthread_mutex_destroy(&o->replace_mutex); @@ -1123,14 +1164,14 @@ void odb_prepare(struct object_database *o, enum odb_prepare_flags flags) * Reprepare alt odbs, in case the alternates file was modified * during the course of this process. This only _adds_ odbs to * the linked list, so existing odbs will continue to exist for - * the lifetime of the process. + * the lifetime of the process. Consequently, we don't have to + * reprocess GIT_ALTERNATE_OBJECT_DIRECTORIES here. */ if (flags & ODB_PREPARE_FLUSH_CACHES) { - o->loaded_alternates = 0; + odb_prepare_alternates(o, NULL); o->object_count_valid = 0; } - odb_prepare_alternates(o); for (source = o->sources; source; source = source->next) odb_source_prepare(source, flags); diff --git a/odb.h b/odb.h index fca67e8253e7ad..ae62291dc9c4b3 100644 --- a/odb.h +++ b/odb.h @@ -1,13 +1,16 @@ #ifndef ODB_H #define ODB_H +#include "hashmap.h" #include "object.h" +#include "oid-array.h" #include "oidset.h" #include "oidmap.h" #include "string-list.h" #include "thread-utils.h" struct cached_object_entry; +struct diff_hunks_store; struct list_objects_filter_options; struct odb_source_inmemory; struct packed_git; @@ -15,14 +18,6 @@ struct repository; struct strbuf; struct strvec; -/* - * Set this to 0 to prevent odb_read_object_info_extended() from fetching missing - * blobs. This has a difference only if extensions.partialClone is set. - * - * Its default value is 1. - */ -extern int fetch_if_missing; - /* * Compute the exact path an alternate is at and returns it. In case of * error NULL is returned and the human readable error is added to `err` @@ -54,16 +49,19 @@ struct object_database { */ struct odb_source *sources; struct odb_source **sources_tail; - struct kh_odb_path_map *source_by_path; - int loaded_alternates; + /* + * Map of object database sources, keyed by their respective paths. + * This map is used to detect the case where the same source is + * registered multiple times. + */ + struct hashmap source_by_path; /* - * A list of alternate object directories loaded from the environment; - * this should not generally need to be accessed directly, but will - * populate the "sources" list when odb_prepare_alternates() is run. + * Whether source paths shall be compared case-insensitively, as + * determined by "core.ignoreCase". */ - char *alternate_db; + int source_paths_icase; /* * Objects that should be substituted by other objects @@ -76,6 +74,9 @@ struct object_database { struct commit_graph *commit_graph; unsigned commit_graph_attempted : 1; /* if loading has been attempted */ + struct diff_hunks_store *diff_hunks_store; + unsigned diff_hunks_store_attempted : 1; /* if loading has been attempted */ + /* * This is meant to hold a *small* number of objects that you would * want odb_read_object() to be able to return, but yet you do not want @@ -100,6 +101,20 @@ struct object_database { struct string_list submodule_source_paths; }; +enum odb_new_flags { + /* + * Honor environment variables when constructing the object database + * sources. This makes us respect the following environment variables: + * + * - GIT_OBJECT_DIRECTORY to override the primary object directory. + * + * - GIT_ALTERNATE_OBJECT_DIRECTORIES to override alternates. + * + * Environment variables may be backend-specific. + */ + ODB_NEW_HONOR_ENV = (1 << 0), +}; + /* * Create a new object database for the given repository. * @@ -112,8 +127,7 @@ struct object_database { * Returns the newly created object database. */ struct object_database *odb_new(struct repository *repo, - const char *primary_source, - const char *alternate_sources); + enum odb_new_flags flags); /* Free the object database and release all resources. */ void odb_free(struct object_database *o); @@ -199,10 +213,12 @@ struct odb_source *odb_find_source_or_die(struct object_database *odb, const cha /* * Replace the current writable object directory with the specified temporary - * object directory; returns the former primary source. + * object directory and return the newly installed primary source. The former + * primary source is reported via `prev_source` when non-NULL. */ struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, - const char *dir, int will_destroy); + const char *dir, int will_destroy, + struct odb_source **prev_source); /* * Restore the primary source that was previously replaced by @@ -247,12 +263,6 @@ void odb_for_each_alternate_ref(struct object_database *odb, int odb_mkstemp(struct object_database *odb, struct strbuf *temp_filename, const char *pattern); -/* - * Prepare alternate object sources for the given database by reading - * "objects/info/alternates" and opening the respective sources. - */ -void odb_prepare_alternates(struct object_database *odb); - /* * Check whether the object database has any alternates. The primary object * source does not count as alternate. @@ -671,18 +681,166 @@ static inline int odb_write_object(struct object_database *odb, return odb_write_object_ext(odb, buf, len, type, oid, NULL, 0); } -struct odb_write_stream; +struct odb_stream; int odb_write_object_stream(struct object_database *odb, - struct odb_write_stream *stream, size_t len, + struct odb_stream *stream, struct object_id *oid); +/* + * Options for generating a packfile via `odb_generate_pack()`. + */ +struct odb_generate_pack_options { + /* Tips of the object graph that shall be packed. */ + struct oid_array wants; + + /* + * Boundary of the object graph. Objects reachable from any of these + * tips are expected to already be available to whoever consumes the + * pack and shall thus not be packed. + */ + struct oid_array haves; + + /* + * The shallow boundary that shall be used when computing object + * reachability. When set, any shallow information of the repository + * itself shall be ignored in favor of these objects. + */ + struct oid_array shallows; + + /* + * Pre-expanded object filter specification that limits the set of + * objects that shall be packed. May be `NULL` in case no filter shall + * be applied. + */ + const char *filter_spec; + + /* + * Protocols that may be used to offload objects via packfile URIs. + * May be `NULL` in case packfile URIs shall not be used. + */ + const struct string_list *uri_protocols; + + /* + * Hook command that shall be executed instead of the internal + * machinery to generate the pack. It is up to the specific backend + * whether or not this hook is supported. May be `NULL` in case no + * hook shall be executed. + */ + const char *pack_objects_hook; + + /* + * File descriptor that the generated pack shall be written to. If set + * to `-1`, a pipe will be created and exposed via the pack generator's + * `out` field. If set to `0`, the pack will be written to the standard + * output stream. Otherwise, the provided descriptor will be written to + * and is consumed by the generator. + */ + int pack_fd; + + /* + * File descriptor that progress output shall be written to. The same + * semantics as for `pack_fd` apply, except that `0` will cause the + * generator to write to stderr instead of stdout. + */ + int progress_fd; + + /* Whether to print progress or not. */ + enum { + /* Don't print progress output. */ + ODB_GENERATE_PACK_PROGRESS_NONE, + + /* + * Print progress while computing the packfile, but stop + * printing progress once starting to write it. + */ + ODB_GENERATE_PACK_PROGRESS_STANDARD, + + /* + * Similar to STANDARD, but also print progress when writing + * the packfile. + */ + ODB_GENERATE_PACK_PROGRESS_VERBOSE, + } progress; + + /* Allow the pack to contain deltas against unpacked objects. */ + unsigned thin:1; + + /* Use offset deltas instead of reference deltas. */ + unsigned ofs_delta:1; + + /* Include unasked-for annotated tags of packed objects. */ + unsigned include_tag:1; + + /* The generated pack is destined for a shallow consumer. */ + unsigned shallow:1; + + /* Allow objects that may be missing due to a promisor remote. */ + unsigned missing_allow_promisor:1; + + /* Do not use bitmap indices when computing reachability. */ + unsigned disable_bitmaps:1; +}; + +#define ODB_GENERATE_PACK_OPTIONS_INIT { \ + .wants = OID_ARRAY_INIT, \ + .haves = OID_ARRAY_INIT, \ + .shallows = OID_ARRAY_INIT, \ + .pack_fd = -1, \ +} + +/* Release resources associated with the options. */ +void odb_generate_pack_options_release(struct odb_generate_pack_options *opts); + +/* + * A handle for an ongoing packfile generation as started via + * `odb_generate_pack()`. + */ +struct odb_pack_generator { + /* + * File descriptor from which the generated pack can be read. Only set + * when the pack generation was started with `pack_fd == -1`. The + * caller is responsible for closing the descriptor. + */ + int out; + + /* + * File descriptor from which progress output can be read. Only set + * when the pack generation was started with `progress_fd == -1`. The + * caller is responsible for closing the descriptor. + */ + int err; + + /* + * Callback function to finish this generator. This callback is + * expected to wait for the packfile generation to complete and to then + * free the generator itself. + */ + int (*finish)(struct odb_pack_generator *); +}; + +/* + * Start generating a packfile from the object database with the given + * options. The pack is generated asynchronously; the caller is expected to + * consume the file descriptors exposed via the pack generator and to then + * wait for completion via `odb_pack_generator_finish()`. + * + * Returns 0 on success and populates the `out` pointer with the pack + * generator. Returns a negative error code otherwise. + */ +int odb_generate_pack(struct object_database *odb, + struct odb_pack_generator **out, + const struct odb_generate_pack_options *opts); + +/* + * Wait for the packfile generation to complete and free the pack generator. + * Returns 0 on success, a negative error code otherwise. + */ +int odb_pack_generator_finish(struct odb_pack_generator *generator); + void parse_alternates(const char *string, int sep, const char *relative_base, struct strvec *out); -/* Free pointers inside of object_info, but not object_info itself */ -void free_object_info_contents(struct object_info *object_info); - #endif /* ODB_H */ diff --git a/odb/source-files.c b/odb/source-files.c index 5a68af7d84c250..4a723b357a8d29 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -4,6 +4,7 @@ #include "chdir-notify.h" #include "config.h" #include "gettext.h" +#include "hex.h" #include "lockfile.h" #include "object-file.h" #include "odb.h" @@ -22,8 +23,7 @@ #include "tree.h" #include "write-or-die.h" -static void odb_source_files_reparent(const char *name UNUSED, - const char *old_cwd, +static void odb_source_files_reparent(const char *old_cwd, const char *new_cwd, void *cb_data) { @@ -37,7 +37,7 @@ static void odb_source_files_reparent(const char *name UNUSED, static void odb_source_files_free(struct odb_source *source) { struct odb_source_files *files = odb_source_files_downcast(source); - chdir_notify_unregister(NULL, odb_source_files_reparent, files); + chdir_notify_unregister(odb_source_files_reparent, files); odb_source_free(&files->loose->base); odb_source_free(&files->packed->base); odb_source_release(&files->base); @@ -51,6 +51,23 @@ static void odb_source_files_close(struct odb_source *source) odb_source_close(&files->packed->base); } +static int odb_source_files_create_on_disk(struct odb_source *source) +{ + struct strbuf path = STRBUF_INIT; + + safe_create_dir(source->odb->repo, source->path, 1); + + strbuf_addf(&path, "%s/pack", source->path); + safe_create_dir(source->odb->repo, path.buf, 1); + + strbuf_reset(&path); + strbuf_addf(&path, "%s/info", source->path); + safe_create_dir(source->odb->repo, path.buf, 1); + + strbuf_release(&path); + return 0; +} + static void odb_source_files_prepare(struct odb_source *source, enum odb_prepare_flags flags) { @@ -73,7 +90,7 @@ static int odb_source_files_read_object_info(struct odb_source *source, return -1; } -static int odb_source_files_read_object_stream(struct odb_read_stream **out, +static int odb_source_files_read_object_stream(struct odb_stream **out, struct odb_source *source, const struct object_id *oid) { @@ -184,12 +201,11 @@ static int odb_source_files_write_object(struct odb_source *source, } static int odb_source_files_write_object_stream(struct odb_source *source, - struct odb_write_stream *stream, - size_t len, + struct odb_stream *stream, struct object_id *oid) { struct odb_source_files *files = odb_source_files_downcast(source); - return odb_source_write_object_stream(&files->loose->base, stream, len, oid); + return odb_source_write_object_stream(&files->loose->base, stream, oid); } static int odb_source_files_begin_transaction(struct odb_source *source, @@ -729,6 +745,148 @@ int odb_source_files_optimize(struct odb_source *source, return ret; } +struct odb_pack_generator_files { + struct odb_pack_generator base; + struct child_process cp; +}; + +static int odb_pack_generator_files_finish(struct odb_pack_generator *_generator) +{ + struct odb_pack_generator_files *generator = + (struct odb_pack_generator_files *)_generator; + int ret; + + ret = finish_command(&generator->cp); + free(generator); + + if (ret) { + /* + * On failure, pack-objects is expected to have written a + * useful error message to its standard error stream already. + * Death by signal is worth mentioning, though, with the + * exception of SIGPIPE: that is a normal occurrence when the + * consumer of the pack hangs up. + */ + if (ret > 128 && ret - 128 == SIGPIPE) + return -1; + if (ret > 128) + error(_("pack-objects died of signal %d"), ret - 128); + return -1; + } + + return 0; +} + +static int odb_source_files_generate_pack(struct odb_source *source UNUSED, + struct odb_pack_generator **out, + const struct odb_generate_pack_options *opts) +{ + struct child_process cp = CHILD_PROCESS_INIT; + struct odb_pack_generator_files *generator; + FILE *in; + + /* + * The hook is expected to spawn "$hook git pack-objects " + * and to behave like git-pack-objects(1) would have. This can for + * example be used to serve precomputed packfiles. + */ + if (opts->pack_objects_hook) { + strvec_push(&cp.args, opts->pack_objects_hook); + strvec_push(&cp.args, "git"); + cp.use_shell = 1; + } else { + cp.git_cmd = 1; + } + + /* + * The caller-provided shallow boundary overrides any shallow state + * that the repository itself may have, so the shallow file needs to + * be neutralized. + */ + if (opts->shallows.nr) { + strvec_push(&cp.args, "--shallow-file"); + strvec_push(&cp.args, ""); + } + strvec_push(&cp.args, "pack-objects"); + strvec_push(&cp.args, "--revs"); + strvec_push(&cp.args, "--stdout"); + if (opts->thin) + strvec_push(&cp.args, "--thin"); + if (opts->shallow) + strvec_push(&cp.args, "--shallow"); + if (opts->ofs_delta) + strvec_push(&cp.args, "--delta-base-offset"); + if (opts->include_tag) + strvec_push(&cp.args, "--include-tag"); + if (opts->missing_allow_promisor) + strvec_push(&cp.args, "--missing=allow-promisor"); + if (opts->disable_bitmaps) + strvec_push(&cp.args, "--no-use-bitmap-index"); + switch (opts->progress) { + case ODB_GENERATE_PACK_PROGRESS_NONE: + strvec_push(&cp.args, "--quiet"); + break; + case ODB_GENERATE_PACK_PROGRESS_STANDARD: + strvec_push(&cp.args, "--progress"); + break; + case ODB_GENERATE_PACK_PROGRESS_VERBOSE: + strvec_push(&cp.args, "--all-progress"); + break; + default: + BUG("unknown progress option %d", opts->progress); + } + if (opts->filter_spec) + strvec_pushf(&cp.args, "--filter=%s", opts->filter_spec); + if (opts->uri_protocols) + for (size_t i = 0; i < opts->uri_protocols->nr; i++) + strvec_pushf(&cp.args, "--uri-protocol=%s", + opts->uri_protocols->items[i].string); + + cp.in = -1; + cp.out = opts->pack_fd; + cp.err = opts->progress_fd; + cp.clean_on_exit = 1; + + if (start_command(&cp)) + return error(_("could not spawn pack-objects")); + + /* + * Feed the objects to pack-objects. This is safe to do synchronously + * because pack-objects consumes all of its standard input before it + * starts to generate the pack. + */ + in = xfdopen(cp.in, "w"); + for (size_t i = 0; i < opts->shallows.nr; i++) + fprintf(in, "--shallow %s\n", oid_to_hex(&opts->shallows.oid[i])); + for (size_t i = 0; i < opts->wants.nr; i++) + fprintf(in, "%s\n", oid_to_hex(&opts->wants.oid[i])); + fprintf(in, "--not\n"); + for (size_t i = 0; i < opts->haves.nr; i++) + fprintf(in, "%s\n", oid_to_hex(&opts->haves.oid[i])); + fprintf(in, "\n"); + fflush(in); + if (ferror(in)) { + error(_("error writing to pack-objects")); + fclose(in); + if (opts->pack_fd < 0) + close(cp.out); + if (opts->progress_fd < 0) + close(cp.err); + finish_command(&cp); + return -1; + } + fclose(in); + + CALLOC_ARRAY(generator, 1); + generator->base.out = opts->pack_fd < 0 ? cp.out : -1; + generator->base.err = opts->progress_fd < 0 ? cp.err : -1; + generator->base.finish = odb_pack_generator_files_finish; + generator->cp = cp; + + *out = &generator->base; + return 0; +} + struct odb_source_files *odb_source_files_new(struct object_database *odb, const char *path, bool local) @@ -742,6 +900,7 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, files->base.free = odb_source_files_free; files->base.close = odb_source_files_close; + files->base.create_on_disk = odb_source_files_create_on_disk; files->base.prepare = odb_source_files_prepare; files->base.read_object_info = odb_source_files_read_object_info; files->base.read_object_stream = odb_source_files_read_object_stream; @@ -756,6 +915,7 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, files->base.write_alternate = odb_source_files_write_alternate; files->base.optimize = odb_source_files_optimize; files->base.optimize_required = odb_source_files_optimize_required; + files->base.generate_pack = odb_source_files_generate_pack; /* * Ideally, we would only ever store absolute paths in the source. This @@ -763,7 +923,7 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, * paths in the primary ODB source in some user-facing functionality. */ if (!is_absolute_path(path)) - chdir_notify_register(NULL, odb_source_files_reparent, files); + chdir_notify_register(odb_source_files_reparent, files); return files; } diff --git a/odb/source-files.h b/odb/source-files.h index 044242bc36e4a7..9630b5f962e15f 100644 --- a/odb/source-files.h +++ b/odb/source-files.h @@ -43,7 +43,9 @@ bool odb_source_files_optimize_required(struct odb_source *source, static inline struct odb_source_files *odb_source_files_downcast(struct odb_source *source) { if (source->type != ODB_SOURCE_FILES) - BUG("trying to downcast source of type '%d' to files", source->type); + BUG("trying to downcast source of type '%s' to '%s'", + odb_source_type_to_name(source->type), + odb_source_type_to_name(ODB_SOURCE_FILES)); return container_of(source, struct odb_source_files, base); } diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index 3e71611b8e0071..795672adf255c6 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -73,12 +73,12 @@ static int odb_source_inmemory_read_object_info(struct odb_source *source, } struct odb_read_stream_inmemory { - struct odb_read_stream base; + struct odb_stream base; const unsigned char *buf; size_t offset; }; -static ssize_t odb_read_stream_inmemory_read(struct odb_read_stream *stream, +static ssize_t odb_read_stream_inmemory_read(struct odb_stream *stream, char *buf, size_t buf_len) { struct odb_read_stream_inmemory *inmemory = @@ -94,12 +94,12 @@ static ssize_t odb_read_stream_inmemory_read(struct odb_read_stream *stream, return bytes; } -static int odb_read_stream_inmemory_close(struct odb_read_stream *stream UNUSED) +static int odb_read_stream_inmemory_close(struct odb_stream *stream UNUSED) { return 0; } -static int odb_source_inmemory_read_object_stream(struct odb_read_stream **out, +static int odb_source_inmemory_read_object_stream(struct odb_stream **out, struct odb_source *source, const struct object_id *oid) { @@ -256,8 +256,7 @@ static int odb_source_inmemory_write_object(struct odb_source *source, } static int odb_source_inmemory_write_object_stream(struct odb_source *source, - struct odb_write_stream *stream, - size_t len, + struct odb_stream *stream, struct object_id *oid) { char buf[16384]; @@ -265,12 +264,19 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source, char *data; int ret; - CALLOC_ARRAY(data, len); - while (!stream->is_finished) { + CALLOC_ARRAY(data, stream->size); + while (1) { ssize_t bytes_read; - bytes_read = odb_write_stream_read(stream, buf, sizeof(buf)); - if (total_read + bytes_read > len) { + bytes_read = odb_stream_read(stream, buf, sizeof(buf)); + if (bytes_read < 0) { + ret = error("failed to read object stream"); + goto out; + } + if (!bytes_read) + break; + + if (total_read + bytes_read > stream->size) { ret = error("object stream yielded more bytes than expected"); goto out; } @@ -279,15 +285,16 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source, total_read += bytes_read; } - if (total_read != len) { + if (total_read != stream->size) { ret = error("object stream yielded less bytes than expected"); goto out; } - hash_object_file(source->odb->repo->hash_algo, data, total_read, OBJ_BLOB, oid); + hash_object_file(source->odb->repo->hash_algo, data, total_read, + stream->type, oid); - ret = odb_source_inmemory_write_object(source, data, len, OBJ_BLOB, oid, - NULL, NULL, 0); + ret = odb_source_inmemory_write_object(source, data, stream->size, + stream->type, oid, NULL, NULL, 0); if (ret < 0) goto out; diff --git a/odb/source-inmemory.h b/odb/source-inmemory.h index a88fc2e320ed5c..adbad23e8b26af 100644 --- a/odb/source-inmemory.h +++ b/odb/source-inmemory.h @@ -26,7 +26,9 @@ struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb) static inline struct odb_source_inmemory *odb_source_inmemory_downcast(struct odb_source *source) { if (source->type != ODB_SOURCE_INMEMORY) - BUG("trying to downcast source of type '%d' to in-memory", source->type); + BUG("trying to downcast source of type '%s' to '%s'", + odb_source_type_to_name(source->type), + odb_source_type_to_name(ODB_SOURCE_INMEMORY)); return container_of(source, struct odb_source_inmemory, base); } diff --git a/odb/source-loose.c b/odb/source-loose.c index ef0e9192777c4a..0921a8748082ba 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -278,7 +278,7 @@ static void *odb_source_loose_map_object(struct odb_source_loose *loose, } struct odb_loose_read_stream { - struct odb_read_stream base; + struct odb_stream base; git_zstream z; enum { ODB_LOOSE_READ_STREAM_INUSE, @@ -292,7 +292,7 @@ struct odb_loose_read_stream { int hdr_used; }; -static ssize_t read_istream_loose(struct odb_read_stream *_st, char *buf, size_t sz) +static ssize_t read_istream_loose(struct odb_stream *_st, char *buf, size_t sz) { struct odb_loose_read_stream *st = container_of(_st, struct odb_loose_read_stream, base); @@ -339,7 +339,7 @@ static ssize_t read_istream_loose(struct odb_read_stream *_st, char *buf, size_t return total_read; } -static int close_istream_loose(struct odb_read_stream *_st) +static int close_istream_loose(struct odb_stream *_st) { struct odb_loose_read_stream *st = container_of(_st, struct odb_loose_read_stream, base); @@ -350,7 +350,7 @@ static int close_istream_loose(struct odb_read_stream *_st) return 0; } -static int odb_source_loose_read_object_stream(struct odb_read_stream **out, +static int odb_source_loose_read_object_stream(struct odb_stream **out, struct odb_source *source, const struct object_id *oid) { @@ -845,8 +845,7 @@ static int odb_source_loose_write_object(struct odb_source *source, } static int odb_source_loose_write_object_stream(struct odb_source *source, - struct odb_write_stream *in_stream, - size_t len, + struct odb_stream *in_stream, struct object_id *oid) { struct odb_source_loose *loose = odb_source_loose_downcast(source); @@ -860,6 +859,7 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, struct strbuf filename = STRBUF_INIT; unsigned char buf[8192]; int dirlen; + bool is_finished = false; char hdr[MAX_HEADER_LEN]; int hdrlen; @@ -868,7 +868,7 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, /* Since oid is not determined, save tmp file to odb path. */ strbuf_addf(&filename, "%s/", loose->base.path); - hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len); + hdrlen = format_object_header(hdr, sizeof(hdr), in_stream->type, in_stream->size); /* * Common steps for write_loose_object and stream_loose_object to @@ -890,21 +890,24 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, do { unsigned char *in0 = stream.next_in; - if (!stream.avail_in && !in_stream->is_finished) { - ssize_t read_len = odb_write_stream_read(in_stream, buf, - sizeof(buf)); + if (!stream.avail_in && !is_finished) { + ssize_t read_len = odb_stream_read(in_stream, buf, + sizeof(buf)); if (read_len < 0) { close(fd); err = -1; goto cleanup; } + /* All data has been read. */ + if (!read_len) { + is_finished = true; + flush = 1; + } + stream.avail_in = read_len; stream.next_in = buf; in0 = buf; - /* All data has been read. */ - if (in_stream->is_finished) - flush = 1; } ret = write_loose_object_common(loose, &c, &compat_c, &stream, flush, in0, fd, compressed, sizeof(compressed)); @@ -916,9 +919,9 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, */ } while (ret == Z_OK || ret == Z_BUF_ERROR); - if (stream.total_in != len + hdrlen) + if (stream.total_in != in_stream->size + hdrlen) die(_("write stream object %"PRIuMAX" != %"PRIuMAX), (uintmax_t)stream.total_in, - (uintmax_t)len + hdrlen); + (uintmax_t)in_stream->size + hdrlen); /* * Common steps for write_loose_object and stream_loose_object to @@ -1006,8 +1009,7 @@ static void odb_source_loose_close(struct odb_source *source UNUSED) /* Nothing to do. */ } -static void odb_source_loose_reparent(const char *name UNUSED, - const char *old_cwd, +static void odb_source_loose_reparent(const char *old_cwd, const char *new_cwd, void *cb_data) { @@ -1023,7 +1025,7 @@ static void odb_source_loose_free(struct odb_source *source) struct odb_source_loose *loose = odb_source_loose_downcast(source); odb_source_loose_clear_cache(loose); loose_object_map_clear(&loose->map); - chdir_notify_unregister(NULL, odb_source_loose_reparent, loose); + chdir_notify_unregister(odb_source_loose_reparent, loose); odb_source_release(&loose->base); free(loose); } @@ -1053,7 +1055,9 @@ struct odb_source_loose *odb_source_loose_new(struct object_database *odb, loose->base.write_alternate = odb_source_loose_write_alternate; if (!is_absolute_path(loose->base.path)) - chdir_notify_register(NULL, odb_source_loose_reparent, loose); + chdir_notify_register(odb_source_loose_reparent, loose); + + loose_object_map_load(loose); return loose; } diff --git a/odb/source-loose.h b/odb/source-loose.h index 6070aaf3ce6ab2..3cf2e1f8f1d5e2 100644 --- a/odb/source-loose.h +++ b/odb/source-loose.h @@ -41,7 +41,9 @@ struct odb_source_loose *odb_source_loose_new(struct object_database *odb, static inline struct odb_source_loose *odb_source_loose_downcast(struct odb_source *source) { if (source->type != ODB_SOURCE_LOOSE) - BUG("trying to downcast source of type '%d' to loose", source->type); + BUG("trying to downcast source of type '%s' to '%s'", + odb_source_type_to_name(source->type), + odb_source_type_to_name(ODB_SOURCE_LOOSE)); return container_of(source, struct odb_source_loose, base); } diff --git a/odb/source-packed.c b/odb/source-packed.c index 0890704e76879b..9b23ea15dc1900 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -70,7 +70,7 @@ static int odb_source_packed_read_object_info(struct odb_source *source, return 0; } -static int odb_source_packed_read_object_stream(struct odb_read_stream **out, +static int odb_source_packed_read_object_stream(struct odb_stream **out, struct odb_source *source, const struct object_id *oid) { @@ -609,8 +609,7 @@ static int odb_source_packed_write_object(struct odb_source *source UNUSED, } static int odb_source_packed_write_object_stream(struct odb_source *source UNUSED, - struct odb_write_stream *stream UNUSED, - size_t len UNUSED, + struct odb_stream *stream UNUSED, struct object_id *oid UNUSED) { return error("packed backend cannot write object streams"); @@ -786,8 +785,7 @@ static void odb_source_packed_prepare(struct odb_source *source, packed->initialized = true; } -static void odb_source_packed_reparent(const char *name UNUSED, - const char *old_cwd, +static void odb_source_packed_reparent(const char *old_cwd, const char *new_cwd, void *cb_data) { @@ -816,7 +814,7 @@ static void odb_source_packed_free(struct odb_source *source) { struct odb_source_packed *packed = odb_source_packed_downcast(source); - chdir_notify_unregister(NULL, odb_source_packed_reparent, packed); + chdir_notify_unregister(odb_source_packed_reparent, packed); for (struct packfile_list_entry *e = packed->packs.head; e; e = e->next) free(e->pack); @@ -853,7 +851,7 @@ struct odb_source_packed *odb_source_packed_new(struct object_database *odb, packed->base.write_alternate = odb_source_packed_write_alternate; if (!is_absolute_path(path)) - chdir_notify_register(NULL, odb_source_packed_reparent, packed); + chdir_notify_register(odb_source_packed_reparent, packed); return packed; } diff --git a/odb/source-packed.h b/odb/source-packed.h index 77309ddd0932b6..a0f6b5096dcd0f 100644 --- a/odb/source-packed.h +++ b/odb/source-packed.h @@ -78,7 +78,9 @@ struct odb_source_packed *odb_source_packed_new(struct object_database *odb, static inline struct odb_source_packed *odb_source_packed_downcast(struct odb_source *source) { if (source->type != ODB_SOURCE_PACKED) - BUG("trying to downcast source of type '%d' to packed", source->type); + BUG("trying to downcast source of type '%s' to '%s'", + odb_source_type_to_name(source->type), + odb_source_type_to_name(ODB_SOURCE_PACKED)); return container_of(source, struct odb_source_packed, base); } diff --git a/odb/source.c b/odb/source.c index 7993dcbd659399..30188b806d524d 100644 --- a/odb/source.c +++ b/odb/source.c @@ -4,6 +4,25 @@ #include "odb/source.h" #include "packfile.h" +static const char * const odb_source_names_by_type[] = { + [ODB_SOURCE_UNKNOWN] = "unknown", + [ODB_SOURCE_FILES] = "files", + [ODB_SOURCE_LOOSE] = "loose", + [ODB_SOURCE_PACKED] = "packed", + [ODB_SOURCE_INMEMORY] = "in-memory", +}; + +const char *odb_source_type_to_name(enum odb_source_type type) +{ + const char *name; + if (type < 0 || type >= ARRAY_SIZE(odb_source_names_by_type)) + type = ODB_SOURCE_UNKNOWN; + name = odb_source_names_by_type[type]; + if (!name) + BUG("name missing in `odb_source_names_by_type` for '%d'", type); + return name; +} + struct odb_source *odb_source_new(struct object_database *odb, const char *path, bool local) diff --git a/odb/source.h b/odb/source.h index d69f8e2d1cf626..559e2ea2e9ae8f 100644 --- a/odb/source.h +++ b/odb/source.h @@ -1,6 +1,7 @@ #ifndef ODB_SOURCE_H #define ODB_SOURCE_H +#include "hashmap.h" #include "object.h" #include "odb.h" #include "odb/transaction.h" @@ -25,8 +26,14 @@ enum odb_source_type { ODB_SOURCE_INMEMORY, }; +/* + * Convert between the enum and its name. Returns the equivalent of "unknown" + * for unknown types. + */ +const char *odb_source_type_to_name(enum odb_source_type type); + struct object_id; -struct odb_read_stream; +struct odb_stream; struct strvec; /* @@ -44,6 +51,12 @@ struct strvec; struct odb_source { struct odb_source *next; + /* + * Entry in the object database's map of sources, keyed by this + * source's path. + */ + struct hashmap_entry by_path_entry; + /* Object database that owns this object source. */ struct object_database *odb; @@ -83,6 +96,18 @@ struct odb_source { */ void (*close)(struct odb_source *source); + /* + * This callback is expected to create on-disk data structures that are + * required for this source to operate. + * + * The callback is expected to return 0 on success, a negative error + * code otherwise. + * + * This callback may be NULL in case the source does not need any + * on-disk setup. + */ + int (*create_on_disk)(struct odb_source *source); + /* * This callback is expected to prepare the source so that it becomes * ready for use. It optionally clears underlying caches of the object @@ -125,7 +150,7 @@ struct odb_source { * The callback is expected to return a negative error code in case * creating the object stream has failed, 0 otherwise. */ - int (*read_object_stream)(struct odb_read_stream **out, + int (*read_object_stream)(struct odb_stream **out, struct odb_source *source, const struct object_id *oid); @@ -221,7 +246,7 @@ struct odb_source { * otherwise. */ int (*write_object_stream)(struct odb_source *source, - struct odb_write_stream *stream, size_t len, + struct odb_stream *stream, struct object_id *oid); /* @@ -278,6 +303,23 @@ struct odb_source { */ bool (*optimize_required)(struct odb_source *source, const struct odb_optimize_options *opts); + + /* + * This callback is expected to start generating a packfile with the + * given options. The pack shall be generated asynchronously so that + * the caller can consume the pack data and progress output while the + * pack is being generated. + * + * This callback is optional. Sources that cannot generate packfiles + * shall leave it unset. + * + * The callback is expected to return 0 on success and populate the + * `out` pointer with the pack generator, a negative error code + * otherwise. + */ + int (*generate_pack)(struct odb_source *source, + struct odb_pack_generator **out, + const struct odb_generate_pack_options *opts); }; /* @@ -327,6 +369,17 @@ static inline void odb_source_close(struct odb_source *source) source->close(source); } +/* + * Create on-disk data structures that are required for this source to operate + * correctly. Returns 0 on success, a negative error code otherwise. + */ +static inline int odb_source_create_on_disk(struct odb_source *source) +{ + if (!source->create_on_disk) + return 0; + return source->create_on_disk(source); +} + /* * Prepare the object database source and clear any caches. Depending on the * backend used this may have the effect that concurrently-written objects @@ -354,7 +407,7 @@ static inline int odb_source_read_object_info(struct odb_source *source, * Create a new read stream for the given object ID. Returns 0 on success, a * negative error code otherwise. */ -static inline int odb_source_read_object_stream(struct odb_read_stream **out, +static inline int odb_source_read_object_stream(struct odb_stream **out, struct odb_source *source, const struct object_id *oid) { @@ -451,11 +504,10 @@ static inline int odb_source_write_object(struct odb_source *source, * out pointer for the object ID. */ static inline int odb_source_write_object_stream(struct odb_source *source, - struct odb_write_stream *stream, - size_t len, + struct odb_stream *stream, struct object_id *oid) { - return source->write_object_stream(source, stream, len, oid); + return source->write_object_stream(source, stream, oid); } /* @@ -520,4 +572,20 @@ static inline bool odb_source_optimize_required(struct odb_source *source, return source->optimize_required(source, opts); } +/* + * Start generating a packfile from the given source with the given options. + * The pack is generated asynchronously; the caller is expected to consume the + * file descriptors exposed via the pack generator and to then wait for + * completion via `odb_pack_generator_finish()`. + * + * Returns 0 on success and populates the `out` pointer with the pack + * generator, a negative error code otherwise. + */ +static inline int odb_source_generate_pack(struct odb_source *source, + struct odb_pack_generator **out, + const struct odb_generate_pack_options *opts) +{ + return source->generate_pack(source, out, opts); +} + #endif diff --git a/odb/streaming.c b/odb/streaming.c index 20531e864c9561..8f2143cab5d846 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -20,8 +20,8 @@ *****************************************************************/ struct odb_filtered_read_stream { - struct odb_read_stream base; - struct odb_read_stream *upstream; + struct odb_stream base; + struct odb_stream *upstream; struct stream_filter *filter; char ibuf[FILTER_BUFFER]; char obuf[FILTER_BUFFER]; @@ -30,14 +30,14 @@ struct odb_filtered_read_stream { int input_finished; }; -static int close_istream_filtered(struct odb_read_stream *_fs) +static int close_istream_filtered(struct odb_stream *_fs) { struct odb_filtered_read_stream *fs = (struct odb_filtered_read_stream *)_fs; free_stream_filter(fs->filter); - return odb_read_stream_close(fs->upstream); + return odb_stream_close(fs->upstream); } -static ssize_t read_istream_filtered(struct odb_read_stream *_fs, char *buf, +static ssize_t read_istream_filtered(struct odb_stream *_fs, char *buf, size_t sz) { struct odb_filtered_read_stream *fs = (struct odb_filtered_read_stream *)_fs; @@ -86,7 +86,7 @@ static ssize_t read_istream_filtered(struct odb_read_stream *_fs, char *buf, /* refill the input from the upstream */ if (!fs->input_finished) { - fs->i_end = odb_read_stream_read(fs->upstream, fs->ibuf, FILTER_BUFFER); + fs->i_end = odb_stream_read(fs->upstream, fs->ibuf, FILTER_BUFFER); if (fs->i_end < 0) return -1; if (fs->i_end) @@ -97,8 +97,8 @@ static ssize_t read_istream_filtered(struct odb_read_stream *_fs, char *buf, return filled; } -static struct odb_read_stream *attach_stream_filter(struct odb_read_stream *st, - struct stream_filter *filter) +static struct odb_stream *attach_stream_filter(struct odb_stream *st, + struct stream_filter *filter) { struct odb_filtered_read_stream *fs; @@ -120,19 +120,19 @@ static struct odb_read_stream *attach_stream_filter(struct odb_read_stream *st, *****************************************************************/ struct odb_incore_read_stream { - struct odb_read_stream base; + struct odb_stream base; char *buf; /* from odb_read_object_info_extended() */ unsigned long read_ptr; }; -static int close_istream_incore(struct odb_read_stream *_st) +static int close_istream_incore(struct odb_stream *_st) { struct odb_incore_read_stream *st = (struct odb_incore_read_stream *)_st; free(st->buf); return 0; } -static ssize_t read_istream_incore(struct odb_read_stream *_st, char *buf, size_t sz) +static ssize_t read_istream_incore(struct odb_stream *_st, char *buf, size_t sz) { struct odb_incore_read_stream *st = (struct odb_incore_read_stream *)_st; size_t read_size = sz; @@ -147,7 +147,7 @@ static ssize_t read_istream_incore(struct odb_read_stream *_st, char *buf, size_ return read_size; } -static int open_istream_incore(struct odb_read_stream **out, +static int open_istream_incore(struct odb_stream **out, struct object_database *odb, const struct object_id *oid) { @@ -178,13 +178,12 @@ static int open_istream_incore(struct odb_read_stream **out, * static helpers variables and functions for users of streaming interface *****************************************************************************/ -static int istream_source(struct odb_read_stream **out, +static int istream_source(struct odb_stream **out, struct object_database *odb, const struct object_id *oid) { struct odb_source *source; - odb_prepare_alternates(odb); for (source = odb->sources; source; source = source->next) if (!odb_source_read_object_stream(out, source, oid)) return 0; @@ -196,23 +195,23 @@ static int istream_source(struct odb_read_stream **out, * Users of streaming interface ****************************************************************/ -int odb_read_stream_close(struct odb_read_stream *st) +int odb_stream_close(struct odb_stream *st) { int r = st->close(st); free(st); return r; } -ssize_t odb_read_stream_read(struct odb_read_stream *st, void *buf, size_t sz) +ssize_t odb_stream_read(struct odb_stream *st, void *buf, size_t sz) { return st->read(st, buf, sz); } -struct odb_read_stream *odb_read_stream_open(struct object_database *odb, - const struct object_id *oid, - struct stream_filter *filter) +struct odb_stream *odb_stream_from_object(struct object_database *odb, + const struct object_id *oid, + struct stream_filter *filter) { - struct odb_read_stream *st; + struct odb_stream *st; const struct object_id *real = lookup_replace_object(odb->repo, oid); int ret = istream_source(&st, odb, real); @@ -221,9 +220,9 @@ struct odb_read_stream *odb_read_stream_open(struct object_database *odb, if (filter) { /* Add "&& !is_null_stream_filter(filter)" for performance */ - struct odb_read_stream *nst = attach_stream_filter(st, filter); + struct odb_stream *nst = attach_stream_filter(st, filter); if (!nst) { - odb_read_stream_close(st); + odb_stream_close(st); return NULL; } st = nst; @@ -232,27 +231,17 @@ struct odb_read_stream *odb_read_stream_open(struct object_database *odb, return st; } -ssize_t odb_write_stream_read(struct odb_write_stream *st, void *buf, size_t sz) -{ - return st->read(st, buf, sz); -} - -void odb_write_stream_release(struct odb_write_stream *st) -{ - free(st->data); -} - int odb_stream_blob_to_fd(struct object_database *odb, int fd, const struct object_id *oid, struct stream_filter *filter, int can_seek) { - struct odb_read_stream *st; + struct odb_stream *st; ssize_t kept = 0; int result = -1; - st = odb_read_stream_open(odb, oid, filter); + st = odb_stream_from_object(odb, oid, filter); if (!st) { if (filter) free_stream_filter(filter); @@ -263,7 +252,7 @@ int odb_stream_blob_to_fd(struct object_database *odb, for (;;) { char buf[1024 * 16]; ssize_t wrote, holeto; - ssize_t readlen = odb_read_stream_read(st, buf, sizeof(buf)); + ssize_t readlen = odb_stream_read(st, buf, sizeof(buf)); if (readlen < 0) goto close_and_exit; @@ -294,47 +283,53 @@ int odb_stream_blob_to_fd(struct object_database *odb, result = 0; close_and_exit: - odb_read_stream_close(st); + odb_stream_close(st); return result; } -struct read_object_fd_data { +struct fd_stream { + struct odb_stream base; int fd; size_t remaining; }; -static ssize_t read_object_fd(struct odb_write_stream *stream, - unsigned char *buf, size_t len) +static ssize_t fd_stream_read(struct odb_stream *stream, + char *buf, size_t len) { - struct read_object_fd_data *data = stream->data; + struct fd_stream *fds = container_of(stream, struct fd_stream, base); ssize_t read_result; size_t count; - if (stream->is_finished) + if (!fds->remaining) return 0; - count = data->remaining < len ? data->remaining : len; - read_result = read_in_full(data->fd, buf, count); + count = fds->remaining < len ? fds->remaining : len; + read_result = read_in_full(fds->fd, buf, count); if (read_result < 0 || (size_t)read_result != count) return -1; - data->remaining -= count; - if (!data->remaining) - stream->is_finished = 1; + fds->remaining -= count; return read_result; } -void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd, - size_t size) +static int fd_stream_close(struct odb_stream *stream UNUSED) +{ + /* The file descriptor is owned by the caller for now. */ + return 0; +} + +struct odb_stream *odb_stream_from_fd(int fd, size_t size, enum object_type type) { - struct read_object_fd_data *data; + struct fd_stream *fds; - CALLOC_ARRAY(data, 1); - data->fd = fd; - data->remaining = size; + CALLOC_ARRAY(fds, 1); + fds->base.read = fd_stream_read; + fds->base.close = fd_stream_close; + fds->base.size = size; + fds->base.type = type; + fds->fd = fd; + fds->remaining = size; - stream->data = data; - stream->read = read_object_fd; - stream->is_finished = 0; + return &fds->base; } diff --git a/odb/streaming.h b/odb/streaming.h index c0236717802301..b522ff513f26d2 100644 --- a/odb/streaming.h +++ b/odb/streaming.h @@ -8,68 +8,53 @@ #include "odb.h" struct object_database; -struct odb_read_stream; +struct odb_stream; struct stream_filter; -typedef int (*odb_read_stream_close_fn)(struct odb_read_stream *); -typedef ssize_t (*odb_read_stream_read_fn)(struct odb_read_stream *, char *, size_t); +typedef int (*odb_stream_close_fn)(struct odb_stream *); +typedef ssize_t (*odb_stream_read_fn)(struct odb_stream *, char *, size_t); /* - * A stream that can be used to read an object from the object database without - * loading all of it into memory. + * A stream that can be used to read an object from or write an object into the + * object database without loading all of it into memory. */ -struct odb_read_stream { - odb_read_stream_close_fn close; - odb_read_stream_read_fn read; +struct odb_stream { + odb_stream_close_fn close; + odb_stream_read_fn read; enum object_type type; size_t size; /* inflated size of full object */ }; /* - * Create a new object stream for the given object database. An optional filter - * can be used to transform the object's content. + * Create a new object stream for the given object. An optional filter can be + * used to transform the object's content. * * Returns the stream on success, a `NULL` pointer otherwise. */ -struct odb_read_stream *odb_read_stream_open(struct object_database *odb, - const struct object_id *oid, - struct stream_filter *filter); +struct odb_stream *odb_stream_from_object(struct object_database *odb, + const struct object_id *oid, + struct stream_filter *filter); /* - * Close the given read stream and release all resources associated with it. + * Create a new object stream for the given file descriptor. This can be used + * to, for example, stream an object into the object database. This function + * does _not_ take ownership of the file descriptor. It's the responsibility of + * the caller to close it after the stream has been closed. + */ +struct odb_stream *odb_stream_from_fd(int fd, size_t size, enum object_type type); + +/* + * Close the given object stream and release all resources associated with it. * Returns 0 on success, a negative error code otherwise. */ -int odb_read_stream_close(struct odb_read_stream *stream); +int odb_stream_close(struct odb_stream *stream); /* * Read data from the stream into the buffer. Returns 0 on EOF and the number * of bytes read on success. Returns a negative error code in case reading from * the stream fails. */ -ssize_t odb_read_stream_read(struct odb_read_stream *stream, void *buf, size_t len); - -/* - * A stream that provides an object to be written to the object database without - * loading all of it into memory. - */ -struct odb_write_stream { - ssize_t (*read)(struct odb_write_stream *, unsigned char *, size_t); - void *data; - int is_finished; -}; - -/* - * Read data from the stream into the buffer. Returns 0 when finished and the - * number of bytes read on success. Returns a negative error code in case - * reading from the stream fails. - */ -ssize_t odb_write_stream_read(struct odb_write_stream *stream, void *buf, - size_t len); - -/* - * Releases memory allocated for underlying stream data. - */ -void odb_write_stream_release(struct odb_write_stream *stream); +ssize_t odb_stream_read(struct odb_stream *stream, void *buf, size_t len); /* * Look up the object by its ID and write the full contents to the file @@ -88,10 +73,4 @@ int odb_stream_blob_to_fd(struct object_database *odb, struct stream_filter *filter, int can_seek); -/* - * Sets up an ODB write stream that reads from an fd. - */ -void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd, - size_t size); - #endif /* STREAMING_H */ diff --git a/odb/transaction.c b/odb/transaction.c index dab7da6a9a4f55..f6f20088ec1fde 100644 --- a/odb/transaction.c +++ b/odb/transaction.c @@ -33,16 +33,37 @@ int odb_transaction_commit(struct odb_transaction *transaction) ret = transaction->commit(transaction); transaction->source->odb->transaction = NULL; + + return ret; +} + +int odb_transaction_finalize(struct odb_transaction *transaction) +{ + int ret = 0; + + if (!transaction) + return 0; + + if (transaction->finalize) + ret = transaction->finalize(transaction); + free(transaction); return ret; } int odb_transaction_write_object_stream(struct odb_transaction *transaction, - struct odb_write_stream *stream, - size_t len, struct object_id *oid) + struct odb_stream *stream, + struct object_id *oid) +{ + return transaction->write_object_stream(transaction, stream, oid); +} + +int odb_transaction_write_pack(struct odb_transaction *transaction, int pack_fd, + struct strbuf *err_msg, + const struct odb_transaction_write_pack_opts *opts) { - return transaction->write_object_stream(transaction, stream, len, oid); + return transaction->write_pack(transaction, pack_fd, err_msg, opts); } int odb_transaction_env(struct odb_transaction *transaction, struct strvec *env) diff --git a/odb/transaction.h b/odb/transaction.h index 4cb2eafcbf08f5..2a40c85bc819c8 100644 --- a/odb/transaction.h +++ b/odb/transaction.h @@ -4,6 +4,50 @@ #include "gettext.h" #include "odb.h" +/* + * Options controlling how odb_transaction_write_pack() ingests a packfile. + */ +struct odb_transaction_write_pack_opts { + /* + * Optional fsck severity configuration to apply when incoming objects + * are verified. + */ + const char *fsck_msg_types; + + /* + * Path to an alternative shallow file describing the shallow boundaries + * to honor while ingesting the pack. + */ + const char *shallow_file; + + /* + * The max size in bytes of the incoming packfile allowed. No limit is + * enforced when set to 0. + */ + off_t max_input_size; + + /* + * Whether the validity of incoming objects should be verified. + */ + int fsck_objects; + + /* + * Whether to reject an incoming packfile if it is "thin". + */ + int reject_thin; + + /* + * Optional file descriptor for reporting progress and errors. Set to 0 + * for none. + */ + int err_fd; + + /* + * Suppresses progress reporting. + */ + int quiet; +}; + /* * A transaction may be started for an object database prior to writing new * objects via odb_transaction_begin(). These objects are not committed until @@ -22,17 +66,33 @@ struct odb_transaction { */ int (*commit)(struct odb_transaction *transaction); + /* + * Optional ODB source specific callback invoked when the transaction + * needs to perform any deferred cleanup after objects have been + * committed. Returns 0 on success, a negative error code otherwise. + */ + int (*finalize)(struct odb_transaction *transaction); + /* * This callback is expected to write the given object stream into - * the ODB transaction. Note that for now, only blobs support streaming. + * the ODB transaction. * * The resulting object ID shall be written into the out pointer. The * callback is expected to return 0 on success, a negative error code * otherwise. */ int (*write_object_stream)(struct odb_transaction *transaction, - struct odb_write_stream *stream, size_t len, + struct odb_stream *stream, struct object_id *oid); + /* + * This callback is expected to ingest the packfile readable via + * `pack_fd` into the transaction. Returns 0 on success, a negative + * error code otherwise. On failure, a human-readable description is + * appended to `err_msg`. + */ + int (*write_pack)(struct odb_transaction *transaction, int pack_fd, + struct strbuf *err_msg, + const struct odb_transaction_write_pack_opts *opts); /* * This callback is expected to populate the provided strvec with the @@ -75,14 +135,39 @@ static inline void odb_transaction_begin_or_die(struct object_database *odb, */ int odb_transaction_commit(struct odb_transaction *transaction); +/* + * Finalizes an ODB transaction, performing any deferred cleanup and freeing it. + * Must be called for every successfully started transaction. Note that, if the + * specified transaction is NULL, the function is a no-op. Returns 0 on success, + * a negative error code otherwise. + */ +int odb_transaction_finalize(struct odb_transaction *transaction); + +static inline void odb_transaction_commit_and_finalize_or_die(struct odb_transaction *transaction) +{ + if (odb_transaction_commit(transaction)) + die(_("failed to commit ODB transaction")); + if (odb_transaction_finalize(transaction)) + die(_("failed to finalize ODB transaction")); +} + /* * Writes the object in the provided stream into the transaction. The resulting * object ID is written into the out pointer. Returns 0 on success, a negative * error code otherwise. */ int odb_transaction_write_object_stream(struct odb_transaction *transaction, - struct odb_write_stream *stream, - size_t len, struct object_id *oid); + struct odb_stream *stream, + struct object_id *oid); + +/* + * Ingests the packfile readable via `pack_fd` into the transaction. Returns 0 + * on success, a negative error code otherwise. On failure, a human-readable + * description is appended to `err_msg`. + */ +int odb_transaction_write_pack(struct odb_transaction *transaction, int pack_fd, + struct strbuf *err_msg, + const struct odb_transaction_write_pack_opts *opts); /* * Populates the provided strvec with the environment variables that a child diff --git a/oss-fuzz/fuzz-pack-headers.c b/oss-fuzz/fuzz-pack-headers.c index ef61ab577c5098..e44afe0b8d232b 100644 --- a/oss-fuzz/fuzz-pack-headers.c +++ b/oss-fuzz/fuzz-pack-headers.c @@ -9,7 +9,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) size_t len; unpack_object_header_buffer((const unsigned char *)data, - (unsigned long)size, &type, &len); + size, &type, &len); return 0; } diff --git a/pack-bitmap.c b/pack-bitmap.c index e85bd69ba446aa..9782e5a21ae7a5 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -717,7 +717,6 @@ static int open_bitmap(struct repository *r, assert(!bitmap_git->map); - odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); @@ -2383,7 +2382,8 @@ static void reuse_partial_packfile_from_bitmap_1(struct bitmap_index *bitmap_git struct pack_window *w_curs = NULL; size_t pos = pack->bitmap_pos / BITS_IN_EWORD; - if (!pack->bitmap_pos) { + if (!pack->bitmap_pos && + pack->bitmap_nr == pack->p->num_objects) { /* * If we're processing the first (in the case of a MIDX, the * preferred pack) or the only (in the case of single-pack @@ -2399,6 +2399,9 @@ static void reuse_partial_packfile_from_bitmap_1(struct bitmap_index *bitmap_git * all ties are broken in favor of that pack (i.e. the one * we're currently processing). So any duplicate bases will be * resolved in favor of the pack we're processing. + * + * The range must also contain every physical pack entry so that + * bitmap and pack positions correspond. */ while (pos < result->word_alloc && pos < pack->bitmap_nr / BITS_IN_EWORD && @@ -2527,14 +2530,26 @@ void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, } else { struct packed_git *pack; uint32_t pack_int_id; + uint32_t bitmap_nr; if (bitmap_is_midx(bitmap_git)) { + struct bitmapped_pack bitmapped_pack; struct multi_pack_index *m = bitmap_git->midx; uint32_t preferred_pack_pos; while (m->base_midx) m = m->base_midx; + if (!m->chunk_bitmapped_packs) { + /* + * Without BTMP, determining the preferred + * pack's range requires scanning the pseudo-pack. + * Skip reuse and leave the bitmap result for + * normal packing. + */ + return; + } + if (midx_preferred_pack(m, &preferred_pack_pos) < 0) { warning(_("unable to compute preferred pack, disabling pack-reuse")); return; @@ -2542,6 +2557,12 @@ void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, pack = nth_midxed_pack(m, preferred_pack_pos); pack_int_id = preferred_pack_pos; + + if (nth_bitmapped_pack(m, &bitmapped_pack, + pack_int_id) < 0) + return; + pack = bitmapped_pack.p; + bitmap_nr = bitmapped_pack.bitmap_nr; } else { pack = bitmap_git->pack; /* @@ -2554,13 +2575,14 @@ void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, * that we do not expect to read this field. */ pack_int_id = -1; + bitmap_nr = pack->num_objects; } if (is_pack_valid(pack)) { ALLOC_GROW(packs, packs_nr + 1, packs_alloc); packs[packs_nr].p = pack; packs[packs_nr].pack_int_id = pack_int_id; - packs[packs_nr].bitmap_nr = pack->num_objects; + packs[packs_nr].bitmap_nr = bitmap_nr; packs[packs_nr].bitmap_pos = 0; packs[packs_nr].from_midx = bitmap_git->midx; packs_nr++; @@ -3417,7 +3439,6 @@ int verify_bitmap_files(struct repository *r) struct packed_git *p; int res = 0; - odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); struct multi_pack_index *m = get_multi_pack_index(files->packed); diff --git a/pack-check.c b/pack-check.c index c3b8db7c5c41a6..c7275f4b9282f5 100644 --- a/pack-check.c +++ b/pack-check.c @@ -34,7 +34,7 @@ int check_pack_crc(struct packed_git *p, struct pack_window **w_curs, uint32_t data_crc = crc32(0, NULL, 0); do { - unsigned long avail; + size_t avail; void *data = use_pack(p, w_curs, offset, &avail); if (avail > len) avail = len; @@ -71,7 +71,7 @@ static int verify_packfile(struct repository *r, git_hash_init(&ctx, r->hash_algo); do { - unsigned long remaining; + size_t remaining; unsigned char *in = use_pack(p, w_curs, offset, &remaining); offset += remaining; if (!pack_sig_ofs) @@ -106,7 +106,7 @@ static int verify_packfile(struct repository *r, QSORT(entries, nr_objects, compare_entries); for (i = 0; i < nr_objects; i++) { - struct odb_read_stream *stream = NULL; + struct odb_stream *stream = NULL; void *data; struct object_id oid; enum object_type type; @@ -171,7 +171,7 @@ static int verify_packfile(struct repository *r, display_progress(progress, base_count + i); if (stream) - odb_read_stream_close(stream); + odb_stream_close(stream); free(data); } diff --git a/pack-write.c b/pack-write.c index 24033a9101545a..85674e4b726bab 100644 --- a/pack-write.c +++ b/pack-write.c @@ -469,10 +469,11 @@ void fixup_pack_header_footer(const struct git_hash_algo *hash_algo, fsync_component_or_die(FSYNC_COMPONENT_PACK, pack_fd, pack_name); } -char *index_pack_lockfile(struct repository *r, int ip_out, int *is_well_formed) +char *index_pack_lockfile(struct odb_source *source, int ip_out, + int *is_well_formed) { char packname[GIT_MAX_HEXSZ + 6]; - const int len = r->hash_algo->hexsz + 6; + const int len = source->odb->repo->hash_algo->hexsz + 6; /* * The first thing we expect from index-pack's output @@ -489,7 +490,7 @@ char *index_pack_lockfile(struct repository *r, int ip_out, int *is_well_formed) packname[len-1] = 0; if (skip_prefix(packname, "keep\t", &name)) return xstrfmt("%s/pack/pack-%s.keep", - repo_get_object_directory(r), name); + source->path, name); return NULL; } if (is_well_formed) diff --git a/pack.h b/pack.h index 1cde92082be903..68dcf08cf339fa 100644 --- a/pack.h +++ b/pack.h @@ -3,6 +3,7 @@ #include "object.h" #include "csum-file.h" +#include "odb/source.h" struct packed_git; struct pack_window; @@ -105,7 +106,8 @@ off_t write_pack_header(struct hashfile *f, uint32_t); void fixup_pack_header_footer(const struct git_hash_algo *, int, unsigned char *, const char *, uint32_t, unsigned char *, off_t); -char *index_pack_lockfile(struct repository *r, int fd, int *is_well_formed); +char *index_pack_lockfile(struct odb_source *source, int fd, + int *is_well_formed); struct ref; diff --git a/packfile-list.c b/packfile-list.c index 01fb913abf78fc..d6d411823c34a9 100644 --- a/packfile-list.c +++ b/packfile-list.c @@ -57,11 +57,12 @@ void packfile_list_prepend(struct packfile_list *list, struct packed_git *pack) list->tail = entry; } -void packfile_list_append(struct packfile_list *list, struct packed_git *pack) +void packfile_list_append(struct packfile_list *list, struct packed_git *pack, + int skip_dup_check) { struct packfile_list_entry *entry; - entry = packfile_list_remove_internal(list, pack); + entry = skip_dup_check ? NULL : packfile_list_remove_internal(list, pack); if (!entry) { entry = xmalloc(sizeof(*entry)); entry->pack = pack; diff --git a/packfile-list.h b/packfile-list.h index 1b05e2aa36de08..2b4b98b22673a4 100644 --- a/packfile-list.h +++ b/packfile-list.h @@ -15,7 +15,8 @@ struct packfile_list_entry { void packfile_list_clear(struct packfile_list *list); void packfile_list_remove(struct packfile_list *list, struct packed_git *pack); void packfile_list_prepend(struct packfile_list *list, struct packed_git *pack); -void packfile_list_append(struct packfile_list *list, struct packed_git *pack); +void packfile_list_append(struct packfile_list *list, struct packed_git *pack, + int skip_dup_check); /* * Find the pack within the "packs" list whose index contains the object diff --git a/packfile.c b/packfile.c index 0eee45055f833e..ebcfba0ff2552c 100644 --- a/packfile.c +++ b/packfile.c @@ -10,6 +10,7 @@ #include "dir.h" #include "packfile.h" #include "delta.h" +#include "ewah/ewok.h" #include "hash-lookup.h" #include "commit.h" #include "object.h" @@ -620,7 +621,7 @@ static int in_window(struct repository *r, struct pack_window *win, unsigned char *use_pack(struct packed_git *p, struct pack_window **w_cursor, off_t offset, - unsigned long *left) + size_t *left) { struct pack_window *win = *w_cursor; @@ -781,7 +782,7 @@ void packfile_store_add_pack(struct odb_source_packed *store, if (pack->pack_fd != -1) pack_open_fds++; - packfile_list_append(&store->packs, pack); + packfile_list_append(&store->packs, pack, 1); strmap_put(&store->packs_by_path, pack->pack_name, pack); } @@ -866,12 +867,11 @@ struct packfile_list_entry *packfile_store_get_packs(struct odb_source_packed *s return store->packs.head; } -unsigned long unpack_object_header_buffer(const unsigned char *buf, - unsigned long len, enum object_type *type, size_t *sizep) +size_t unpack_object_header_buffer(const unsigned char *buf, size_t len, + enum object_type *type, size_t *sizep) { unsigned shift; - size_t size, c; - unsigned long used = 0; + size_t size, c, used = 0; c = buf[used++]; *type = (c >> 4) & 7; @@ -960,8 +960,7 @@ int unpack_object_header(struct packed_git *p, size_t *sizep) { unsigned char *base; - unsigned long left; - unsigned long used; + size_t left, used; enum object_type type; /* use_pack() assures us we have [base, base + 20) available @@ -1077,6 +1076,160 @@ static int get_delta_base_oid(struct packed_git *p, return -1; } +/* + * Search duplicate representations for a chain ending in a full object. + * Representations of the same OID are interchangeable as delta bases. + * + * Each path entry is the search frame for one OID. It walks that OID's + * contiguous .idx entries and retains the chosen entry's .pack offset + * for replay. + */ +struct delta_path_entry { + off_t selected_offset; /* zero until a candidate is selected */ + uint32_t next; /* index position of the next candidate */ + uint32_t remaining; /* total number of candidates remaining */ +}; + +struct delta_path { + struct delta_path_entry *entries; + size_t nr, alloc; +}; + +static int push_oid_group(struct packed_git *p, + const struct object_id *oid, + struct bitmap *visited, struct delta_path *path) +{ + struct object_id candidate; + struct delta_path_entry *entry; + uint32_t first_index_pos, group_index_pos, index_pos; + + /* + * Determine the range of index positions referring to duplicate + * copies of the given object. + * + * Any position within that range is OK, since we will determine + * the exact range below. + */ + if (!bsearch_pack(oid, p, &group_index_pos)) + return 0; + if (bitmap_get(visited, group_index_pos)) + return 0; + + first_index_pos = group_index_pos; + while (first_index_pos > 0) { + if (nth_packed_object_id(&candidate, p, first_index_pos - 1) < 0) + return -1; + if (!oideq(&candidate, oid)) + break; + first_index_pos--; + } + + for (index_pos = first_index_pos; index_pos < p->num_objects; index_pos++) { + if (nth_packed_object_id(&candidate, p, index_pos) < 0) + return -1; + if (!oideq(&candidate, oid)) + break; + } + + bitmap_set(visited, group_index_pos); + + ALLOC_GROW(path->entries, path->nr + 1, path->alloc); + entry = &path->entries[path->nr++]; + entry->selected_offset = 0; + entry->next = first_index_pos; + entry->remaining = index_pos - first_index_pos; + + return 1; +} + +static enum object_type find_delta_path(struct packed_git *p, + struct pack_window **w_curs, + off_t offset, + struct delta_path *path) +{ + struct object_id oid; + uint32_t pack_pos; + struct bitmap *visited; + enum object_type result = OBJ_BAD; + + if (offset_to_pack_pos(p, offset, &pack_pos) < 0) + return OBJ_BAD; + if (nth_packed_object_id(&oid, p, pack_pos_to_index(p, pack_pos)) < 0) + return OBJ_BAD; + + visited = bitmap_new(); + if (push_oid_group(p, &oid, visited, path) != 1) + goto done; + + /* + * Search depth-first for a chain ending in a full object. Each frame + * tries every representation of one OID; a delta pushes its base OID, + * while exhausting a frame backtracks to its parent. + */ + while (path->nr) { + struct delta_path_entry *entry = &path->entries[path->nr - 1]; + enum object_type candidate_type; + off_t curpos; + size_t size; + + /* + * This OID has no path to a full object. Let its parent try + * another representation; exhausting the root fails the search. + */ + if (!entry->remaining) { + path->nr--; + continue; + } + + entry->selected_offset = + nth_packed_object_offset(p, entry->next++); + entry->remaining--; + curpos = entry->selected_offset; + candidate_type = unpack_object_header(p, w_curs, &curpos, &size); + + /* + * A full object terminates the chain, and its type is + * inherited by every delta above it. A delta continues + * at its base; any other type rejects only this + * representation. + */ + switch (candidate_type) { + case OBJ_COMMIT: + case OBJ_TREE: + case OBJ_BLOB: + case OBJ_TAG: + result = candidate_type; + goto done; + case OBJ_OFS_DELTA: + case OBJ_REF_DELTA: + break; + default: + /* + * A bad or unknown type rejects only this copy; + * another representation of the same OID may + * still work. + */ + continue; + } + + /* + * Descend to this delta's base. A malformed reference + * or a missing or already-visited base rejects this + * copy. A newly pushed base is examined next; an index + * error aborts the search. + */ + if (get_delta_base_oid(p, w_curs, curpos, &oid, candidate_type, + entry->selected_offset)) + continue; + if (push_oid_group(p, &oid, visited, path) < 0) + goto done; + } + +done: + bitmap_free(visited); + return result; +} + static int retry_bad_packed_offset(struct repository *r, struct packed_git *p, off_t obj_offset) @@ -1105,11 +1258,27 @@ static enum object_type packed_to_object_type(struct repository *r, { off_t small_poi_stack[POI_STACK_PREALLOC]; off_t *poi_stack = small_poi_stack; + off_t root_offset = obj_offset; int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC; while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { off_t base_offset; size_t size; + + if (poi_stack_nr > 0 && poi_stack_nr % 2 == 0 && + obj_offset == poi_stack[poi_stack_nr / 2]) { + struct delta_path path = { 0 }; + /* + * Normal lookup returned to the same pack + * entry. Restart from the requested object + * using alternate representations. + */ + type = find_delta_path(p, w_curs, root_offset, &path); + free(path.entries); + if (type == OBJ_BAD) + goto unwind; + break; + } /* Push the object we're going to leave behind */ if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) { poi_stack_alloc = alloc_nr(poi_stack_nr); @@ -1525,9 +1694,11 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, { struct pack_window *w_curs = NULL; off_t curpos = obj_offset; + off_t root_offset = obj_offset; void *data = NULL; size_t size; enum object_type type; + struct delta_path path = { 0 }; struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC]; struct unpack_entry_stack_ent *delta_stack = small_delta_stack; int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC; @@ -1553,6 +1724,22 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, break; } + if (!path.nr && + delta_stack_nr > 0 && delta_stack_nr % 2 == 0 && + obj_offset == delta_stack[delta_stack_nr / 2].obj_offset) { + /* + * Normal lookup returned to the same pack + * entry. Find an acyclic path if one exists, + * discard this walk, and replay that path. + */ + if (find_delta_path(p, &w_curs, root_offset, + &path) == OBJ_BAD) + break; + delta_stack_nr = 0; + curpos = obj_offset = path.entries[0].selected_offset; + continue; + } + if (do_check_packed_object_crc && p->index_version > 1) { uint32_t pack_pos, index_pos; off_t len; @@ -1591,6 +1778,15 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, break; } + /* Use the base chosen by recovery, not the one from normal lookup. */ + if (path.nr) { + size_t path_pos = (size_t)delta_stack_nr + 1; + + if (path_pos >= path.nr) + BUG("alternate delta path ends in a delta"); + base_offset = path.entries[path_pos].selected_offset; + } + /* push object, proceed to base */ if (delta_stack_nr >= delta_stack_alloc && delta_stack == small_delta_stack) { @@ -1731,6 +1927,7 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, out: unuse_pack(&w_curs); + free(path.entries); if (delta_stack != small_delta_stack) free(delta_stack); @@ -1938,7 +2135,6 @@ int has_object_pack(struct repository *r, const struct object_id *oid) { struct odb_source *source; - odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); if (!odb_source_read_object_info(&files->packed->base, oid, NULL, 0)) @@ -2115,7 +2311,7 @@ int parse_pack_header_option(const char *in, unsigned char *out, unsigned int *l } struct odb_packed_read_stream { - struct odb_read_stream base; + struct odb_stream base; struct packed_git *pack; git_zstream z; enum { @@ -2127,7 +2323,7 @@ struct odb_packed_read_stream { off_t pos; }; -static ssize_t read_istream_pack_non_delta(struct odb_read_stream *_st, char *buf, +static ssize_t read_istream_pack_non_delta(struct odb_stream *_st, char *buf, size_t sz) { struct odb_packed_read_stream *st = (struct odb_packed_read_stream *)_st; @@ -2187,7 +2383,7 @@ static ssize_t read_istream_pack_non_delta(struct odb_read_stream *_st, char *bu return total_read; } -static int close_istream_pack_non_delta(struct odb_read_stream *_st) +static int close_istream_pack_non_delta(struct odb_stream *_st) { struct odb_packed_read_stream *st = (struct odb_packed_read_stream *)_st; if (st->z_state == ODB_PACKED_READ_STREAM_INUSE) @@ -2195,7 +2391,7 @@ static int close_istream_pack_non_delta(struct odb_read_stream *_st) return 0; } -int packfile_read_object_stream(struct odb_read_stream **out, +int packfile_read_object_stream(struct odb_stream **out, const struct object_id *oid, struct packed_git *pack, off_t offset) diff --git a/packfile.h b/packfile.h index e1f77152b5c4bf..6b2768695831fe 100644 --- a/packfile.h +++ b/packfile.h @@ -12,7 +12,7 @@ /* in odb.h */ struct object_info; -struct odb_read_stream; +struct odb_stream; struct packed_git { struct pack_window *windows; @@ -77,8 +77,6 @@ static inline struct repo_for_each_pack_data repo_for_eack_pack_data_init(struct { struct repo_for_each_pack_data data = { 0 }; - odb_prepare_alternates(repo->objects); - for (struct odb_source *source = repo->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); struct packfile_list_entry *entry = packfile_store_get_packs(files->packed); @@ -240,7 +238,8 @@ uint32_t get_pack_fanout(struct packed_git *p, uint32_t value); struct object_database; -unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); +unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, + size_t *); void close_pack_windows(struct packed_git *); void close_pack(struct packed_git *); void unuse_pack(struct pack_window **); @@ -299,14 +298,15 @@ int packfile_fill_entry(struct packed_git *p, int is_pack_valid(struct packed_git *); void *unpack_entry(struct repository *r, struct packed_git *, off_t, enum object_type *, size_t *); -unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, size_t *sizep); +size_t unpack_object_header_buffer(const unsigned char *buf, size_t len, + enum object_type *type, size_t *sizep); size_t get_size_from_delta(struct packed_git *, struct pack_window **, off_t); int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, size_t *); off_t get_delta_base(struct packed_git *p, struct pack_window **w_curs, off_t *curpos, enum object_type type, off_t delta_obj_offset); -int packfile_read_object_stream(struct odb_read_stream **out, +int packfile_read_object_stream(struct odb_stream **out, const struct object_id *oid, struct packed_git *pack, off_t offset); diff --git a/parse-options.c b/parse-options.c index 08c21d9fc0a585..4519ead9dc77b8 100644 --- a/parse-options.c +++ b/parse-options.c @@ -1414,6 +1414,8 @@ static enum parse_opt_result usage_with_options_internal(struct parse_opt_ctx_t if (opts->type == OPTION_SUBCOMMAND) continue; + if (!full && (opts->flags & PARSE_OPT_HIDDEN)) + continue; if (opts->type == OPTION_GROUP) { fputc('\n', outfile); need_newline = 0; @@ -1421,8 +1423,6 @@ static enum parse_opt_result usage_with_options_internal(struct parse_opt_ctx_t fprintf(outfile, "%s\n", _(opts->help)); continue; } - if (!full && (opts->flags & PARSE_OPT_HIDDEN)) - continue; if (need_newline) { fputc('\n', outfile); diff --git a/parse-options.h b/parse-options.h index 3ec8ba5cc83c60..d7f896a9337024 100644 --- a/parse-options.h +++ b/parse-options.h @@ -237,6 +237,11 @@ struct option { .type = OPTION_GROUP, \ .help = (h), \ } +#define OPT_HIDDEN_GROUP(h) { \ + .type = OPTION_GROUP, \ + .help = (h), \ + .flags = PARSE_OPT_HIDDEN, \ +} #define OPT_BIT(s, l, v, h, b) OPT_BIT_F(s, l, v, h, b, 0) #define OPT_BITOP(s, l, v, h, set, clear) { \ .type = OPTION_BITOP, \ diff --git a/parse.c b/parse.c index d77f28046a0916..266bbd539be8dc 100644 --- a/parse.c +++ b/parse.c @@ -134,6 +134,15 @@ int git_parse_ssize_t(const char *value, ssize_t *ret) return 1; } +int git_parse_size_t(const char *value, size_t *ret) +{ + uintmax_t tmp; + if (!git_parse_unsigned(value, &tmp, maximum_signed_value_of_type(size_t))) + return 0; + *ret = tmp; + return 1; +} + int git_parse_double(const char *value, double *ret) { char *end; diff --git a/parse.h b/parse.h index a6dd37c4cba273..db742f35fb0e8e 100644 --- a/parse.h +++ b/parse.h @@ -4,6 +4,7 @@ int git_parse_signed(const char *value, intmax_t *ret, intmax_t max); int git_parse_unsigned(const char *value, uintmax_t *ret, uintmax_t max); int git_parse_ssize_t(const char *, ssize_t *); +int git_parse_size_t(const char *, size_t *); int git_parse_ulong(const char *, unsigned long *); int git_parse_uint(const char *value, unsigned int *ret); int git_parse_int(const char *value, int *ret); diff --git a/pkt-line.c b/pkt-line.c index 3fc3e9ea7059be..338075558c2e9e 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -378,10 +378,10 @@ int packet_length(const char lenbuf_hex[4], size_t size) { if (size < 4) BUG("buffer too small"); - return hexval(lenbuf_hex[0]) << 12 | - hexval(lenbuf_hex[1]) << 8 | - hexval(lenbuf_hex[2]) << 4 | - hexval(lenbuf_hex[3]); + return hexval(lenbuf_hex[0], HEX_KIND_MIXED) << 12 | + hexval(lenbuf_hex[1], HEX_KIND_MIXED) << 8 | + hexval(lenbuf_hex[2], HEX_KIND_MIXED) << 4 | + hexval(lenbuf_hex[3], HEX_KIND_MIXED); } static const char *find_packfile_uri_path(const char *buffer) diff --git a/promisor-remote.c b/promisor-remote.c index 43505d1e1ac8fd..65496c69cf07bd 100644 --- a/promisor-remote.c +++ b/promisor-remote.c @@ -31,15 +31,6 @@ static int fetch_objects(struct repository *repo, FILE *child_in; int quiet; - if (git_env_bool(NO_LAZY_FETCH_ENVIRONMENT, 0)) { - static int warning_shown; - if (!warning_shown) { - warning_shown = 1; - warning(_("lazy fetching disabled; some objects may not be available")); - } - return -1; - } - child.git_cmd = 1; child.in = -1; if (repo != the_repository) @@ -270,10 +261,15 @@ static int remove_fetched_oids(struct repository *repo, return remaining_nr; } -static int try_promisor_remotes(struct repository *repo, - struct object_id **remaining_oids, - int *remaining_nr, int *to_free, - bool accepted_only) +/* + * Return 'true' if all the objects could be fetched from the + * (non-)accepted remotes, 'false' otherwise. + */ +static bool try_promisor_remotes(struct repository *repo, + struct object_id **remaining_oids, + int *remaining_nr, + int *to_free, + bool accepted_only) { struct promisor_remote *r = repo->promisor_remote_config->promisors; @@ -290,9 +286,37 @@ static int try_promisor_remotes(struct repository *repo, continue; } } - return 1; /* all fetched */ + return true; /* all fetched */ } - return 0; + return false; +} + +/* + * Return 'true' if all the objects could be fetched, 'false' otherwise. + */ +static bool lazy_fetch_objects(struct repository *repo, + struct object_id **remaining_oids, + int *remaining_nr, + int *to_free) +{ + if (git_env_bool(NO_LAZY_FETCH_ENVIRONMENT, 0)) { + static int warning_shown; + if (!warning_shown) { + warning_shown = 1; + warning(_("lazy fetching disabled; some objects may not be available")); + } + return false; + } + + promisor_remote_init(repo); + + /* Try accepted remotes first (those the server told us to use) */ + if (try_promisor_remotes(repo, remaining_oids, remaining_nr, + to_free, true)) + return true; + + return try_promisor_remotes(repo, remaining_oids, remaining_nr, + to_free, false); } void promisor_remote_get_direct(struct repository *repo, @@ -302,28 +326,18 @@ void promisor_remote_get_direct(struct repository *repo, struct object_id *remaining_oids = (struct object_id *)oids; int remaining_nr = oid_nr; int to_free = 0; - int i; if (oid_nr == 0) return; - promisor_remote_init(repo); - - /* Try accepted remotes first (those the server told us to use) */ - if (try_promisor_remotes(repo, &remaining_oids, &remaining_nr, - &to_free, true)) - goto all_fetched; - if (try_promisor_remotes(repo, &remaining_oids, &remaining_nr, - &to_free, false)) - goto all_fetched; - - for (i = 0; i < remaining_nr; i++) { - if (is_promisor_object(repo, &remaining_oids[i])) - die(_("could not fetch %s from promisor remote"), - oid_to_hex(&remaining_oids[i])); + if (!lazy_fetch_objects(repo, &remaining_oids, &remaining_nr, &to_free)) { + for (int i = 0; i < remaining_nr; i++) { + if (is_promisor_object(repo, &remaining_oids[i])) + die(_("could not fetch %s from promisor remote"), + oid_to_hex(&remaining_oids[i])); + } } -all_fetched: if (to_free) free(remaining_oids); } diff --git a/protocol-caps.c b/protocol-caps.c index 02261be14d817a..27e0f85b100cb9 100644 --- a/protocol-caps.c +++ b/protocol-caps.c @@ -11,7 +11,8 @@ #include "strbuf.h" struct requested_info { - unsigned size : 1; + unsigned size:1; + unsigned type:1; }; /* @@ -73,15 +74,20 @@ static void send_info(struct repository *r, struct packet_writer *writer, if (info->size) packet_writer_write(writer, "size"); + if (info->type) + packet_writer_write(writer, "type"); + for_each_string_list_item (item, oid_str_list) { const char *oid_str = item->string; + enum object_type object_type; struct object_id oid; size_t object_size; if (get_oid_hex_algop(oid_str, &oid, r->hash_algo) < 0) { packet_writer_error( writer, - "object-info: protocol error, expected to get oid, not '%s'", + "object-info: protocol error, expected to get " + "oid, not '%s'", oid_str); continue; } @@ -93,7 +99,8 @@ static void send_info(struct repository *r, struct packet_writer *writer, * If an object is not recognized by the server append SP to * the response. */ - if (get_object_info(r->objects, &oid, &object_size) <= OBJ_NONE) { + object_type = get_object_info(r->objects, &oid, &object_size); + if (object_type <= OBJ_NONE) { strbuf_addstr(&send_buffer, " "); goto write; } @@ -103,6 +110,9 @@ static void send_info(struct repository *r, struct packet_writer *writer, (uintmax_t)object_size); } + if (info->type) + strbuf_addf(&send_buffer, " %s", type_name(object_type)); + write: packet_writer_write(writer, "%s", send_buffer.buf); strbuf_reset(&send_buffer); @@ -124,6 +134,11 @@ int cap_object_info(struct repository *r, struct packet_reader *request) continue; } + if (!strcmp("type", request->line)) { + info.type = 1; + continue; + } + if (parse_oid(request->line, &oid_str_list)) continue; diff --git a/range-diff.c b/range-diff.c index 8e2dd2eb193eb9..3cadbcd7e1d6cb 100644 --- a/range-diff.c +++ b/range-diff.c @@ -52,6 +52,12 @@ static int read_patches(const char *range, struct string_list *list, int ret = -1; strvec_pushl(&cp.args, "log", "--no-color", "-p", + /* + * The patches being compared must be the builtin + * diff's: an external diff command or diff process + * could change either side of the comparison. + */ + "--no-ext-diff", "--reverse", "--date-order", "--decorate=no", "--no-prefix", "--submodule=short", /* diff --git a/read-cache.c b/read-cache.c index c0769848587b1a..0d17cd823c5084 100644 --- a/read-cache.c +++ b/read-cache.c @@ -353,15 +353,15 @@ static int ce_match_stat_basic(const struct cache_entry *ce, struct stat *st) static int is_racy_stat(const struct index_state *istate, const struct stat_data *sd) { + int use_nsec = repo_config_values(istate->repo)->use_nanosec; + return (istate->timestamp.sec && -#ifdef USE_NSEC - /* nanosecond timestamped files can also be racy! */ - (istate->timestamp.sec < sd->sd_mtime.sec || - (istate->timestamp.sec == sd->sd_mtime.sec && - istate->timestamp.nsec <= sd->sd_mtime.nsec)) -#else - istate->timestamp.sec <= sd->sd_mtime.sec -#endif + /* nanosecond timestamped files can also be racy! */ + use_nsec + ? (istate->timestamp.sec < sd->sd_mtime.sec || + (istate->timestamp.sec == sd->sd_mtime.sec && + istate->timestamp.nsec <= sd->sd_mtime.nsec)) + : istate->timestamp.sec <= sd->sd_mtime.sec ); } @@ -4062,7 +4062,7 @@ int add_files_to_cache(struct repository *repo, const char *prefix, odb_transaction_begin_or_die(repo->objects, &transaction, 0); run_diff_files(&rev, DIFF_RACY_IS_MODIFIED); if (!inflight) - odb_transaction_commit(transaction); + odb_transaction_commit_and_finalize_or_die(transaction); release_revisions(&rev); return !!data.add_errors; diff --git a/ref-filter.c b/ref-filter.c index 29aca08ce7b333..cd02677e42ad25 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -2744,6 +2744,72 @@ static int filter_exclude_match(struct ref_filter *filter, const char *refname) return match_pattern(filter->exclude.v, refname, filter->ignore_case); } +static const char *short_upstream_name(const char *full_ref) +{ + const char *short_name = full_ref; + + if (!skip_prefix(short_name, "refs/heads/", &short_name)) + skip_prefix(short_name, "refs/remotes/", &short_name); + return short_name; +} + +/* + * Match the configured upstream of a branch against the registered + * --forked patterns. Exact patterns are compared against the full + * upstream refname so they are unambiguous; glob patterns are matched + * against the abbreviated upstream so that a glob such as origin/... + * works as typed. + */ +static int filter_forked_match(struct ref_filter *filter, const char *refname) +{ + const char *short_name; + struct branch *branch; + const char *upstream; + + if (!skip_prefix(refname, "refs/heads/", &short_name)) + return 0; + branch = branch_get(short_name); + if (!branch) + return 0; + upstream = branch_get_upstream(branch, NULL); + if (!upstream) + return 0; + + for (size_t i = 0; i < filter->forked.nr; i++) { + const char *pattern = filter->forked.v[i]; + if (has_glob_specials(pattern)) { + if (!wildmatch(pattern, short_upstream_name(upstream), + WM_PATHNAME)) + return 1; + } else if (!strcmp(pattern, upstream)) { + return 1; + } + } + return 0; +} + +int ref_filter_forked_add(struct ref_filter *filter, const char *arg) +{ + struct object_id oid; + char *full_ref = NULL; + + if (has_glob_specials(arg)) { + strvec_push(&filter->forked, arg); + return 0; + } + + if (repo_dwim_ref(the_repository, arg, strlen(arg), &oid, + &full_ref, 0) == 1 && + (starts_with(full_ref, "refs/heads/") || + starts_with(full_ref, "refs/remotes/"))) { + strvec_push(&filter->forked, full_ref); + free(full_ref); + return 0; + } + free(full_ref); + return -1; +} + /* * We need to seek to the reference right after a given marker but excluding any * matching references. So we seek to the lexicographically next reference. @@ -2979,6 +3045,9 @@ static struct ref_array_item *apply_ref_filter(const struct reference *ref, if (filter->points_at.nr && !match_points_at(&filter->points_at, ref->oid, ref->name)) return NULL; + if (filter->forked.nr && !filter_forked_match(filter, ref->name)) + return NULL; + /* * A merge filter is applied on refs pointing to commits. Hence * obtain the commit using the 'oid' available and discard all @@ -3567,7 +3636,7 @@ static void append_literal(const char *cp, const char *ep, struct ref_formatting if (cp[1] == '%') cp++; else { - int ch = hex2chr(cp + 1); + int ch = hex2chr(cp + 1, HEX_KIND_MIXED); if (0 <= ch) { strbuf_addch(s, ch); cp += 3; @@ -3764,6 +3833,7 @@ void ref_filter_init(struct ref_filter *filter) void ref_filter_clear(struct ref_filter *filter) { strvec_clear(&filter->exclude); + strvec_clear(&filter->forked); oid_array_clear(&filter->points_at); commit_list_free(filter->with_commit); commit_list_free(filter->no_commit); diff --git a/ref-filter.h b/ref-filter.h index 120221b47fa30d..9361296e2a7440 100644 --- a/ref-filter.h +++ b/ref-filter.h @@ -67,6 +67,7 @@ struct ref_filter { const char **name_patterns; const char *start_after; struct strvec exclude; + struct strvec forked; struct oid_array points_at; struct commit_list *with_commit; struct commit_list *no_commit; @@ -110,6 +111,7 @@ struct ref_format { #define REF_FILTER_INIT { \ .points_at = OID_ARRAY_INIT, \ .exclude = STRVEC_INIT, \ + .forked = STRVEC_INIT, \ } #define REF_FORMAT_INIT { \ .use_color = GIT_COLOR_UNKNOWN, \ @@ -172,6 +174,14 @@ void ref_sorting_release(struct ref_sorting *); struct ref_sorting *ref_sorting_options(struct string_list *); /* Function to parse --merged and --no-merged options */ int parse_opt_merge_filter(const struct option *opt, const char *arg, int unset); +/* + * Register a --forked pattern on the filter. The argument is + * either a ref, which is resolved to its full refname, or a shell-style + * glob. Branches are kept only when their configured upstream matches + * one of the registered patterns. Returns -1 if the argument is not a + * valid ref or pattern. + */ +int ref_filter_forked_add(struct ref_filter *filter, const char *arg); /* Get the current HEAD's description */ char *get_head_description(void); /* Set up translated strings in the output. */ diff --git a/refs/files-backend.c b/refs/files-backend.c index 1cc20aa486c263..71628550f20a57 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -111,8 +111,7 @@ static void clear_loose_ref_cache(struct files_ref_store *refs) } } -static void files_ref_store_reparent(const char *name UNUSED, - const char *old_cwd, +static void files_ref_store_reparent(const char *old_cwd, const char *new_cwd, void *payload) { @@ -182,7 +181,7 @@ static struct ref_store *files_ref_store_init(struct repository *repo, packed_ref_store_init(repo, NULL, refs->gitcommondir, opts); refs->store_flags = opts->access_flags; - chdir_notify_register(NULL, files_ref_store_reparent, refs); + chdir_notify_register(files_ref_store_reparent, refs); strbuf_release(&refdir); @@ -234,7 +233,7 @@ static void files_ref_store_release(struct ref_store *ref_store) free(refs->gitcommondir); ref_store_release(refs->packed_ref_store); free(refs->packed_ref_store); - chdir_notify_unregister(NULL, files_ref_store_reparent, refs); + chdir_notify_unregister(files_ref_store_reparent, refs); } static void files_reflog_path(struct files_ref_store *refs, diff --git a/refs/packed-backend.c b/refs/packed-backend.c index b9b04b70105f9a..a73fc6aca78023 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -217,8 +217,7 @@ static size_t snapshot_hexsz(const struct snapshot *snapshot) return snapshot->refs->base.repo->hash_algo->hexsz; } -static void packed_ref_store_reparent(const char *name UNUSED, - const char *old_cwd, +static void packed_ref_store_reparent(const char *old_cwd, const char *new_cwd, void *payload) { @@ -248,7 +247,7 @@ struct ref_store *packed_ref_store_init(struct repository *repo, strbuf_addf(&sb, "%s/packed-refs", gitdir); refs->path = strbuf_detach(&sb, NULL); - chdir_notify_register(NULL, packed_ref_store_reparent, refs); + chdir_notify_register(packed_ref_store_reparent, refs); return ref_store; } @@ -293,7 +292,7 @@ static void packed_ref_store_release(struct ref_store *ref_store) clear_snapshot(refs); rollback_lock_file(&refs->lock); delete_tempfile(&refs->tempfile); - chdir_notify_unregister(NULL, packed_ref_store_reparent, refs); + chdir_notify_unregister(packed_ref_store_reparent, refs); free(refs->path); } diff --git a/refs/reftable-backend.c b/refs/reftable-backend.c index 028f0211af3d0f..08a75fb3287502 100644 --- a/refs/reftable-backend.c +++ b/refs/reftable-backend.c @@ -391,8 +391,7 @@ static const struct reftable_be_write_options *reftable_be_write_options(struct return opts; } -static void reftable_be_reparent(const char *name UNUSED, - const char *old_cwd, +static void reftable_be_reparent(const char *old_cwd, const char *new_cwd, void *payload) { @@ -465,7 +464,7 @@ static struct ref_store *reftable_be_init(struct repository *repo, goto done; } - chdir_notify_register(NULL, reftable_be_reparent, refs); + chdir_notify_register(reftable_be_reparent, refs); done: assert(refs->err != REFTABLE_API_ERROR); @@ -492,7 +491,7 @@ static void reftable_be_release(struct ref_store *ref_store) free(be); } strmap_clear(&refs->worktree_backends, 0); - chdir_notify_unregister(NULL, reftable_be_reparent, refs); + chdir_notify_unregister(reftable_be_reparent, refs); } static int reftable_be_create_on_disk(struct ref_store *ref_store, diff --git a/reftable/block.c b/reftable/block.c index 1fa81405d2680f..de5af03c3b7bb3 100644 --- a/reftable/block.c +++ b/reftable/block.c @@ -87,7 +87,10 @@ int block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *block, REFTABLE_CALLOC_ARRAY(bw->zstream, 1); if (!bw->zstream) return REFTABLE_OUT_OF_MEMORY_ERROR; - deflateInit(bw->zstream, 9); + if (deflateInit(bw->zstream, 9) != Z_OK) { + REFTABLE_FREE_AND_NULL(bw->zstream); + return REFTABLE_ZLIB_ERROR; + } } return 0; diff --git a/reftable/writer.c b/reftable/writer.c index d969a6a0210e8b..073b9bbd8924ef 100644 --- a/reftable/writer.c +++ b/reftable/writer.c @@ -150,6 +150,7 @@ int reftable_writer_new(struct reftable_writer **out, { struct reftable_write_options opts = {0}; struct reftable_writer *wp; + int err; if (_opts) opts = *_opts; @@ -177,7 +178,12 @@ int reftable_writer_new(struct reftable_writer **out, wp->opts = opts; wp->hash_id = hash_id; wp->flush = flush_func; - writer_reinit_block_writer(wp, REFTABLE_BLOCK_TYPE_REF); + err = writer_reinit_block_writer(wp, REFTABLE_BLOCK_TYPE_REF); + if (err < 0) { + reftable_free(wp->block); + reftable_free(wp); + return err; + } *out = wp; diff --git a/repack-filtered.c b/repack-filtered.c index edcf7667c5c378..869b9fc6e3b94c 100644 --- a/repack-filtered.c +++ b/repack-filtered.c @@ -3,6 +3,12 @@ #include "repository.h" #include "run-command.h" #include "string-list.h" +#include "hex.h" +#include "packfile.h" +#include "list-objects-filter-options.h" +#include "list-objects-filter.h" +#include "odb.h" +#include "promisor-remote.h" int write_filtered_pack(const struct write_pack_opts *opts, struct existing_packs *existing, @@ -49,3 +55,79 @@ int write_filtered_pack(const struct write_pack_opts *opts, return finish_pack_objects_cmd(existing->repo->hash_algo, opts, &cmd, names); } + +struct collect_cb_data { + struct repository *repo; + struct oidset *set; +}; + +static int collect_promisor_blob(const struct object_id *oid, + struct object_info *oi UNUSED, + void *cb_data) +{ + struct collect_cb_data *data = cb_data; + struct object_info info = OBJECT_INFO_INIT; + enum object_type type; + + info.typep = &type; + + /* + * Use OBJECT_INFO_SKIP_FETCH_OBJECT to avoid triggering a + * lazy fetch while collecting promisor blobs. + */ + if (odb_read_object_info_extended(data->repo->objects, oid, &info, + OBJECT_INFO_SKIP_FETCH_OBJECT) < 0) + return 0; + + if (type == OBJ_BLOB) + oidset_insert(data->set, oid); + + return 0; +} + +int enumerate_promisor_blobs(struct repository *repo, + const struct list_objects_filter_options *filter, + struct oidset *to_drop) +{ + struct oidset all_promisor_blobs = OIDSET_INIT; + struct collect_cb_data cb = { + .repo = repo, + .set = &all_promisor_blobs + }; + int ret = 0; + + /* + * The caller (cmd_repack) is responsible for validating that a + * blob:limit filter and a promisor remote are present before + * calling this function. + * + * Walk only promisor objects. Every object visited here is a + * promisor object, so it is recoverable from the promisor remote + * as long as the remote still has it, the same assumption the rest + * of partial clone relies on. + * + * We do not use write_filtered_pack() here because git repack + * routes promisor objects through repack_promisor_objects() + * before the filter machinery runs, so the filtered pack never + * contains promisor blobs. Direct enumeration via + * ODB_FOR_EACH_OBJECT_PROMISOR_ONLY is the correct approach. + */ + ret = odb_for_each_object(repo->objects, NULL, + collect_promisor_blob, &cb, + ODB_FOR_EACH_OBJECT_PROMISOR_ONLY); + if (ret) + goto cleanup; + + /* + * Apply the filter to find which blobs exceed the threshold. + * The caller has to_drop and is responsible for clearing it. + */ + ret = list_objects_filter__filter_oidset(repo, + filter, + &all_promisor_blobs, + to_drop); + +cleanup: + oidset_clear(&all_promisor_blobs); + return ret; +} diff --git a/repack-promisor.c b/repack-promisor.c index 90318ce15093f5..fabfdc168a9b86 100644 --- a/repack-promisor.c +++ b/repack-promisor.c @@ -6,10 +6,12 @@ #include "path.h" #include "repository.h" #include "run-command.h" +#include "oidset.h" struct write_oid_context { struct child_process *cmd; const struct git_hash_algo *algop; + const struct oidset *to_drop; }; /* @@ -23,6 +25,15 @@ static int write_oid(const struct object_id *oid, struct write_oid_context *ctx = data; struct child_process *cmd = ctx->cmd; + /* + * Objects in to_drop are being removed from the repository, so + * omit them from the rebuilt promisor pack. Each such object is a + * promisor object and therefore remains recoverable from the + * promisor remote. + */ + if (ctx->to_drop && oidset_contains(ctx->to_drop, oid)) + return 0; + if (cmd->in == -1) { if (start_command(cmd)) die(_("could not start pack-objects to repack promisor objects")); @@ -81,7 +92,8 @@ static void finish_repacking_promisor_objects(struct repository *repo, void repack_promisor_objects(struct repository *repo, const struct pack_objects_args *args, - struct string_list *names, const char *packtmp) + struct string_list *names, const char *packtmp, + const struct oidset *to_drop) { struct write_oid_context ctx; struct child_process cmd = CHILD_PROCESS_INIT; @@ -98,6 +110,7 @@ void repack_promisor_objects(struct repository *repo, */ ctx.cmd = &cmd; ctx.algop = repo->hash_algo; + ctx.to_drop = to_drop; odb_for_each_object(repo->objects, NULL, write_oid, &ctx, ODB_FOR_EACH_OBJECT_PROMISOR_ONLY); diff --git a/repack.h b/repack.h index f9fbc895f02940..61e554e4ed383d 100644 --- a/repack.h +++ b/repack.h @@ -3,6 +3,7 @@ #include "list-objects-filter-options.h" #include "string-list.h" +#include "oidset.h" struct pack_objects_args { char *window; @@ -100,7 +101,8 @@ void generated_pack_install(struct generated_pack *pack, const char *name, void repack_promisor_objects(struct repository *repo, const struct pack_objects_args *args, - struct string_list *names, const char *packtmp); + struct string_list *names, const char *packtmp, + const struct oidset *to_drop); struct pack_geometry { struct packed_git **pack; @@ -165,6 +167,10 @@ int write_filtered_pack(const struct write_pack_opts *opts, struct existing_packs *existing, struct string_list *names); +int enumerate_promisor_blobs(struct repository *repo, + const struct list_objects_filter_options *filter, + struct oidset *to_drop); + int write_cruft_pack(const struct write_pack_opts *opts, const char *cruft_expiration, unsigned long combine_cruft_below_size, diff --git a/replay.c b/replay.c index 463c900d6c7c56..1e1bc7c10a8ffe 100644 --- a/replay.c +++ b/replay.c @@ -254,9 +254,9 @@ static void set_up_replay_mode(struct repository *repo, strset_clear(&rinfo.positive_refs); } -static struct commit *mapped_commit(kh_oid_map_t *replayed_commits, - struct commit *commit, - struct commit *fallback) +static struct commit *get_mapped_commit(kh_oid_map_t *replayed_commits, + struct commit *commit, + struct commit *fallback) { khint_t pos; if (!commit) @@ -267,27 +267,36 @@ static struct commit *mapped_commit(kh_oid_map_t *replayed_commits, return kh_value(replayed_commits, pos); } +static void put_mapped_commit(kh_oid_map_t *replayed_commits, + struct commit *commit, + struct commit *new_commit) +{ + khint_t pos; + int ret; + + pos = kh_put_oid_map(replayed_commits, commit->object.oid, &ret); + if (ret == 0) + BUG("Duplicate rewritten commit: %s", + oid_to_hex(&commit->object.oid)); + + kh_value(replayed_commits, pos) = new_commit; +} + static struct commit *pick_regular_commit(struct repository *repo, struct commit *pickme, - kh_oid_map_t *replayed_commits, - struct commit *onto, + struct commit *replayed_base, struct merge_options *merge_opt, struct merge_result *result, enum replay_mode mode, enum replay_empty_commit_action empty) { - struct commit *base, *replayed_base; struct tree *pickme_tree, *base_tree, *replayed_base_tree; - if (pickme->parents) { - base = pickme->parents->item; - base_tree = repo_get_commit_tree(repo, base); - } else { - base = NULL; + if (pickme->parents) + base_tree = repo_get_commit_tree(repo, pickme->parents->item); + else base_tree = lookup_tree(repo, repo->hash_algo->empty_tree); - } - replayed_base = mapped_commit(replayed_commits, base, onto); replayed_base_tree = repo_get_commit_tree(repo, replayed_base); pickme_tree = repo_get_commit_tree(repo, pickme); @@ -395,6 +404,12 @@ int replay_revisions(struct rev_info *revs, set_up_replay_mode(revs->repo, &revs->cmdline, opts->onto, &detached_head, &advance, &revert, &onto, &update_refs); + if (opts->linearize && + update_refs && strset_get_size(update_refs) > 1) { + ret = error(_("'--linearize' cannot be used with multiple revision ranges")); + goto out; + } + if (opts->ref) { struct object_id oid; @@ -427,24 +442,46 @@ int replay_revisions(struct rev_info *revs, replayed_commits = kh_init_oid_map(); while ((commit = get_revision(revs))) { const struct name_decoration *decoration; - khint_t pos; - int hr; - if (commit->parents && commit->parents->next) - die(_("replaying merge commits is not supported yet!")); + if (commit->parents && commit->parents->next) { + if (!opts->linearize) + die(_("replaying merge commits is not supported yet!")); + /* + * Drop the merge commit: do not pick it, leave + * `last_commit` unchanged, and fall through to the + * rest of the loop. As a result: + * - refs pointing to the merge commit will be updated + * to `last_commit`. + * - the next replayed commit uses `last_commit` as its + * `base`. + */ + } else { + /* + * Decide where to replay this commit onto. + * If the parent commit was replayed already, the replayed result + * can be found in `replayed_commits`. Otherwise fall back to `onto`. + * When reverting, commits are replayed in reverse order and thus + * its parent isn't replayed yet. Therefore revert commits are + * always replayed onto `last_commit`. + * Also when opts->linearize is true, set the base to + * `last_commit` to create a single linear history. + */ + struct commit *parent = commit->parents ? commit->parents->item : NULL; + struct commit *base = get_mapped_commit(replayed_commits, parent, onto); + + if (opts->linearize || mode == REPLAY_MODE_REVERT) + base = last_commit; + + last_commit = pick_regular_commit(revs->repo, commit, base, + &merge_opt, &result, + mode, opts->empty); + } - last_commit = pick_regular_commit(revs->repo, commit, replayed_commits, - mode == REPLAY_MODE_REVERT ? last_commit : onto, - &merge_opt, &result, mode, opts->empty); if (!last_commit) break; /* Record commit -> last_commit mapping */ - pos = kh_put_oid_map(replayed_commits, commit->object.oid, &hr); - if (hr == 0) - BUG("Duplicate rewritten commit: %s\n", - oid_to_hex(&commit->object.oid)); - kh_value(replayed_commits, pos) = last_commit; + put_mapped_commit(replayed_commits, commit, last_commit); /* Update any necessary branches */ if (ref) diff --git a/replay.h b/replay.h index 491db145e30151..2c71afbfde05b4 100644 --- a/replay.h +++ b/replay.h @@ -62,6 +62,11 @@ struct replay_revisions_options { * Defaults to REPLAY_EMPTY_COMMIT_DROP. */ enum replay_empty_commit_action empty; + + /* + * Whether to linearize the commits (i.e. drop merge commits). + */ + int linearize; }; /* This struct is used as an out-parameter by `replay_revisions()`. */ diff --git a/repo-settings.c b/repo-settings.c index f3be3b8c5a3d09..c3015356ba81e6 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -77,6 +77,7 @@ void prepare_repo_settings(struct repository *r) repo_cfg_bool(r, "pack.usesparse", &r->settings.pack_use_sparse, 1); repo_cfg_bool(r, "pack.usepathwalk", &r->settings.pack_use_path_walk, 0); repo_cfg_bool(r, "core.multipackindex", &r->settings.core_multi_pack_index, 1); + repo_cfg_bool(r, "core.diffhunks", &r->settings.core_diff_hunks, 1); repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 0); repo_cfg_bool(r, "index.skiphash", &r->settings.index_skip_hash, r->settings.index_skip_hash); repo_cfg_bool(r, "pack.readreverseindex", &r->settings.pack_read_reverse_index, 1); diff --git a/repo-settings.h b/repo-settings.h index e5253ead025c83..615a55cac49b1c 100644 --- a/repo-settings.h +++ b/repo-settings.h @@ -22,6 +22,7 @@ struct repo_settings { int core_commit_graph; int commit_graph_generation_version; int commit_graph_changed_paths_version; + int core_diff_hunks; int gc_write_commit_graph; int fetch_write_commit_graph; int command_requires_full_index; diff --git a/repository.c b/repository.c index 651b0f69339452..ccdf7d63869b57 100644 --- a/repository.c +++ b/repository.c @@ -5,6 +5,7 @@ #include "odb.h" #include "odb/source.h" #include "config.h" +#include "diff-provider.h" #include "gettext.h" #include "object.h" #include "lockfile.h" @@ -74,6 +75,7 @@ void initialize_repository(struct repository *repo) index_state_init(repo->index, repo); repo->check_deprecated_config = true; repo->bare_cfg = -1; + repo->fetch_if_missing = 1; repo_config_values_init(&repo->config_values_private_); /* @@ -201,8 +203,6 @@ void repo_set_compat_hash_algo(struct repository *repo MAYBE_UNUSED, uint32_t al if (hash_algo_by_ptr(repo->hash_algo) == algo) BUG("hash_algo and compat_hash_algo match"); repo->compat_hash_algo = algo ? &hash_algos[algo] : NULL; - if (repo->compat_hash_algo) - repo_read_loose_object_map(repo); #else if (algo) die(_("compatibility hash algorithm support requires Rust")); @@ -296,6 +296,7 @@ int repo_init(struct repository *repo, warning("%s", err.buf); goto error; } + repo->objects = odb_new(repo, 0); if (worktree) repo_set_worktree(repo, worktree); @@ -383,6 +384,8 @@ void repo_clear(struct repository *repo) FREE_AND_NULL(repo->submodule_prefix); FREE_AND_NULL(repo->ref_storage_payload); + diff_providers_clear(repo); + odb_free(repo->objects); repo->objects = NULL; diff --git a/repository.h b/repository.h index 3b467a25133670..a33704c78319dc 100644 --- a/repository.h +++ b/repository.h @@ -7,6 +7,7 @@ #include "environment.h" struct config_set; +struct diff_provider; struct git_hash_algo; struct index_state; struct lock_file; @@ -161,6 +162,13 @@ struct repository { /* Repository's remotes and associated structures. */ struct remote_state *remote_state; + /* + * The repository's diff hunk provider chain, NULL until the + * first consultation builds it (diff-provider.c); repo_clear() + * releases it. + */ + struct diff_provider *diff_providers; + /* Repository's current hash algorithm, as serialized on disk. */ const struct git_hash_algo *hash_algo; @@ -184,6 +192,12 @@ struct repository { /* True if commit-graph has been disabled within this process. */ int commit_graph_disabled; + /* + * Controls whether the repository should lazily fetch missing + * objects from promisor remotes. Defaults to 1. + */ + int fetch_if_missing; + /* * Lazily-populated cache mapping hook event names to configured hooks. * NULL until first hook use. diff --git a/revision.c b/revision.c index 50dc8b199137c7..32efcd20a27d09 100644 --- a/revision.c +++ b/revision.c @@ -750,7 +750,9 @@ static int check_maybe_different_in_bloom_filter(struct rev_info *revs, struct commit *commit) { struct bloom_filter *filter; - int result = 0; + + if (!revs->bloom_keyvecs_nr) + return -1; if (commit_graph_generation(commit) == GENERATION_NUMBER_INFINITY) return -1; @@ -762,18 +764,44 @@ static int check_maybe_different_in_bloom_filter(struct rev_info *revs, return -1; } - for (size_t nr = 0; !result && nr < revs->bloom_keyvecs_nr; nr++) { - result = bloom_filter_contains_vec(filter, - revs->bloom_keyvecs[nr], - revs->bloom_filter_settings); + if (revs_maybe_changed_in_bloom(revs, filter)) { + count_bloom_filter_maybe++; + return 1; } - if (result) - count_bloom_filter_maybe++; - else - count_bloom_filter_definitely_not++; + count_bloom_filter_definitely_not++; + + return 0; +} + +bool revs_maybe_changed_in_bloom(struct rev_info *revs, + struct bloom_filter *filter) +{ + if (!revs->bloom_keyvecs_nr || !filter) + return true; + + for (size_t nr = 0; nr < revs->bloom_keyvecs_nr; nr++) + if (bloom_filter_contains_vec(filter, + revs->bloom_keyvecs[nr], + revs->bloom_filter_settings)) + return true; + + return false; +} + +bool revs_maybe_changed_in_bloom_with_parents(struct rev_info *revs, + struct bloom_filter *filter) +{ + if (!revs->bloom_keyvecs_nr || !filter) + return true; + + for (size_t nr = 0; nr < revs->bloom_keyvecs_nr; nr++) + if (bloom_filter_contains_any_vec(filter, + revs->bloom_keyvecs[nr], + revs->bloom_filter_settings)) + return true; - return result; + return false; } static int rev_compare_tree(struct rev_info *revs, @@ -806,7 +834,7 @@ static int rev_compare_tree(struct rev_info *revs, return REV_TREE_SAME; } - if (revs->bloom_keyvecs_nr && !nth_parent) { + if (!nth_parent) { bloom_ret = check_maybe_different_in_bloom_filter(revs, commit); if (bloom_ret == 0) @@ -833,7 +861,7 @@ static int rev_same_tree_as_empty(struct rev_info *revs, struct commit *commit, if (!t1) return 0; - if (!nth_parent && revs->bloom_keyvecs_nr) { + if (!nth_parent) { bloom_ret = check_maybe_different_in_bloom_filter(revs, commit); if (!bloom_ret) return 1; @@ -2732,7 +2760,7 @@ static int handle_revision_opt(struct rev_info *revs, int argc, const char **arg revs->ignore_missing = 1; } else if (opt && opt->allow_exclude_promisor_objects && !strcmp(arg, "--exclude-promisor-objects")) { - if (fetch_if_missing) + if (revs->repo->fetch_if_missing) BUG("exclude_promisor_objects can only be used when fetch_if_missing is 0"); revs->exclude_promisor_objects = 1; } else { @@ -3229,8 +3257,10 @@ int setup_revisions(int argc, const char **argv, struct rev_info *revs, struct s (revs->diffopt.output_format & ~(DIFF_FORMAT_PATCH | DIFF_FORMAT_NO_OUTPUT | DIFF_FORMAT_RAW | DIFF_FORMAT_NAME | - DIFF_FORMAT_NAME_STATUS | DIFF_FORMAT_SUMMARY)))) - die(_("-L does not yet support the requested diff format")); + DIFF_FORMAT_NAME_STATUS | DIFF_FORMAT_SUMMARY | + DIFF_FORMAT_NUMSTAT | DIFF_FORMAT_DIFFSTAT | + DIFF_FORMAT_SHORTSTAT | DIFF_FORMAT_CHECKDIFF)))) + die(_("-L does not support the requested diff format")); if (revs->expand_tabs_in_log < 0) revs->expand_tabs_in_log = revs->expand_tabs_in_log_default; diff --git a/revision.h b/revision.h index acf6d06b24126c..192001ff799f9c 100644 --- a/revision.h +++ b/revision.h @@ -68,6 +68,7 @@ struct string_list; struct saved_parents; struct follow_pathspec_slab; struct bloom_keyvec; +struct bloom_filter; struct bloom_filter_settings; struct option; struct parse_opt_ctx_t; @@ -495,6 +496,25 @@ void reset_revision_walk(void); */ int prepare_revision_walk(struct rev_info *revs); +/** + * Consult a changed-path Bloom filter to determine if the commit to which the + * filter belongs might have changed any of the paths in the `revs`. + * prepare_revision_walk() needs to be called in advance to ensure + * pathspec key vectors are set up. + * + * Returns false iff the commit definitely did not change any of the paths. + */ +bool revs_maybe_changed_in_bloom(struct rev_info *revs, + struct bloom_filter *filter); + +/** + * Same as revs_maybe_changed_in_bloom(), but a change to any of the directories + * leading up to a path counts as well. Callers that track the tree entries + * containing the paths, and not just the paths themselves, need this. + */ +bool revs_maybe_changed_in_bloom_with_parents(struct rev_info *revs, + struct bloom_filter *filter); + /* Drain the commits linked list into the priority queue. */ void rev_info_commit_list_to_queue(struct rev_info *revs); /** diff --git a/send-pack.c b/send-pack.c index 3bb5afc687aaf0..f20460fbf487bf 100644 --- a/send-pack.c +++ b/send-pack.c @@ -42,16 +42,17 @@ int option_parse_push_signed(const struct option *opt, die("bad %s argument: %s", opt->long_name, arg); } -static void feed_object(struct repository *r, - const struct object_id *oid, FILE *fh, int negative) +static void append_negative_object(struct repository *r, + struct oid_array *haves, + const struct object_id *oid) { - if (negative && !odb_has_object(r->objects, oid, 0)) + /* + * The remote end may have advertised objects that we do not have in + * our object database. Skip those, as we cannot use them as boundary. + */ + if (!odb_has_object(r->objects, oid, 0)) return; - - if (negative) - putc('^', fh); - fputs(oid_to_hex(oid), fh); - putc('\n', fh); + oid_array_append(haves, oid); } /* @@ -62,92 +63,58 @@ static int pack_objects(struct repository *r, struct oid_array *negotiated, struct send_pack_args *args) { - /* - * The child becomes pack-objects --revs; we feed - * the revision parameters to it via its stdin and - * let its stdout go back to the other end. - */ - struct child_process po = CHILD_PROCESS_INIT; - FILE *po_in; + struct odb_generate_pack_options opts = ODB_GENERATE_PACK_OPTIONS_INIT; + struct odb_pack_generator *generator; int rc; trace2_region_enter("send_pack", "pack_objects", r); - strvec_push(&po.args, "pack-objects"); - strvec_push(&po.args, "--all-progress-implied"); - strvec_push(&po.args, "--revs"); - strvec_push(&po.args, "--stdout"); - if (args->use_thin_pack) - strvec_push(&po.args, "--thin"); - if (args->use_ofs_delta) - strvec_push(&po.args, "--delta-base-offset"); - if (args->quiet || !args->progress) - strvec_push(&po.args, "-q"); + + opts.thin = args->use_thin_pack; + opts.ofs_delta = args->use_ofs_delta; if (args->progress) - strvec_push(&po.args, "--progress"); - if (is_repository_shallow(r)) - strvec_push(&po.args, "--shallow"); - if (args->disable_bitmaps) - strvec_push(&po.args, "--no-use-bitmap-index"); - po.in = -1; - po.out = args->stateless_rpc ? -1 : fd; - po.git_cmd = 1; - po.clean_on_exit = 1; - if (start_command(&po)) - die_errno("git pack-objects failed"); + opts.progress = ODB_GENERATE_PACK_PROGRESS_VERBOSE; + opts.shallow = is_repository_shallow(r); + opts.disable_bitmaps = args->disable_bitmaps; /* - * We feed the pack-objects we just spawned with revision - * parameters by writing to the pipe. + * The pack is either written directly to the remote's descriptor, or, + * in the case of a stateless RPC, read back from a pipe so that we + * can wrap the pack data into pkt-lines. */ - po_in = xfdopen(po.in, "w"); + opts.pack_fd = args->stateless_rpc ? -1 : fd; + for (size_t i = 0; i < advertised->nr; i++) - feed_object(r, &advertised->oid[i], po_in, 1); + append_negative_object(r, &opts.haves, &advertised->oid[i]); for (size_t i = 0; i < negotiated->nr; i++) - feed_object(r, &negotiated->oid[i], po_in, 1); + append_negative_object(r, &opts.haves, &negotiated->oid[i]); while (refs) { if (!is_null_oid(&refs->old_oid)) - feed_object(r, &refs->old_oid, po_in, 1); + append_negative_object(r, &opts.haves, &refs->old_oid); if (!is_null_oid(&refs->new_oid)) - feed_object(r, &refs->new_oid, po_in, 0); + oid_array_append(&opts.wants, &refs->new_oid); refs = refs->next; } - fflush(po_in); - if (ferror(po_in)) - die_errno("error writing to pack-objects"); - fclose(po_in); + if (odb_generate_pack(r->objects, &generator, &opts)) + die("git pack-objects failed"); + odb_generate_pack_options_release(&opts); if (args->stateless_rpc) { char *buf = xmalloc(LARGE_PACKET_MAX); while (1) { - ssize_t n = xread(po.out, buf, LARGE_PACKET_MAX); + ssize_t n = xread(generator->out, buf, LARGE_PACKET_MAX); if (n <= 0) break; send_sideband(fd, -1, buf, n, LARGE_PACKET_MAX); } free(buf); - close(po.out); - po.out = -1; + close(generator->out); } - rc = finish_command(&po); - if (rc) { - /* - * For a normal non-zero exit, we assume pack-objects wrote - * something useful to stderr. For death by signal, though, - * we should mention it to the user. The exception is SIGPIPE - * (141), because that's a normal occurrence if the remote end - * hangs up (and we'll report that by trying to read the unpack - * status). - */ - if (rc > 128 && rc != 141) - error("pack-objects died of signal %d", rc - 128); - trace2_region_leave("send_pack", "pack_objects", r); - return -1; - } + rc = odb_pack_generator_finish(generator); trace2_region_leave("send_pack", "pack_objects", r); - return 0; + return rc; } static int receive_unpack_status(struct packet_reader *reader) @@ -768,7 +735,7 @@ int send_pack(struct repository *r, goto out; } if (!args->stateless_rpc) - /* Closed by pack_objects() via start_command() */ + /* Consumed by the pack generator in pack_objects() */ fd[1] = -1; } if (args->stateless_rpc && cmds_sent) diff --git a/sequencer.c b/sequencer.c index 83c38492052b46..4bcb9f463a2aae 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1127,6 +1127,7 @@ static int run_git_commit(const char *defmsg, struct child_process cmd = CHILD_PROCESS_INIT; cmd.git_cmd = 1; + cmd.odb_to_close = the_repository->objects; if (is_rebase_i(opts) && ((opts->committer_date_is_author_date && !opts->ignore_date) || @@ -1880,18 +1881,38 @@ static int is_pick_or_similar(enum todo_command command) } } -enum todo_item_flags { - TODO_EDIT_MERGE_MSG = (1 << 0), - TODO_REPLACE_FIXUP_MSG = (1 << 1), - TODO_EDIT_FIXUP_MSG = (1 << 2), -}; - static const char first_commit_msg_str[] = N_("This is the 1st commit message:"); static const char nth_commit_msg_fmt[] = N_("This is the commit message #%d:"); static const char skip_first_commit_msg_str[] = N_("The 1st commit message will be skipped:"); static const char skip_nth_commit_msg_fmt[] = N_("The commit message #%d will be skipped:"); static const char combined_commit_msg_fmt[] = N_("This is a combination of %d commits."); +void add_squash_combination_header(struct strbuf *buf, int n) +{ + strbuf_addf(buf, "%s ", comment_line_str); + strbuf_addf(buf, _(combined_commit_msg_fmt), n); +} + +void add_squash_message_header(struct strbuf *buf, int n, int skip) +{ + strbuf_addf(buf, "%s ", comment_line_str); + if (n == 1) + strbuf_addstr(buf, skip ? _(skip_first_commit_msg_str) : + _(first_commit_msg_str)); + else + strbuf_addf(buf, skip ? _(skip_nth_commit_msg_fmt) : + _(nth_commit_msg_fmt), n); +} + +size_t squash_subject_comment_len(const char *body, int squashing) +{ + if (starts_with(body, "amend!") || + (squashing && (starts_with(body, "squash!") || + starts_with(body, "fixup!")))) + return commit_subject_length(body); + return 0; +} + static int is_fixup_flag(enum todo_command command, unsigned flag) { return command == TODO_FIXUP && ((flag & TODO_REPLACE_FIXUP_MSG) || @@ -2012,20 +2033,13 @@ static int append_squash_message(struct strbuf *buf, const char *body, { struct replay_ctx *ctx = opts->ctx; const char *fixup_msg; - size_t commented_len = 0, fixup_off; - /* - * amend is non-interactive and not normally used with fixup! - * or squash! commits, so only comment out those subjects when - * squashing commit messages. - */ - if (starts_with(body, "amend!") || - ((command == TODO_SQUASH || seen_squash(ctx)) && - (starts_with(body, "squash!") || starts_with(body, "fixup!")))) - commented_len = commit_subject_length(body); + size_t commented_len, fixup_off; - strbuf_addf(buf, "\n%s ", comment_line_str); - strbuf_addf(buf, _(nth_commit_msg_fmt), - ++ctx->current_fixup_count + 1); + commented_len = squash_subject_comment_len(body, + command == TODO_SQUASH || seen_squash(ctx)); + + strbuf_addch(buf, '\n'); + add_squash_message_header(buf, ++ctx->current_fixup_count + 1, 0); strbuf_addstr(buf, "\n\n"); strbuf_add_commented_lines(buf, body, commented_len, comment_line_str); /* buf->buf may be reallocated so store an offset into the buffer */ @@ -2090,9 +2104,8 @@ static int update_squash_messages(struct repository *r, eol = !starts_with(buf.buf, comment_line_str) ? buf.buf : strchrnul(buf.buf, '\n'); - strbuf_addf(&header, "%s ", comment_line_str); - strbuf_addf(&header, _(combined_commit_msg_fmt), - ctx->current_fixup_count + 2); + add_squash_combination_header(&header, + ctx->current_fixup_count + 2); strbuf_splice(&buf, 0, eol - buf.buf, header.buf, header.len); strbuf_release(&header); if (is_fixup_flag(command, flag) && !seen_squash(ctx)) @@ -2116,12 +2129,9 @@ static int update_squash_messages(struct repository *r, repo_unuse_commit_buffer(r, head_commit, head_message); return error(_("cannot write '%s'"), rebase_path_fixup_msg()); } - strbuf_addf(&buf, "%s ", comment_line_str); - strbuf_addf(&buf, _(combined_commit_msg_fmt), 2); - strbuf_addf(&buf, "\n%s ", comment_line_str); - strbuf_addstr(&buf, is_fixup_flag(command, flag) ? - _(skip_first_commit_msg_str) : - _(first_commit_msg_str)); + add_squash_combination_header(&buf, 2); + strbuf_addch(&buf, '\n'); + add_squash_message_header(&buf, 1, is_fixup_flag(command, flag)); strbuf_addstr(&buf, "\n\n"); if (is_fixup_flag(command, flag)) strbuf_add_commented_lines(&buf, body, strlen(body), @@ -2140,9 +2150,8 @@ static int update_squash_messages(struct repository *r, if (command == TODO_SQUASH || is_fixup_flag(command, flag)) { res = append_squash_message(&buf, body, command, opts, flag); } else if (command == TODO_FIXUP) { - strbuf_addf(&buf, "\n%s ", comment_line_str); - strbuf_addf(&buf, _(skip_nth_commit_msg_fmt), - ++ctx->current_fixup_count + 1); + strbuf_addch(&buf, '\n'); + add_squash_message_header(&buf, ++ctx->current_fixup_count + 1, 1); strbuf_addstr(&buf, "\n\n"); strbuf_add_commented_lines(&buf, body, strlen(body), comment_line_str); @@ -2223,6 +2232,25 @@ static int should_edit(struct replay_opts *opts) { return opts->edit; } +static int should_edit_rebase_continue(struct replay_opts *opts) +{ + if (opts->edit < 0) + return 1; + return opts->edit; +} + +static void finalize_continue_edit_flags(struct replay_opts *opts, + unsigned int *flags) +{ + if (*flags & CLEANUP_MSG) + return; + + if (should_edit_rebase_continue(opts)) + *flags |= EDIT_MSG; + else + *flags &= ~EDIT_MSG; +} + static void refer_to_commit(struct repository *r, struct strbuf *msgbuf, const struct commit *commit, bool use_commit_reference) @@ -4796,7 +4824,8 @@ void create_autostash_ref(struct repository *r, const char *refname, static int apply_save_autostash_oid(const char *stash_oid, int attempt_apply, const char *label_ours, const char *label_theirs, const char *label_base, - const char *stash_msg) + const char *stash_msg, + bool *conflicted) { struct child_process child = CHILD_PROCESS_INIT; int ret = 0; @@ -4831,14 +4860,16 @@ static int apply_save_autostash_oid(const char *stash_oid, int attempt_apply, strvec_push(&store.args, stash_oid); if (run_command(&store)) ret = error(_("cannot store %s"), stash_oid); - else if (attempt_apply) + else if (attempt_apply) { + if (conflicted) + *conflicted = true; fprintf(stderr, _("Your local changes are stashed, however applying them\n" "resulted in conflicts. You can either resolve the conflicts\n" "and then discard the stash with \"git stash drop\", or, if you\n" "do not want to resolve them now, run \"git reset --hard\" and\n" "apply the local changes later by running \"git stash pop\".\n")); - else + } else fprintf(stderr, _("Autostash exists; creating a new stash entry.\n" "Your changes are safe in the stash.\n" @@ -4862,7 +4893,7 @@ static int apply_save_autostash(const char *path, int attempt_apply) strbuf_trim(&stash_oid); ret = apply_save_autostash_oid(stash_oid.buf, attempt_apply, - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); unlink(path); strbuf_release(&stash_oid); @@ -4881,19 +4912,24 @@ int apply_autostash(const char *path) int apply_autostash_oid(const char *stash_oid) { - return apply_save_autostash_oid(stash_oid, 1, NULL, NULL, NULL, NULL); + return apply_save_autostash_oid(stash_oid, 1, NULL, NULL, NULL, NULL, + NULL); } static int apply_save_autostash_ref(struct repository *r, const char *refname, int attempt_apply, const char *label_ours, const char *label_theirs, const char *label_base, - const char *stash_msg) + const char *stash_msg, + bool *conflicted) { struct object_id stash_oid; char stash_oid_hex[GIT_MAX_HEXSZ + 1]; int flag, ret; + if (conflicted) + *conflicted = false; + if (!refs_ref_exists(get_main_ref_store(r), refname)) return 0; @@ -4906,7 +4942,7 @@ static int apply_save_autostash_ref(struct repository *r, const char *refname, oid_to_hex_r(stash_oid_hex, &stash_oid); ret = apply_save_autostash_oid(stash_oid_hex, attempt_apply, label_ours, label_theirs, label_base, - stash_msg); + stash_msg, conflicted); refs_delete_ref(get_main_ref_store(r), "", refname, &stash_oid, REF_NO_DEREF); @@ -4917,16 +4953,17 @@ static int apply_save_autostash_ref(struct repository *r, const char *refname, int save_autostash_ref(struct repository *r, const char *refname) { return apply_save_autostash_ref(r, refname, 0, - NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); } int apply_autostash_ref(struct repository *r, const char *refname, const char *label_ours, const char *label_theirs, - const char *label_base, const char *stash_msg) + const char *label_base, const char *stash_msg, + bool *conflicted) { return apply_save_autostash_ref(r, refname, 1, label_ours, label_theirs, label_base, - stash_msg); + stash_msg, conflicted); } static int checkout_onto(struct repository *r, struct replay_opts *opts, @@ -5351,7 +5388,7 @@ static int commit_staged_changes(struct repository *r, struct todo_list *todo_list) { struct replay_ctx *ctx = opts->ctx; - unsigned int flags = ALLOW_EMPTY | EDIT_MSG; + unsigned int flags = ALLOW_EMPTY; unsigned int final_fixup = 0, is_clean; struct strbuf rev = STRBUF_INIT; const char *reflog_action = reflog_message(opts, "continue", NULL); @@ -5519,6 +5556,8 @@ static int commit_staged_changes(struct repository *r, } } + finalize_continue_edit_flags(opts, &flags); + if (run_git_commit(final_fixup ? NULL : rebase_path_message(), reflog_action, opts, flags)) { ret = error(_("could not commit staged changes.")); @@ -5576,6 +5615,12 @@ int sequencer_continue(struct repository *r, struct replay_opts *opts) res = -1; goto release_todo_list; } + + /* + * Command-line --[no-]edit applies only to this + * --continue invocation, not to subsequent picks. + */ + opts->edit = -1; } else if (!file_exists(get_todo_path(opts))) return continue_single_pick(r, opts); else if ((res = read_populate_todo(r, &todo_list, opts))) @@ -6277,7 +6322,6 @@ int sequencer_make_script(struct repository *r, struct strbuf *out, revs.sort_order = REV_SORT_IN_GRAPH_ORDER; revs.topo_order = 1; - revs.pretty_given = 1; repo_config_get_string(the_repository, "rebase.instructionFormat", &format); if (!format || !*format) { free(format); @@ -6540,34 +6584,65 @@ struct todo_add_branch_context { size_t items_alloc; struct strbuf *buf; struct string_list refs_to_oids; + struct string_list symref_update_targets; }; static int add_decorations_to_list(const struct commit *commit, struct todo_add_branch_context *ctx) { const struct name_decoration *decoration = get_name_decoration(&commit->object); - const char *head_ref = refs_resolve_ref_unsafe(get_main_ref_store(the_repository), - "HEAD", - RESOLVE_REF_READING, - NULL, - NULL); + struct ref_store *refs = get_main_ref_store(the_repository); + char *head_ref = refs_resolve_refdup(refs, "HEAD", + RESOLVE_REF_READING, + NULL, NULL); while (decoration) { struct todo_item *item; const char *path; + const char *checked_ref; + char *resolved_ref; + int flags = 0; size_t base_offset = ctx->buf->len; /* - * If the branch is the current HEAD, then it will be - * updated by the default rebase behavior. - * Exclude it from the list of refs to update, - * as well as any non-branch decorations. * Non-branch decorations may be present if the pretty format * includes "%d", which would have loaded all refs * into the global decoration table. */ - if ((head_ref && !strcmp(head_ref, decoration->name)) || - (decoration->type != DECORATION_REF_LOCAL)) { + if (decoration->type != DECORATION_REF_LOCAL) { + decoration = decoration->next; + continue; + } + + resolved_ref = refs_resolve_refdup(refs, decoration->name, + RESOLVE_REF_READING, + NULL, &flags); + if (resolved_ref && (flags & REF_ISSYMREF) && + starts_with(resolved_ref, "refs/heads/")) { + free(resolved_ref); + decoration = decoration->next; + continue; + } + + /* + * If the branch is the current HEAD, then it will be + * updated by the default rebase behavior. + */ + if (head_ref && !strcmp(head_ref, decoration->name)) { + free(resolved_ref); + decoration = decoration->next; + continue; + } + + path = branch_checked_out(decoration->name); + if (!path && resolved_ref && (flags & REF_ISSYMREF)) { + checked_ref = resolved_ref; + path = branch_checked_out(checked_ref); + } + if (!path && resolved_ref && (flags & REF_ISSYMREF) && + string_list_has_string(&ctx->symref_update_targets, + resolved_ref)) { + free(resolved_ref); decoration = decoration->next; continue; } @@ -6579,13 +6654,17 @@ static int add_decorations_to_list(const struct commit *commit, memset(item, 0, sizeof(*item)); /* If the branch is checked out, then leave a comment instead. */ - if ((path = branch_checked_out(decoration->name))) { + if (path) { item->command = TODO_COMMENT; strbuf_commented_addf(ctx->buf, comment_line_str, "Ref %s checked out at '%s'\n", decoration->name, path); } else { struct string_list_item *sti; + + if (resolved_ref && (flags & REF_ISSYMREF)) + string_list_insert(&ctx->symref_update_targets, + resolved_ref); item->command = TODO_UPDATE_REF; strbuf_addf(ctx->buf, "%s\n", decoration->name); @@ -6599,9 +6678,11 @@ static int add_decorations_to_list(const struct commit *commit, item->arg_len = ctx->buf->len - base_offset; ctx->items_nr++; + free(resolved_ref); decoration = decoration->next; } + free(head_ref); return 0; } @@ -6615,6 +6696,7 @@ static int todo_list_add_update_ref_commands(struct todo_list *todo_list) struct todo_add_branch_context ctx = { .buf = &todo_list->buf, .refs_to_oids = STRING_LIST_INIT_DUP, + .symref_update_targets = STRING_LIST_INIT_DUP, }; ctx.items_alloc = 2 * todo_list->nr + 1; @@ -6640,6 +6722,7 @@ static int todo_list_add_update_ref_commands(struct todo_list *todo_list) res = write_update_refs_state(&ctx.refs_to_oids); string_list_clear(&ctx.refs_to_oids, 1); + string_list_clear(&ctx.symref_update_targets, 0); if (res) { /* we failed, so clean up the new list. */ diff --git a/sequencer.h b/sequencer.h index 64a9c7fb1beccd..0f3bae12aa6b8f 100644 --- a/sequencer.h +++ b/sequencer.h @@ -119,6 +119,13 @@ enum todo_command { TODO_COMMENT }; +/* Bits for the "flags" member of struct todo_item */ +enum todo_item_flags { + TODO_EDIT_MERGE_MSG = (1 << 0), + TODO_REPLACE_FIXUP_MSG = (1 << 1), + TODO_EDIT_FIXUP_MSG = (1 << 2), +}; + struct todo_item { enum todo_command command; struct commit *commit; @@ -208,6 +215,29 @@ int todo_list_rearrange_squash(struct todo_list *todo_list); */ void append_signoff(struct strbuf *msgbuf, size_t ignore_footer, unsigned flag); +/* + * Append the "This is a combination of N commits." banner that "git rebase + * -i" writes at the top of a squashed commit's message, commented out with + * the comment character. + */ +void add_squash_combination_header(struct strbuf *buf, int n); + +/* + * Append the header (1-based N) that "git rebase -i" writes above each message + * when squashing, commented out with the comment character. With SKIP it reads + * "The ... commit message will be skipped" for a message that is dropped (a + * fixup), otherwise "This is the ... commit message". + */ +void add_squash_message_header(struct strbuf *buf, int n, int skip); + +/* + * Return the length of the leading subject of BODY when it should be commented + * out in a squash message, or 0 otherwise. An "amend!" subject always + * qualifies; "squash!" and "fixup!" subjects only when SQUASHING, since a + * plain fixup chain keeps them. + */ +size_t squash_subject_comment_len(const char *body, int squashing); + void append_conflicts_hint(struct index_state *istate, struct strbuf *msgbuf, enum commit_msg_cleanup_mode cleanup_mode); enum commit_msg_cleanup_mode get_cleanup_mode(const char *cleanup_arg, @@ -237,7 +267,8 @@ int apply_autostash(const char *path); int apply_autostash_oid(const char *stash_oid); int apply_autostash_ref(struct repository *r, const char *refname, const char *label_ours, const char *label_theirs, - const char *label_base, const char *stash_msg); + const char *label_base, const char *stash_msg, + bool *conflicted); #define SUMMARY_INITIAL_COMMIT (1 << 0) #define SUMMARY_SHOW_AUTHOR_DATE (1 << 1) diff --git a/serve.c b/serve.c index 2b07d922b3dde1..1b4369fb683965 100644 --- a/serve.c +++ b/serve.c @@ -46,6 +46,9 @@ static int promisor_remote_advertise(struct repository *r, static void promisor_remote_receive(struct repository *r, const char *remotes) { + if (!remotes) + die("promisor-remote capability requires an argument"); + mark_promisor_remotes_as_accepted(r, remotes); } @@ -97,9 +100,9 @@ static int object_info_advertise(struct repository *r, struct strbuf *value) /* disabled by default */ advertise_object_info = 0; } - /* Currently only size is supported */ + /* Currently only size and type are supported */ if (value && advertise_object_info) - strbuf_addstr(value, "size"); + strbuf_addstr(value, "size type"); return advertise_object_info; } diff --git a/setup.c b/setup.c index 95909e96031262..fb9233c0887741 100644 --- a/setup.c +++ b/setup.c @@ -1057,8 +1057,7 @@ static void apply_gitdir_and_environment(struct repository *repo, const char *pa strvec_clear(&to_free); } -static void update_relative_gitdir(const char *name UNUSED, - const char *old_cwd, +static void update_relative_gitdir(const char *old_cwd, const char *new_cwd, void *data) { @@ -1086,7 +1085,7 @@ static void apply_and_export_relative_gitdir(struct repository *repo, const char xsetenv(GIT_DIR_ENVIRONMENT, path, 1); if (!is_absolute_path(path)) - chdir_notify_register(NULL, update_relative_gitdir, repo); + chdir_notify_register(update_relative_gitdir, repo); strbuf_release(&realpath); } @@ -1339,6 +1338,65 @@ static int canonicalize_ceiling_entry(struct string_list_item *item, } } +void path_allowlist_apply(const char *key, const char *value, + const char *target_path, int *is_match, + bool allow_dot) +{ + char *allowed = NULL; + char *normalized = NULL; + + if (!value || !*value) { + *is_match = 0; + return; + } + + if (!strcmp(value, "*")) { + *is_match = 1; + return; + } + + if (git_config_pathname(&allowed, key, value) || !allowed) + return; + + /* + * Setting the config variable to a non-absolute path makes + * little sense---it won't be relative to the configuration + * file the item is defined in. Except for ".", which means + * "if we are at the top level of a repository, then it is + * OK", which is slightly tighter than "*" that allows + * discovery. + */ + if (!is_absolute_path(allowed) && (!allow_dot || strcmp(allowed, "."))) { + warning(_("%s '%s' not absolute"), key, allowed); + goto end; + } + + /* + * A .gitconfig in $HOME may be shared across different + * machines and the config variable entries may or may not + * exist as paths on all of these machines. In other words, + * it is not a warning worthy event when there is no such path + * on this machine---the entry may be useful elsewhere. + */ + normalized = real_pathdup(allowed, 0); + if (!normalized) + goto end; + + if (ends_with(normalized, "/*")) { + size_t len = strlen(normalized); + if (!fspathncmp(normalized, target_path, len - 1)) + *is_match = 1; + goto end; + } + + if (!fspathcmp(target_path, normalized)) + *is_match = 1; + +end: + free(normalized); + free(allowed); +} + struct safe_directory_data { char *path; int is_safe; @@ -1352,54 +1410,7 @@ static int safe_directory_cb(const char *key, const char *value, if (strcmp(key, "safe.directory")) return 0; - if (!value || !*value) { - data->is_safe = 0; - } else if (!strcmp(value, "*")) { - data->is_safe = 1; - } else { - char *allowed = NULL; - - if (!git_config_pathname(&allowed, key, value) && allowed) { - char *normalized = NULL; - - /* - * Setting safe.directory to a non-absolute path - * makes little sense---it won't be relative to - * the configuration file the item is defined in. - * Except for ".", which means "if we are at the top - * level of a repository, then it is OK", which is - * slightly tighter than "*" that allows discovery. - */ - if (!is_absolute_path(allowed) && strcmp(allowed, ".")) { - warning(_("safe.directory '%s' not absolute"), - allowed); - goto next; - } - - /* - * A .gitconfig in $HOME may be shared across - * different machines and safe.directory entries - * may or may not exist as paths on all of these - * machines. In other words, it is not a warning - * worthy event when there is no such path on this - * machine---the entry may be useful elsewhere. - */ - normalized = real_pathdup(allowed, 0); - if (!normalized) - goto next; - - if (ends_with(normalized, "/*")) { - size_t len = strlen(normalized); - if (!fspathncmp(normalized, data->path, len - 1)) - data->is_safe = 1; - } else if (!fspathcmp(data->path, normalized)) { - data->is_safe = 1; - } - next: - free(normalized); - free(allowed); - } - } + path_allowlist_apply(key, value, data->path, &data->is_safe, true); return 0; } @@ -1765,8 +1776,6 @@ int apply_repository_format(struct repository *repo, enum apply_repository_format_flags flags, struct strbuf *err) { - char *object_directory = NULL, *alternate_object_directories = NULL; - if (verify_repository_format(format, err) < 0) return -1; @@ -1779,8 +1788,6 @@ int apply_repository_format(struct repository *repo, if (flags & APPLY_REPOSITORY_FORMAT_HONOR_ENV) { const char *shallow_file; - object_directory = xstrdup_or_null(getenv(DB_ENVIRONMENT)); - alternate_object_directories = xstrdup_or_null(getenv(ALTERNATE_DB_ENVIRONMENT)); shallow_file = getenv(GIT_SHALLOW_FILE_ENVIRONMENT); if (shallow_file) set_alternate_shallow_file(repo, shallow_file); @@ -1788,8 +1795,6 @@ int apply_repository_format(struct repository *repo, repo->bare_cfg = format->is_bare; repo_set_hash_algo(repo, format->hash_algo); - repo->objects = odb_new(repo, object_directory, - alternate_object_directories); repo_set_compat_hash_algo(repo, format->compat_hash_algo); repo_set_ref_storage_format(repo, format->ref_storage_format, @@ -1805,8 +1810,6 @@ int apply_repository_format(struct repository *repo, repo->repository_format_precious_objects = format->precious_objects; - free(alternate_object_directories); - free(object_directory); return 0; } @@ -1890,6 +1893,7 @@ const char *enter_repo(struct repository *repo, const char *path, unsigned flags read_and_verify_repository_format(&fmt, ".", NULL); if (apply_repository_format(repo, &fmt, APPLY_REPOSITORY_FORMAT_HONOR_ENV, &err) < 0) die("%s", err.buf); + repo->objects = odb_new(repo, ODB_NEW_HONOR_ENV); startup_info->have_repository = 1; clear_repository_format(&fmt); @@ -2092,6 +2096,7 @@ const char *setup_git_directory_gently(struct repository *repo, int *nongit_ok) if (apply_repository_format(repo, &discovery.format, APPLY_REPOSITORY_FORMAT_HONOR_ENV, &err) < 0) die("%s", err.buf); + repo->objects = odb_new(repo, ODB_NEW_HONOR_ENV); clear_repository_format(&discovery.format); strbuf_release(&err); @@ -2653,25 +2658,29 @@ static int create_default_files(struct repository *repo, return reinit; } -static void create_object_directory(struct repository *repo) +static void create_object_database(struct repository *repo) { - struct strbuf path = STRBUF_INIT; - size_t baselen; - - strbuf_addstr(&path, repo_get_object_directory(repo)); - baselen = path.len; - - safe_create_dir(repo, path.buf, 1); - - strbuf_setlen(&path, baselen); - strbuf_addstr(&path, "/pack"); - safe_create_dir(repo, path.buf, 1); + /* + * Create the "objects" directory in the common directory. This is done + * so that the repository can be discovered regardless of the backend + * used. + * + * Note that we only do this in case the object directory wasn't + * overwritten via an environment variable. If it _is_ being overridden + * then we skip this step, as the repository won't be discoverable + * anyway without the environment variable. + */ + if (!getenv(DB_ENVIRONMENT)) { + struct strbuf objects_dir = STRBUF_INIT; + repo_common_path_append(repo, &objects_dir, "objects"); + safe_create_dir(repo, objects_dir.buf, 1); + strbuf_release(&objects_dir); + } - strbuf_setlen(&path, baselen); - strbuf_addstr(&path, "/info"); - safe_create_dir(repo, path.buf, 1); + repo->objects = odb_new(repo, ODB_NEW_HONOR_ENV); - strbuf_release(&path); + if (odb_source_create_on_disk(repo->objects->sources) < 0) + die(_("failed creating object database")); } static void separate_git_dir(struct repository *repo, @@ -2869,7 +2878,6 @@ int init_db(struct repository *repo, repository_format_configure(&repo_fmt, hash, ref_storage_format); if (apply_repository_format(repo, &repo_fmt, APPLY_REPOSITORY_FORMAT_HONOR_ENV, &err) < 0) die("%s", err.buf); - startup_info->have_repository = 1; /* * Ensure `core.hidedotfiles` is processed. This must happen after we @@ -2883,10 +2891,6 @@ int init_db(struct repository *repo, reinit = create_default_files(repo, template_dir, original_git_dir, &repo_fmt, init_shared_repository); - if (!(flags & INIT_DB_SKIP_REFDB)) - create_reference_database(repo, initial_branch, flags & INIT_DB_QUIET); - create_object_directory(repo); - if (repo_settings_get_shared_repository(repo)) { char buf[10]; /* We do not spell "group" and such, so that @@ -2908,6 +2912,12 @@ int init_db(struct repository *repo, repo_config_set(repo, "receive.denyNonFastforwards", "true"); } + if (!(flags & INIT_DB_SKIP_REFDB)) + create_reference_database(repo, initial_branch, flags & INIT_DB_QUIET); + create_object_database(repo); + + startup_info->have_repository = 1; + if (!(flags & INIT_DB_QUIET)) { int len = strlen(git_dir); diff --git a/setup.h b/setup.h index 654f10e059b995..053613235800a8 100644 --- a/setup.h +++ b/setup.h @@ -245,8 +245,8 @@ enum apply_repository_format_flags { /* * Apply the given repository format to the repo. This initializes extensions - * and basic data structures required for normal operation. Returns 0 on - * success, a negative error code when the format is not valid as determined by + * required for normal operation. Returns 0 on success, a negative error code + * when the format is not valid as determined by * `verify_repository_format()`. */ int apply_repository_format(struct repository *repo, @@ -304,4 +304,32 @@ struct startup_info { extern struct startup_info *startup_info; extern const char *tmp_original_cwd; +/* + * Apply the path allowlist in 'value' against 'target_path' setting + * '*is_match' accordingly. + * + * `value` is the value of a multi-valued config variable named `key` + * that holds an allowlist of paths. `target_path` is the (normalized) + * path being tested. `*is_match` is updated in place: + * + * - an empty value resets it to 0 (so a later, more specific config + * scope can clear entries from a broader one), + * - "*" sets it to 1 (allow everything), + * - "" sets it to 1 if equals `target_path`, + * - "" + "/" + "*" sets it to 1 if is a leading + * directory of `target_path`, + * - any other (unmatching) value leaves `*is_match` unchanged. + * + * Non-absolute values are rejected with a warning, except "." when + * `allow_dot` is set (used by 'safe.directory' to mean "the top level + * of the current repository"). + * + * Callers are expected to invoke this once per config value, + * typically from a protected-config callback, so that untrusted + * repository config cannot influence the decision. + */ +void path_allowlist_apply(const char *key, const char *value, + const char *target_path, int *is_match, + bool allow_dot); + #endif /* SETUP_H */ diff --git a/sideband.c b/sideband.c index 4a2c5f858f60ad..374187402e5c5b 100644 --- a/sideband.c +++ b/sideband.c @@ -445,6 +445,7 @@ void send_sideband(int fd, int band, const char *data, ssize_t sz, int packet_ma const char *p = data; while (sz) { + struct iovec iov[2]; unsigned n; char hdr[5]; @@ -454,12 +455,19 @@ void send_sideband(int fd, int band, const char *data, ssize_t sz, int packet_ma if (0 <= band) { xsnprintf(hdr, sizeof(hdr), "%04x", n + 5); hdr[4] = band; - write_or_die(fd, hdr, 5); + iov[0].iov_base = hdr; + iov[0].iov_len = 5; } else { xsnprintf(hdr, sizeof(hdr), "%04x", n + 4); - write_or_die(fd, hdr, 4); + iov[0].iov_base = hdr; + iov[0].iov_len = 4; } - write_or_die(fd, p, n); + + iov[1].iov_base = (void *) p; + iov[1].iov_len = n; + + writev_or_die(fd, iov, ARRAY_SIZE(iov)); + p += n; sz -= n; } diff --git a/statinfo.c b/statinfo.c index 5e00af127d657d..d9ddcf93820825 100644 --- a/statinfo.c +++ b/statinfo.c @@ -72,13 +72,13 @@ int match_stat_data(const struct stat_data *sd, struct stat *st) sd->sd_ctime.sec != (unsigned int)st->st_ctime) changed |= CTIME_CHANGED; -#ifdef USE_NSEC - if (cfg->check_stat && sd->sd_mtime.nsec != ST_MTIME_NSEC(*st)) - changed |= MTIME_CHANGED; - if (cfg->trust_ctime && cfg->check_stat && - sd->sd_ctime.nsec != ST_CTIME_NSEC(*st)) - changed |= CTIME_CHANGED; -#endif + if (cfg->use_nanosec) { + if (cfg->check_stat && sd->sd_mtime.nsec != ST_MTIME_NSEC(*st)) + changed |= MTIME_CHANGED; + if (cfg->trust_ctime && cfg->check_stat && + sd->sd_ctime.nsec != ST_CTIME_NSEC(*st)) + changed |= CTIME_CHANGED; + } if (cfg->check_stat) { if (sd->sd_uid != (unsigned int) st->st_uid || diff --git a/strbuf.c b/strbuf.c index 44955669e8c504..88d23f8ac5f5c4 100644 --- a/strbuf.c +++ b/strbuf.c @@ -457,7 +457,7 @@ size_t strbuf_expand_literal(struct strbuf *sb, const char *placeholder) return 1; case 'x': /* %x00 == NUL, %x0a == LF, etc. */ - ch = hex2chr(placeholder + 1); + ch = hex2chr(placeholder + 1, HEX_KIND_MIXED); if (ch < 0) return 0; strbuf_addch(sb, ch); diff --git a/sub-process.c b/sub-process.c index 2d5c965169727b..33bd78961827bb 100644 --- a/sub-process.c +++ b/sub-process.c @@ -49,7 +49,31 @@ int subprocess_read_status(int fd, struct strbuf *status) return (len < 0) ? len : 0; } -void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) +int subprocess_read_status_gently(int fd, struct strbuf *status) +{ + for (;;) { + int pktlen = -1; + enum packet_read_status rs; + const char *value; + + rs = packet_read_with_status(fd, NULL, NULL, packet_buffer, + sizeof(packet_buffer), &pktlen, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF | + PACKET_READ_GENTLE_ON_READ_ERROR); + if (rs == PACKET_READ_FLUSH) + return 0; + if (rs != PACKET_READ_NORMAL || !pktlen) + return -1; + if (skip_prefix(packet_buffer, "status=", &value)) { + /* the last "status=" line wins */ + strbuf_reset(status); + strbuf_addstr(status, value); + } + } +} + +void subprocess_stop_command(struct subprocess_entry *entry) { if (!entry) return; @@ -57,7 +81,14 @@ void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) entry->process.clean_on_exit = 0; kill(entry->process.pid, SIGTERM); finish_command(&entry->process); +} + +void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry) +{ + if (!entry) + return; + subprocess_stop_command(entry); hashmap_remove(hashmap, &entry->ent, NULL); } @@ -72,7 +103,7 @@ static void subprocess_exit_handler(struct child_process *process) finish_command(process); } -int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, +int subprocess_start_command(struct subprocess_entry *entry, const char *cmd, subprocess_start_fn startfn) { int err; @@ -96,15 +127,26 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co return err; } - hashmap_entry_init(&entry->ent, strhash(cmd)); - err = startfn(entry); if (err) { error("initialization for subprocess '%s' failed", cmd); - subprocess_stop(hashmap, entry); + subprocess_stop_command(entry); return err; } + return 0; +} + +int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, + subprocess_start_fn startfn) +{ + int err; + + err = subprocess_start_command(entry, cmd, startfn); + if (err) + return err; + + hashmap_entry_init(&entry->ent, strhash(cmd)); hashmap_add(hashmap, &entry->ent); return 0; } diff --git a/sub-process.h b/sub-process.h index bfc3959a1b4894..8655b388972d64 100644 --- a/sub-process.h +++ b/sub-process.h @@ -52,10 +52,17 @@ int cmd2process_cmp(const void *unused_cmp_data, */ typedef int(*subprocess_start_fn)(struct subprocess_entry *entry); -/* Start a subprocess and add it to the subprocess hashmap. */ +/* Start a subprocess and run the startfn (typically handshake). */ +int subprocess_start_command(struct subprocess_entry *entry, const char *cmd, + subprocess_start_fn startfn); + +/* Start a subprocess, run startfn, and add it to the subprocess hashmap. */ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, subprocess_start_fn startfn); +/* Kill a subprocess. */ +void subprocess_stop_command(struct subprocess_entry *entry); + /* Kill a subprocess and remove it from the subprocess hashmap. */ void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry); @@ -94,4 +101,14 @@ int subprocess_handshake(struct subprocess_entry *entry, int subprocess_read_status(int fd, struct strbuf *status); +/* + * Like subprocess_read_status(), but a malformed status section fails + * instead of dying: a truncated or malformed packet, and an empty + * packet where a status line or the terminating flush belongs, return + * -1 and leave the stream unusable. subprocess_read_status() cannot + * tell an empty packet from the flush that ends the section, and dies + * on a framing error inside packet_read_line_gently(). + */ +int subprocess_read_status_gently(int fd, struct strbuf *status); + #endif diff --git a/submodule.c b/submodule.c index 5c9257588856a9..9eed3cb85bed66 100644 --- a/submodule.c +++ b/submodule.c @@ -1409,6 +1409,7 @@ struct submodule_parallel_fetch { int oid_fetch_tasks_nr, oid_fetch_tasks_alloc; struct strbuf submodules_with_errors; + int submodule_errors; }; #define SPF_INIT { \ .args = STRVEC_INIT, \ @@ -1562,6 +1563,14 @@ static struct fetch_task *fetch_task_create(struct submodule_parallel_fetch *spf return NULL; } +static void record_fetch_error(struct submodule_parallel_fetch *spf, + const char *name) +{ + if (spf->submodule_errors == SUBMODULE_ERRORS_FAIL) + spf->result = 1; + strbuf_addf(&spf->submodules_with_errors, "\t%s\n", name); +} + static struct fetch_task * get_fetch_task_from_index(struct submodule_parallel_fetch *spf, struct strbuf *err) @@ -1599,7 +1608,7 @@ get_fetch_task_from_index(struct submodule_parallel_fetch *spf, ce->name); if (S_ISGITLINK(ce->ce_mode) && !is_empty_dir(empty_submodule_path.buf)) { - spf->result = 1; + record_fetch_error(spf, ce->name); strbuf_addf(err, _("Could not access submodule '%s'\n"), ce->name); @@ -1753,7 +1762,7 @@ static int fetch_start_failure(struct strbuf *err UNUSED, struct submodule_parallel_fetch *spf = cb; struct fetch_task *task = task_cb; - spf->result = 1; + record_fetch_error(spf, task->sub->name); fetch_task_free(task); return 0; @@ -1779,18 +1788,12 @@ static int fetch_finish(int retvalue, struct strbuf *err UNUSED, if (!task || !task->sub) BUG("callback cookie bogus"); - if (retvalue) { + if (retvalue && task->commits) { /* - * NEEDSWORK: This indicates that the overall fetch - * failed, even though there may be a subsequent fetch - * by commit hash that might work. It may be a good - * idea to not indicate failure in this case, and only - * indicate failure if the subsequent fetch fails. + * This is the second pass (OID-based fetch) and it failed. + * The commits are genuinely unavailable from the remote. */ - spf->result = 1; - - strbuf_addf(&spf->submodules_with_errors, "\t%s\n", - task->sub->name); + record_fetch_error(spf, task->sub->name); } /* Is this the second time we process this submodule? */ @@ -1798,9 +1801,17 @@ static int fetch_finish(int retvalue, struct strbuf *err UNUSED, goto out; it = string_list_lookup(&spf->changed_submodule_names, task->sub->name); - if (!it) - /* Could be an unchanged submodule, not contained in the list */ + if (!it) { + /* + * This submodule is not in the changed list (e.g. it was + * fetched because RECURSE_SUBMODULES_ON fetches all populated + * submodules). A phase 1 failure here has no OID-based retry + * to fall back on, so it is a genuine error. + */ + if (retvalue) + record_fetch_error(spf, task->sub->name); goto out; + } cs_data = it->util; oid_array_filter(&cs_data->new_commits, @@ -1809,6 +1820,11 @@ static int fetch_finish(int retvalue, struct strbuf *err UNUSED, /* Are there commits we want, but do not exist? */ if (cs_data->new_commits.nr) { + /* + * Schedule an OID-based phase 2 fetch to retrieve the missing + * commits directly. Defer any error from phase 1: if phase 2 + * succeeds, the overall operation should still succeed. + */ task->commits = &cs_data->new_commits; ALLOC_GROW(spf->oid_fetch_tasks, spf->oid_fetch_tasks_nr + 1, @@ -1818,6 +1834,16 @@ static int fetch_finish(int retvalue, struct strbuf *err UNUSED, return 0; } + /* + * All required commits are already present locally (they were either + * fetched by phase 1 or existed beforehand), so there is no phase 2 + * retry to defer to. If phase 1 failed, the fetch itself went wrong + * (e.g. a transport error) and must still be reported, even though + * the gitlinked commits are available. + */ + if (retvalue) + record_fetch_error(spf, task->sub->name); + out: fetch_task_free(task); return 0; @@ -1827,7 +1853,8 @@ int fetch_submodules(struct repository *r, const struct strvec *options, const char *prefix, int command_line_option, int default_option, - int quiet, int max_parallel_jobs) + int quiet, int max_parallel_jobs, + int submodule_errors) { struct submodule_parallel_fetch spf = SPF_INIT; const struct run_process_parallel_opts opts = { @@ -1847,6 +1874,7 @@ int fetch_submodules(struct repository *r, spf.default_option = default_option; spf.quiet = quiet; spf.prefix = prefix; + spf.submodule_errors = submodule_errors; if (!r->worktree) goto out; diff --git a/submodule.h b/submodule.h index b10e16e6c063d2..c80b687d2a7c67 100644 --- a/submodule.h +++ b/submodule.h @@ -90,12 +90,17 @@ int should_update_submodules(void); */ const struct submodule *submodule_from_ce(const struct cache_entry *ce); void check_for_new_submodule_commits(struct object_id *oid); +/* Values for the submodule_errors parameter of fetch_submodules(). */ +#define SUBMODULE_ERRORS_FAIL 0 /* submodule fetch errors are fatal (default) */ +#define SUBMODULE_ERRORS_WARN 1 /* submodule fetch errors are non-fatal warnings */ + int fetch_submodules(struct repository *r, const struct strvec *options, const char *prefix, int command_line_option, int default_option, - int quiet, int max_parallel_jobs); + int quiet, int max_parallel_jobs, + int submodule_errors); unsigned is_submodule_modified(const char *path, int ignore_untracked); int submodule_uses_gitfile(const char *path); diff --git a/t/helper/meson.build b/t/helper/meson.build index 3235f10ab8aae1..6abcda4afb89c0 100644 --- a/t/helper/meson.build +++ b/t/helper/meson.build @@ -12,6 +12,7 @@ test_tool_sources = [ 'test-date.c', 'test-delete-gpgsig.c', 'test-delta.c', + 'test-diff-process-backend.c', 'test-dir-iterator.c', 'test-drop-caches.c', 'test-dump-cache-tree.c', diff --git a/t/helper/test-bitmap.c b/t/helper/test-bitmap.c index 8547ef67e243eb..6f851e04210198 100644 --- a/t/helper/test-bitmap.c +++ b/t/helper/test-bitmap.c @@ -50,6 +50,9 @@ static int add_packed_object(const struct object_id *oid, oi.typep = &type; + if (packlist_find(packed, oid)) + die("pack contains duplicate object %s", oid_to_hex(oid)); + entry = packlist_alloc(packed, oid); entry->idx.offset = nth_packed_object_offset(pack, pos); if (packed_object_info(NULL, pack, entry->idx.offset, &oi) < 0) diff --git a/t/helper/test-delta.c b/t/helper/test-delta.c index 8223a60229229e..d807afef751b48 100644 --- a/t/helper/test-delta.c +++ b/t/helper/test-delta.c @@ -32,7 +32,7 @@ int cmd__delta(int argc, const char **argv) die_errno("unable to read '%s'", argv[3]); if (argv[1][1] == 'd') { - unsigned long delta_size; + size_t delta_size; out_buf = diff_delta(from.buf, from.len, data.buf, data.len, &delta_size, 0); diff --git a/t/helper/test-diff-process-backend.c b/t/helper/test-diff-process-backend.c new file mode 100644 index 00000000000000..b0bb78d9f34fe8 --- /dev/null +++ b/t/helper/test-diff-process-backend.c @@ -0,0 +1,349 @@ +/* + * Test process implementing the diff process protocol (diff..process). + * + * Speaks the long-running process protocol over stdin/stdout and + * answers command=hunks-by-oid requests from the blob object names + * alone; no content is exchanged. The --mode= switch selects the + * response shape: + * + * oid-fixed packet: git< hunk 5 2 5 2 + * oid-equal packet: git< status=success (zero hunks: equivalent) + * oid-need-content packet: git< status=need-content + * oid-empty packet: git< hunk 0 0 1 2 (empty old side) + * + * and the adversarial shapes the protocol error paths are tested + * with: + * + * oid-trailing a hunk line with a trailing token to ignore + * oid-malformed a hunk line that does not parse + * oid-huge coordinates far past the end of any test blob + * oid-erange a count too large for any long + * oid-overlap two hunks out of order + * oid-misaligned two hunks whose unchanged runs differ in length + * oid-badstart a start of 0 paired with a nonzero count + * oid-unknown-status status=frobnicate + * oid-abort status=abort + * oid-bare-status a status packet without the hunk-section flush + * oid-empty-packet an empty packet (0004) inside the hunk section + * oid-crash one hunk line, then exit with no flush or status + * oid-garbage raw non-pkt-line bytes, then exit + * cap-none handshake announcing no capability at all + * + * Success responses end with: + * + * packet: git< 0000 + * packet: git< status=success + * packet: git< 0000 + * + * Each request is logged to --log as: + * + * command= pathname= old-oid= new-oid= + */ + +#include "test-tool.h" +#include "pkt-line.h" +#include "parse-options.h" +#include "strbuf.h" + +static FILE *logfile; + +enum mode { + MODE_OID_FIXED, + MODE_OID_EQUAL, + MODE_OID_NEED_CONTENT, + MODE_OID_EMPTY, + MODE_OID_TRAILING, + MODE_OID_MALFORMED, + MODE_OID_HUGE, + MODE_OID_ERANGE, + MODE_OID_OVERLAP, + MODE_OID_MISALIGNED, + MODE_OID_BADSTART, + MODE_OID_UNKNOWN_STATUS, + MODE_OID_ABORT, + MODE_OID_BARE_STATUS, + MODE_OID_EMPTY_PACKET, + MODE_OID_CRASH, + MODE_OID_GARBAGE, + MODE_CAP_NONE, +}; + +static enum mode parse_mode(const char *s) +{ + if (!strcmp(s, "oid-fixed")) + return MODE_OID_FIXED; + if (!strcmp(s, "oid-equal")) + return MODE_OID_EQUAL; + if (!strcmp(s, "oid-need-content")) + return MODE_OID_NEED_CONTENT; + if (!strcmp(s, "oid-empty")) + return MODE_OID_EMPTY; + if (!strcmp(s, "oid-trailing")) + return MODE_OID_TRAILING; + if (!strcmp(s, "oid-malformed")) + return MODE_OID_MALFORMED; + if (!strcmp(s, "oid-huge")) + return MODE_OID_HUGE; + if (!strcmp(s, "oid-erange")) + return MODE_OID_ERANGE; + if (!strcmp(s, "oid-overlap")) + return MODE_OID_OVERLAP; + if (!strcmp(s, "oid-misaligned")) + return MODE_OID_MISALIGNED; + if (!strcmp(s, "oid-badstart")) + return MODE_OID_BADSTART; + if (!strcmp(s, "oid-unknown-status")) + return MODE_OID_UNKNOWN_STATUS; + if (!strcmp(s, "oid-abort")) + return MODE_OID_ABORT; + if (!strcmp(s, "oid-bare-status")) + return MODE_OID_BARE_STATUS; + if (!strcmp(s, "oid-empty-packet")) + return MODE_OID_EMPTY_PACKET; + if (!strcmp(s, "oid-crash")) + return MODE_OID_CRASH; + if (!strcmp(s, "oid-garbage")) + return MODE_OID_GARBAGE; + if (!strcmp(s, "cap-none")) + return MODE_CAP_NONE; + die("unknown --mode=%s", s); +} + +/* + * Read "key=value" packets up to a flush, capturing "command" and + * "pathname". Returns 1 if a request was read, 0 on EOF. + * + * The first packet uses the gentle variant so that a clean shutdown + * by Git (EOF) does not produce a spurious "the remote end hung up + * unexpectedly" on stderr. Subsequent packets use the non-gentle + * variant: once inside a request, truncation is a protocol violation + * and dying loudly is the correct response. + */ +static int read_request_header(char **command, char **pathname, + char **old_oid, char **new_oid) +{ + int first = 1; + char *line; + + *command = *pathname = *old_oid = *new_oid = NULL; + for (;;) { + const char *value; + + if (first) { + if (packet_read_line_gently(0, NULL, &line) < 0) + return 0; + first = 0; + } else { + line = packet_read_line(0, NULL); + } + if (!line) + break; + if (skip_prefix(line, "command=", &value)) + *command = xstrdup(value); + else if (skip_prefix(line, "pathname=", &value)) + *pathname = xstrdup(value); + else if (skip_prefix(line, "old-oid=", &value)) + *old_oid = xstrdup(value); + else if (skip_prefix(line, "new-oid=", &value)) + *new_oid = xstrdup(value); + } + return 1; +} + +static void send_status(const char *status) +{ + packet_flush(1); + packet_write_fmt(1, "%s\n", status); + packet_flush(1); +} + +static void command_loop(enum mode mode) +{ + for (;;) { + char *command = NULL, *pathname = NULL; + char *old_oid = NULL, *new_oid = NULL; + + if (!read_request_header(&command, &pathname, + &old_oid, &new_oid)) + break; /* EOF: Git closed its end */ + + if (!command || strcmp(command, "hunks-by-oid")) + die("unexpected command: '%s'", + command ? command : "(none)"); + + if (logfile) { + fprintf(logfile, + "command=%s pathname=%s old-oid=%s new-oid=%s\n", + command, + pathname ? pathname : "(none)", + old_oid ? old_oid : "(none)", + new_oid ? new_oid : "(none)"); + fflush(logfile); + } + + switch (mode) { + case MODE_OID_FIXED: + packet_write_fmt(1, "hunk 5 2 5 2\n"); + send_status("status=success"); + break; + case MODE_OID_EQUAL: + send_status("status=success"); + break; + case MODE_OID_EMPTY: + /* + * An empty old side: the "git diff" convention + * addresses it with a start of 0 and a count of 0. + * Claims two lines added, fewer than the builtin + * would show, so the answer is observable. + */ + packet_write_fmt(1, "hunk 0 0 1 2\n"); + send_status("status=success"); + break; + case MODE_OID_TRAILING: + /* + * Git must ignore trailing space-separated tokens + * on a hunk line (the appendability rule), so this + * must behave exactly like oid-fixed. + */ + packet_write_fmt(1, "hunk 5 2 5 2 moved=yes\n"); + send_status("status=success"); + break; + case MODE_OID_MALFORMED: + packet_write_fmt(1, "hunk five two 5 2\n"); + send_status("status=success"); + break; + case MODE_OID_HUGE: + /* + * In-range for int32 (and for a 32-bit long), so + * only the blob-size bound can reject it. + */ + packet_write_fmt(1, "hunk 1 1000000000 1 1000000000\n"); + send_status("status=success"); + break; + case MODE_OID_ERANGE: + /* Overflows strtol() even where long is 64-bit. */ + packet_write_fmt(1, "hunk 1 99999999999999999999 1 1\n"); + send_status("status=success"); + break; + case MODE_OID_OVERLAP: + packet_write_fmt(1, "hunk 3 2 3 2\n"); + packet_write_fmt(1, "hunk 2 2 2 2\n"); + send_status("status=success"); + break; + case MODE_OID_MISALIGNED: + packet_write_fmt(1, "hunk 2 1 2 1\n"); + packet_write_fmt(1, "hunk 5 1 6 1\n"); + send_status("status=success"); + break; + case MODE_OID_BADSTART: + /* + * A start of 0 names an empty side, so a nonzero + * count beside it names no line; the coordinate is + * rejected per pair while the process stays alive. + */ + packet_write_fmt(1, "hunk 0 2 1 2\n"); + send_status("status=success"); + break; + case MODE_OID_UNKNOWN_STATUS: + send_status("status=frobnicate"); + break; + case MODE_OID_ABORT: + send_status("status=abort"); + break; + case MODE_OID_BARE_STATUS: + /* No hunk-section flush: a protocol violation. */ + packet_write_fmt(1, "status=success\n"); + packet_flush(1); + break; + case MODE_OID_EMPTY_PACKET: + /* + * An empty packet is not a flush; inside the hunk + * section it is a protocol violation. + */ + if (write(1, "0004", 4) < 0) + die_errno("write empty packet"); + send_status("status=success"); + break; + case MODE_OID_CRASH: + packet_write_fmt(1, "hunk 5 2 5 2\n"); + exit(0); + case MODE_OID_GARBAGE: + if (write(1, "@@@@ not a pkt-line @@@@", 24) < 0) + die_errno("write garbage"); + exit(0); + default: + send_status("status=need-content"); + break; + } + + free(command); + free(pathname); + free(old_oid); + free(new_oid); + } +} + +static void handshake(enum mode mode) +{ + char *line; + + line = packet_read_line(0, NULL); + if (!line || strcmp(line, "git-diff-client")) + die("bad welcome: '%s'", line ? line : "(eof)"); + line = packet_read_line(0, NULL); + if (!line || strcmp(line, "version=1")) + die("bad version: '%s'", line ? line : "(eof)"); + if (packet_read_line(0, NULL)) + die("expected flush after version"); + + packet_write_fmt(1, "git-diff-server\n"); + packet_write_fmt(1, "version=1\n"); + packet_flush(1); + + /* Drain capabilities advertised by Git */ + while ((line = packet_read_line(0, NULL))) + ; /* drain */ + + if (mode != MODE_CAP_NONE) + packet_write_fmt(1, "capability=hunks-by-oid\n"); + packet_flush(1); +} + +static const char *const usage_str[] = { + "test-tool diff-process-backend --mode= [--log=]", + NULL +}; + +int cmd__diff_process_backend(int argc, const char **argv) +{ + const char *mode_str = NULL, *log_path = NULL; + enum mode mode = MODE_OID_FIXED; + struct option options[] = { + OPT_STRING(0, "mode", &mode_str, "mode", + "response shape (default oid-fixed);" + " see the file header for the full list of modes"), + OPT_STRING(0, "log", &log_path, "path", + "append per-request summary to this file"), + OPT_END() + }; + + argc = parse_options(argc, argv, NULL, options, usage_str, 0); + if (argc) + usage_with_options(usage_str, options); + + if (mode_str) + mode = parse_mode(mode_str); + + if (log_path) { + logfile = fopen(log_path, "a"); + if (!logfile) + die_errno("failed to open log '%s'", log_path); + } + + handshake(mode); + command_loop(mode); + + if (logfile && fclose(logfile)) + die_errno("error closing log"); + return 0; +} diff --git a/t/helper/test-find-pack.c b/t/helper/test-find-pack.c index 28d5b1fe094345..51093a7030c89a 100644 --- a/t/helper/test-find-pack.c +++ b/t/helper/test-find-pack.c @@ -11,12 +11,15 @@ * Display the path(s), one per line, of the packfile(s) containing * the given object. * + * With '--show-offset', display the offset selected by + * find_pack_entry_one() instead of the packfile path. + * * If '--check-count ' is passed, then error out if the number of * packfiles containing the object is not . */ static const char *const find_pack_usage[] = { - "test-tool find-pack [--check-count ] ", + "test-tool find-pack [--check-count ] [--show-offset] ", NULL }; @@ -24,11 +27,13 @@ int cmd__find_pack(int argc, const char **argv) { struct object_id oid; struct packed_git *p; - int count = -1, actual_count = 0; + int count = -1, actual_count = 0, show_offset = 0; const char *prefix = setup_git_directory(the_repository); struct option options[] = { OPT_INTEGER('c', "check-count", &count, "expected number of packs"), + OPT_BOOL(0, "show-offset", &show_offset, + "show matching pack offsets"), OPT_END(), }; @@ -40,8 +45,13 @@ int cmd__find_pack(int argc, const char **argv) die("cannot parse %s as an object name", argv[0]); repo_for_each_pack(the_repository, p) { - if (find_pack_entry_one(&oid, p)) { - printf("%s\n", p->pack_name); + off_t offset = find_pack_entry_one(&oid, p); + + if (offset) { + if (show_offset) + printf("%"PRIuMAX"\n", (uintmax_t)offset); + else + printf("%s\n", p->pack_name); actual_count++; } } diff --git a/t/helper/test-pack-deltas.c b/t/helper/test-pack-deltas.c index 840797cf0dbabb..959705fecaf144 100644 --- a/t/helper/test-pack-deltas.c +++ b/t/helper/test-pack-deltas.c @@ -22,7 +22,7 @@ static unsigned long do_compress(void **pptr, unsigned long size) { git_zstream stream; void *in, *out; - unsigned long maxsize; + size_t maxsize; git_deflate_init(&stream, 1); maxsize = git_deflate_bound(&stream, size); @@ -49,7 +49,7 @@ static void write_ref_delta(struct hashfile *f, { unsigned char header[MAX_PACK_OBJECT_HEADER]; unsigned long delta_size, compressed_size, hdrlen; - size_t size, base_size; + size_t size, base_size, delta_size_st = 0; enum object_type type; void *base_buf, *delta_buf; void *buf = odb_read_object(the_repository->objects, @@ -65,7 +65,8 @@ static void write_ref_delta(struct hashfile *f, die("unable to read %s", oid_to_hex(base)); delta_buf = diff_delta(base_buf, base_size, - buf, size, &delta_size, 0); + buf, size, &delta_size_st, 0); + delta_size = cast_size_t_to_ulong(delta_size_st); compressed_size = do_compress(&delta_buf, delta_size); diff --git a/t/helper/test-parse-options.c b/t/helper/test-parse-options.c index 68579d83f3939e..f181f0c02d355a 100644 --- a/t/helper/test-parse-options.c +++ b/t/helper/test-parse-options.c @@ -209,6 +209,10 @@ int cmd__parse_options(int argc, const char **argv) OPT_GROUP("Alias"), OPT_STRING('A', "alias-source", &string, "string", "get a string"), OPT_ALIAS('Z', "alias-target", "alias-source"), + OPT_HIDDEN_GROUP("Hidden options"), + OPT_HIDDEN_BOOL(0, "hidden-bool", &boolean, "get a boolean"), + OPT_INTEGER_F('k', "hidden-integer", &integer, "get a integer", + PARSE_OPT_HIDDEN), OPT_END(), }; int ret = 0; diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index b71a22b43bbc9e..3c3f95269c6279 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -22,6 +22,7 @@ static struct test_cmd cmds[] = { { "date", cmd__date }, { "delete-gpgsig", cmd__delete_gpgsig }, { "delta", cmd__delta }, + { "diff-process-backend", cmd__diff_process_backend }, { "dir-iterator", cmd__dir_iterator }, { "drop-caches", cmd__drop_caches }, { "dump-cache-tree", cmd__dump_cache_tree }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index f2885b33d58aa8..a5bb7555162c8e 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -15,6 +15,7 @@ int cmd__csprng(int argc, const char **argv); int cmd__date(int argc, const char **argv); int cmd__delta(int argc, const char **argv); int cmd__delete_gpgsig(int argc, const char **argv); +int cmd__diff_process_backend(int argc, const char **argv); int cmd__dir_iterator(int argc, const char **argv); int cmd__drop_caches(int argc, const char **argv); int cmd__dump_cache_tree(int argc, const char **argv); diff --git a/t/lib-httpd.sh b/t/lib-httpd.sh index a216e5376fd7de..b137330c52a7e7 100644 --- a/t/lib-httpd.sh +++ b/t/lib-httpd.sh @@ -159,6 +159,19 @@ prepare_httpd() { mkdir -p "$HTTPD_DOCUMENT_ROOT_PATH" cp "$TEST_PATH"/passwd "$HTTPD_ROOT_PATH" cp "$TEST_PATH"/proxy-passwd "$HTTPD_ROOT_PATH" + # Apache runs each of these CGI scripts once per request. Apache can run one + # script for several requests at the same time. A helper that keeps state + # between requests must update that state with one atomic operation. A check + # and then a separate action is not safe: two requests can both pass the + # check before either one acts. Test the exit status of one atomic operation + # instead: + # - "mkdir dir" fails if the directory exists, so only one request + # succeeds. http-429.sh selects the first request this way. + # - "rm marker" (without "-f") fails if the marker is gone, so only one + # request consumes it. apply-one-time-script.sh claims its one-shot + # marker this way. + # A scratch file name includes the process ID ($$), so concurrent requests + # do not overwrite each other's files. install_script incomplete-length-upload-pack-v2-http.sh install_script incomplete-body-upload-pack-v2-http.sh install_script error-no-report.sh diff --git a/t/lib-httpd/apply-one-time-script.sh b/t/lib-httpd/apply-one-time-script.sh index b1682944e280e2..8ab97e882abb27 100644 --- a/t/lib-httpd/apply-one-time-script.sh +++ b/t/lib-httpd/apply-one-time-script.sh @@ -6,21 +6,43 @@ # # This can be used to simulate the effects of the repository changing in # between HTTP request-response pairs. -if test -f one-time-script -then - LC_ALL=C - export LC_ALL +# +# Apache can run this CGI for several requests at the same time. For example, a +# partial fetch lazily fetches a missing object while the first response is +# still in flight. To stay correct, the helper removes the marker only after +# the response changes, and only with "rm" (without "-f"). The "rm" fails for +# every request except the one that removes the marker first. That request +# serves the modified body. Every other request serves its response unchanged. +# No request emits an empty body, which Apache would report as HTTP 500. +# +# A scratch file name includes the process ID ($$), so concurrent requests do +# not overwrite each other's files. +# +# The helper can run one-time-script more than once. It consumes the marker +# when the response changes (the "rm" after "cmp"), not when it runs the +# script. A request whose response is not the target runs the script, finds no +# change, and leaves the marker for a later request. This is safe because the +# scripts are stateless filters over the captured response. - "$GIT_EXEC_PATH/git-http-backend" >out - ./one-time-script out >out_modified +test -f one-time-script || exec "$GIT_EXEC_PATH/git-http-backend" - if cmp -s out out_modified - then - cat out - else - cat out_modified - rm one-time-script - fi +LC_ALL=C +export LC_ALL + +out=out.$$ +modified=out-modified.$$ +"$GIT_EXEC_PATH/git-http-backend" >"$out" + +# one-time-script can be gone here: a concurrent request may have consumed it +# since the "test -f" above. Then "./one-time-script" fails, the exit status +# selects the unmodified body, and "2>/dev/null" discards the expected +# "no such file" message. +if ./one-time-script "$out" 2>/dev/null >"$modified" && + ! cmp -s "$out" "$modified" && + rm one-time-script 2>/dev/null +then + cat "$modified" else - "$GIT_EXEC_PATH/git-http-backend" + cat "$out" fi +rm -f "$out" "$modified" diff --git a/t/lib-httpd/http-429.sh b/t/lib-httpd/http-429.sh index c97b16145b7f92..904cdacbd0ede6 100644 --- a/t/lib-httpd/http-429.sh +++ b/t/lib-httpd/http-429.sh @@ -3,7 +3,7 @@ # Script to return HTTP 429 Too Many Requests responses for testing retry logic. # Usage: /http_429/// # -# The test-context is a unique identifier for each test to isolate state files. +# The test-context is a unique identifier for each test to isolate state directories. # The retry-after-value can be: # - A number (e.g., "1", "2", "100") - sets Retry-After header to that many seconds # - "none" - no Retry-After header @@ -26,14 +26,24 @@ repo_path="${remaining#*/}" # Get rest (repo path) # The repo name is the first component before any "/" repo_name="${repo_path%%/*}" -# Use current directory (HTTPD_ROOT_PATH) for state file -# Create a safe filename from test_context, retry_after and repo_name -# This ensures all requests for the same test context share the same state file +# Store state in the current directory (HTTPD_ROOT_PATH). Build a safe name +# from test_context, retry_after, and repo_name, so that all requests for one +# test context share the same state. safe_name=$(echo "${test_context}-${retry_after}-${repo_name}" | tr '/' '_' | tr -cd 'a-zA-Z0-9_-') -state_file="http-429-state-${safe_name}" +state="http-429-state-${safe_name}" -# Check if this is the first call (no state file exists) -if test -f "$state_file" +# This endpoint returns 429 to the first request. It forwards every later +# request to git-http-backend, so the retry succeeds. Apache can run this CGI +# for several requests at the same time. A single atomic "mkdir" selects the +# first request, because only one "mkdir" succeeds. That request returns 429 +# and leaves the directory as the "already rate-limited" marker. Every later +# "mkdir" fails, so the endpoint forwards those requests. +# +# "permanent" is the exception. It must return 429 to every request, so it +# skips the "mkdir" and records no state. A leftover directory would let a +# later "permanent" request find the marker. The endpoint would forward that +# request, which "permanent" must not allow. +if test "$retry_after" != permanent && ! mkdir "$state" 2>/dev/null then # Already returned 429 once, forward to git-http-backend # Set PATH_INFO to just the repo path (without retry-after value) @@ -52,9 +62,6 @@ then exec "$GIT_EXEC_PATH/git-http-backend" fi -# Mark that we've returned 429 -touch "$state_file" - # Output HTTP 429 response printf "Status: 429 Too Many Requests\r\n" @@ -67,8 +74,7 @@ case "$retry_after" in printf "Retry-After: invalid-format-123abc\r\n" ;; permanent) - # Always return 429, don't set state file for success - rm -f "$state_file" + # Always return 429 printf "Retry-After: 1\r\n" printf "Content-Type: text/plain\r\n" printf "\r\n" diff --git a/t/lib-midx.sh b/t/lib-midx.sh index e38c609604c301..b522dbdb0f431a 100644 --- a/t/lib-midx.sh +++ b/t/lib-midx.sh @@ -34,3 +34,9 @@ compare_results_with_midx () { midx_git_two_modes "cat-file --batch-all-objects --batch-check --unordered" sorted ' } + +nth_line() { + local n="$1" + shift + awk "NR==$n" "$@" +} diff --git a/t/meson.build b/t/meson.build index 2133c840da63dc..16b08ad9341898 100644 --- a/t/meson.build +++ b/t/meson.build @@ -407,6 +407,7 @@ integration_tests = [ 't3452-history-split.sh', 't3453-history-fixup.sh', 't3454-history-drop.sh', + 't3455-history-squash.sh', 't3500-cherry.sh', 't3501-revert-cherry-pick.sh', 't3502-cherry-pick-merge.sh', @@ -520,6 +521,7 @@ integration_tests = [ 't4072-diff-max-depth.sh', 't4073-diff-stat-name-width.sh', 't4074-diff-shifted-matched-group.sh', + 't4080-diff-process.sh', 't4100-apply-stat.sh', 't4101-apply-nonl.sh', 't4102-apply-rename.sh', @@ -586,6 +588,7 @@ integration_tests = [ 't4217-log-limit.sh', 't4218-log-graph-indentation.sh', 't4219-log-follow-merge.sh', + 't4220-diff-hunks.sh', 't4252-am-options.sh', 't4253-am-keep-cr-dos.sh', 't4254-am-corrupt.sh', @@ -717,6 +720,7 @@ integration_tests = [ 't5564-http-proxy.sh', 't5565-push-multiple.sh', 't5566-push-group.sh', + 't5567-one-time-script.sh', 't5570-git-daemon.sh', 't5571-pre-push-hook.sh', 't5572-pull-submodule.sh', @@ -796,6 +800,7 @@ integration_tests = [ 't6041-bisect-submodule.sh', 't6050-replace.sh', 't6060-merge-index.sh', + 't6099-merge-base-side-exhaustion.sh', 't6100-rev-list-in-order.sh', 't6101-rev-parse-parents.sh', 't6102-rev-list-unexpected-objects.sh', @@ -965,6 +970,7 @@ integration_tests = [ 't7703-repack-geometric.sh', 't7704-repack-cruft.sh', 't7705-repack-incremental-midx.sh', + 't7706-repack-drop-filtered.sh', 't7800-difftool.sh', 't7810-grep.sh', 't7811-grep-open.sh', @@ -1160,6 +1166,7 @@ benchmarks = [ 'perf/p4205-log-pretty-formats.sh', 'perf/p4209-pickaxe.sh', 'perf/p4211-line-log.sh', + 'perf/p4218-diff-hunks.sh', 'perf/p4220-log-grep-engines.sh', 'perf/p4221-log-grep-engines-fixed.sh', 'perf/p5302-pack-index.sh', diff --git a/t/perf/p4218-diff-hunks.sh b/t/perf/p4218-diff-hunks.sh new file mode 100755 index 00000000000000..f849e97832eeda --- /dev/null +++ b/t/perf/p4218-diff-hunks.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +test_description='diff-hunks store performance' +. ./perf-lib.sh + +test_perf_default_repo + +# Pick a file to blame pseudo-randomly. The sort key is the blob +# hash, so it is stable. +test_expect_success 'select a file' ' + git ls-tree -r HEAD | grep ^100644 | + sort -k 3 | head -n 1 | cut -f 2 >filelist +' + +file=$(cat filelist) +export file + +# Warm the store the way an owner would: a stat walk with writing on. +test_perf 'warm the store' ' + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null +' + +test_expect_success 'ensure the store is warm for the timed reads' ' + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null +' + +test_perf 'log --stat -1000 (store)' ' + git log --stat -1000 >/dev/null +' + +test_perf 'log --stat -1000 (no store)' ' + git -c core.diffhunks=false log --stat -1000 >/dev/null +' + +test_perf 'blame $file (store)' ' + git blame "$file" >/dev/null +' + +test_perf 'blame $file (no store)' ' + git -c core.diffhunks=false blame "$file" >/dev/null +' + +test_expect_success 'clean up store' ' + git diff-hunks clear +' + +test_done diff --git a/t/perf/p5303-many-packs.sh b/t/perf/p5303-many-packs.sh index af173a7b73e398..4221f9dd706243 100755 --- a/t/perf/p5303-many-packs.sh +++ b/t/perf/p5303-many-packs.sh @@ -141,4 +141,8 @@ test_perf "load 10,000 packs" ' git rev-parse --verify "HEAD^{commit}" ' +test_perf "abbreviate with 10,000 packs" ' + git rev-parse --short HEAD +' + test_done diff --git a/t/t0040-parse-options.sh b/t/t0040-parse-options.sh index a22533f9ed6d16..449fff4d34b172 100755 --- a/t/t0040-parse-options.sh +++ b/t/t0040-parse-options.sh @@ -7,7 +7,7 @@ test_description='our own option parser' . ./test-lib.sh -cat >expect <<\EOF +cat >expect-part1 <<\EOF usage: test-tool parse-options A helper function for the parse-options API. @@ -41,6 +41,9 @@ String options --[no-]string2 get another string --[no-]st get another string (pervert ordering) -o get another string +EOF + +cat >expect-part2 <<\EOF --longhelp help text of this entry spans multiple lines --[no-]list add str to list @@ -67,12 +70,32 @@ Alias EOF +cat >expect-noop <<\EOF + --[no-]obsolete no-op (backward compatibility) +EOF + +cat >expect-hidden <<\EOF +Hidden options + --[no-]hidden-bool get a boolean + -k, --[no-]hidden-integer + get a integer + +EOF + test_expect_success 'test help' ' + cat expect-part1 expect-part2 >expect && test-tool parse-options -h >output 2>output.err && test_must_be_empty output.err && test_cmp expect output ' +test_expect_success 'test --help-all shows hidden group and options' ' + cat expect-part1 expect-noop expect-part2 expect-hidden >expect-help-all && + test-tool parse-options --help-all >output 2>output.err && + test_must_be_empty output.err && + test_cmp expect-help-all output +' + mv expect expect.err check () { diff --git a/t/t0041-usage.sh b/t/t0041-usage.sh index 51af7cc0300efb..2a9c5eafcac2a6 100755 --- a/t/t0041-usage.sh +++ b/t/t0041-usage.sh @@ -107,4 +107,11 @@ test_expect_success 'for-each-ref usage error' ' test_grep "usage" actual.err ' +test_expect_success 'git --shallow-file without a value' ' + test_must_fail git --shallow-file >actual 2>actual.err && + test_line_count = 0 actual && + test_grep "no file given for " actual.err && + test_grep "usage" actual.err +' + test_done diff --git a/t/t0060-path-utils.sh b/t/t0060-path-utils.sh index 8545cdfab559b4..56faf5fe732ee0 100755 --- a/t/t0060-path-utils.sh +++ b/t/t0060-path-utils.sh @@ -602,7 +602,8 @@ test_expect_success !VALGRIND,RUNTIME_PREFIX,CAN_EXEC_IN_PWD 'RUNTIME_PREFIX wor echo "echo HERE" | write_script pretend/libexec/git-core/git-here && GIT_EXEC_PATH= ./pretend/bin/git here >actual && echo HERE >expect && - test_cmp expect actual' + test_cmp expect actual +' test_expect_success !VALGRIND,RUNTIME_PREFIX,CAN_EXEC_IN_PWD '%(prefix)/ works' ' git config yes.path "%(prefix)/yes" && @@ -611,4 +612,34 @@ test_expect_success !VALGRIND,RUNTIME_PREFIX,CAN_EXEC_IN_PWD '%(prefix)/ works' test_cmp expect actual ' +test_expect_success MINGW,RUNTIME_PREFIX 'MSYSTEM/PATH is adjusted if necessary' ' + if test -z "$MINGW_PREFIX" + then + MINGW_PREFIX="/$(echo "${MSYSTEM:-MINGW64}" | tr A-Z a-z)" + fi && + mkdir -p "$HOME"/bin pretend"$MINGW_PREFIX"/bin \ + pretend"$MINGW_PREFIX"/libexec/git-core pretend/usr/bin && + cp "$GIT_EXEC_PATH"/git.exe pretend"$MINGW_PREFIX"/bin/ && + cp "$GIT_EXEC_PATH"/git.exe pretend"$MINGW_PREFIX"/libexec/git-core/ && + # copy the .dll files, if any (happens when building via CMake) + if test -n "$(ls "$GIT_EXEC_PATH"/*.dll 2>/dev/null)" + then + cp "$GIT_EXEC_PATH"/*.dll pretend"$MINGW_PREFIX"/bin/ && + cp "$GIT_EXEC_PATH"/*.dll pretend"$MINGW_PREFIX"/libexec/git-core/ + fi && + echo "env | grep MSYSTEM=" | write_script "$HOME"/bin/git-test-home && + echo "echo ${MINGW_PREFIX#/}" | write_script pretend"$MINGW_PREFIX"/bin/git-test-bin && + echo "echo usr" | write_script pretend/usr/bin/git-test-bin2 && + + ( + MSYSTEM= && + GIT_EXEC_PATH= && + pretend"$MINGW_PREFIX"/libexec/git-core/git.exe test-home >actual && + pretend"$MINGW_PREFIX"/libexec/git-core/git.exe test-bin >>actual && + pretend"$MINGW_PREFIX"/bin/git.exe test-bin2 >>actual + ) && + test_write_lines MSYSTEM=$MSYSTEM "${MINGW_PREFIX#/}" usr >expect && + test_cmp expect actual +' + test_done diff --git a/t/t0450/adoc-help-mismatches b/t/t0450/adoc-help-mismatches index c4a55ff4e35a4f..baf3b1d80927d4 100644 --- a/t/t0450/adoc-help-mismatches +++ b/t/t0450/adoc-help-mismatches @@ -12,7 +12,6 @@ column credential credential-cache credential-store -fast-import fetch-pack fmt-merge-msg format-patch diff --git a/t/t1016-compatObjectFormat.sh b/t/t1016-compatObjectFormat.sh index 92d48b96a10932..9cafcee5098692 100755 --- a/t/t1016-compatObjectFormat.sh +++ b/t/t1016-compatObjectFormat.sh @@ -187,6 +187,24 @@ do eval signedtag3_${hash}_oid=$(git hash-object -t tag -w ../${hash}_signedtag3) && eval signedtag4_${hash}_oid=$(git hash-object -t tag -w ../${hash}_signedtag4) ' + + test_expect_success 'rev-parse maps oid of object borrowed from alternate' ' + for repo in alt borrow + do + test_when_finished "rm -rf $repo" && + git init --object-format=$hash $repo && + git -C $repo config set core.repositoryformatversion 1 && + git -C $repo config set extensions.compatObjectFormat $(compat_hash $hash) || exit 1 + done && + + git -C alt commit --allow-empty --message A && + echo "$(pwd)/alt/.git/objects" >borrow/.git/objects/info/alternates && + + oid=$(git -C alt rev-parse HEAD) && + git -C alt rev-parse --output-object-format=$(compat_hash $hash) "$oid" >expect && + git -C borrow rev-parse --output-object-format=$(compat_hash $hash) "$oid" >actual && + test_cmp expect actual + ' done cd "$base" diff --git a/t/t1017-cat-file-remote-object-info.sh b/t/t1017-cat-file-remote-object-info.sh index 116862f9d0b447..e2919aa061830a 100755 --- a/t/t1017-cat-file-remote-object-info.sh +++ b/t/t1017-cat-file-remote-object-info.sh @@ -7,6 +7,7 @@ test_description='git cat-file --batch-command with remote-object-info command' hello_content="Hello World" hello_size=$(strlen "$hello_content") +hello_type="blob" hello_oid=$(echo_without_newline "$hello_content" | git hash-object --stdin) hello_short_oid=$(git rev-parse --short "$hello_oid") @@ -19,6 +20,7 @@ unstored_oid=$(echo_without_newline "$unstored_content" | git hash-object --stdi # file name is hello, which is 5 characters # a space is 1 character and a null is 1 character tree_size=$(($(test_oid rawsz) + 13)) +tree_type="tree" commit_message="Initial commit" @@ -31,6 +33,7 @@ commit_message="Initial commit" # An easier way to calculate is: 1. use `git cat-file commit | wc -c`, # to get 177, 2. then deduct 40 hex characters to get 137 commit_size=$(($(test_oid hexsz) + 137)) +commit_type="commit" tag_header_without_oid="type blob tag hellotag @@ -44,6 +47,7 @@ $tag_description" tag_oid=$(echo_without_newline "$tag_content" | git hash-object -t tag --stdin -w) tag_size=$(strlen "$tag_content") +tag_type="tag" set_transport_variables () { hello_oid=$(echo_without_newline "$hello_content" | git hash-object --stdin) @@ -135,10 +139,10 @@ test_expect_success 'batch-command remote-object-info git:// default filter' ' set_transport_variables "$daemon_parent" && cd "$daemon_parent/daemon_client_empty" && - echo "$hello_oid $hello_size" >expect && - echo "$tree_oid $tree_size" >>expect && - echo "$commit_oid $commit_size" >>expect && - echo "$tag_oid $tag_size" >>expect && + echo "$hello_oid $hello_type $hello_size" >expect && + echo "$tree_oid $tree_type $tree_size" >>expect && + echo "$commit_oid $commit_type $commit_size" >>expect && + echo "$tag_oid $tag_type $tag_size" >>expect && git cat-file --batch-command >actual <<-EOF && remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid $tree_oid @@ -148,7 +152,7 @@ test_expect_success 'batch-command remote-object-info git:// default filter' ' ) ' -test_expect_success 'remote-object-info does not change the default format of info' ' +test_expect_success 'remote-object-info and info can be mixed using the unified default format' ' ( set_transport_variables "$daemon_parent" && cd "$daemon_parent/daemon_client_empty" && @@ -158,7 +162,7 @@ test_expect_success 'remote-object-info does not change the default format of in local_size=$(strlen "$local_content") && echo "$local_oid blob $local_size" >expect && - echo "$hello_oid $hello_size" >>expect && + echo "$hello_oid blob $hello_size" >>expect && echo "$local_oid blob $local_size" >>expect && git cat-file --batch-command >actual <<-EOF && @@ -205,10 +209,10 @@ test_expect_success 'batch-command -Z remote-object-info git:// default filter' set_transport_variables "$daemon_parent" && cd "$daemon_parent/daemon_client_empty" && - printf "%s\0" "$hello_oid $hello_size" >expect && - printf "%s\0" "$tree_oid $tree_size" >>expect && - printf "%s\0" "$commit_oid $commit_size" >>expect && - printf "%s\0" "$tag_oid $tag_size" >>expect && + printf "%s\0" "$hello_oid $hello_type $hello_size" >expect && + printf "%s\0" "$tree_oid $tree_type $tree_size" >>expect && + printf "%s\0" "$commit_oid $commit_type $commit_size" >>expect && + printf "%s\0" "$tag_oid $tag_type $tag_size" >>expect && printf "%s\0" "$hello_oid missing" >>expect && printf "%s\0" "$tree_oid missing" >>expect && @@ -256,14 +260,12 @@ test_expect_success 'remote-object-info does not die on missing oid like info' ' ) ' -# This tests depends on %(objecttype) not being supported yet, once supported -# it needs to be updated. -test_expect_success 'unsupported placeholder on remote returns empty string' ' +test_expect_success 'objecttype is supported by remote-object-info' ' ( set_transport_variables "$daemon_parent" && cd "$daemon_parent/daemon_client_empty" && - echo "" >expect && + echo "$hello_type" >expect && git cat-file --batch-command="%(objecttype)" >actual <<-EOF && remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid EOF @@ -271,6 +273,22 @@ test_expect_success 'unsupported placeholder on remote returns empty string' ' ) ' +test_expect_success 'unsupported placeholders on remote return empty string' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + fmt="%(objectmode) %(objectsize:disk) %(rest) %(deltabase)" && + + # The hardcoded SPs between the atoms are respected. + echo " " >expect && + git cat-file --batch-command="$fmt" >actual <<-EOF && + remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid + EOF + test_cmp expect actual + ) +' + test_expect_success 'requesting only objectname echoes back' ' ( set_transport_variables "$daemon_parent" && @@ -430,10 +448,10 @@ test_expect_success 'batch-command remote-object-info file:// default filter' ' server_path="$(pwd)/server" && cd file_client_empty && - echo "$hello_oid $hello_size" >expect && - echo "$tree_oid $tree_size" >>expect && - echo "$commit_oid $commit_size" >>expect && - echo "$tag_oid $tag_size" >>expect && + echo "$hello_oid $hello_type $hello_size" >expect && + echo "$tree_oid $tree_type $tree_size" >>expect && + echo "$commit_oid $commit_type $commit_size" >>expect && + echo "$tag_oid $tag_type $tag_size" >>expect && git cat-file --batch-command >actual <<-EOF && remote-object-info "file://${server_path}" $hello_oid $tree_oid @@ -449,10 +467,10 @@ test_expect_success 'batch-command -Z remote-object-info file:// default filter' server_path="$(pwd)/server" && cd file_client_empty && - printf "%s\0" "$hello_oid $hello_size" >expect && - printf "%s\0" "$tree_oid $tree_size" >>expect && - printf "%s\0" "$commit_oid $commit_size" >>expect && - printf "%s\0" "$tag_oid $tag_size" >>expect && + printf "%s\0" "$hello_oid $hello_type $hello_size" >expect && + printf "%s\0" "$tree_oid $tree_type $tree_size" >>expect && + printf "%s\0" "$commit_oid $commit_type $commit_size" >>expect && + printf "%s\0" "$tag_oid $tag_type $tag_size" >>expect && printf "%s\0" "$hello_oid missing" >>expect && printf "%s\0" "$tree_oid missing" >>expect && @@ -600,10 +618,10 @@ test_expect_success 'batch-command remote-object-info http:// default filter' ' set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && cd "$HTTPD_DOCUMENT_ROOT_PATH/http_client_empty" && - echo "$hello_oid $hello_size" >expect && - echo "$tree_oid $tree_size" >>expect && - echo "$commit_oid $commit_size" >>expect && - echo "$tag_oid $tag_size" >>expect && + echo "$hello_oid $hello_type $hello_size" >expect && + echo "$tree_oid $tree_type $tree_size" >>expect && + echo "$commit_oid $commit_type $commit_size" >>expect && + echo "$tag_oid $tag_type $tag_size" >>expect && git cat-file --batch-command >actual <<-EOF && remote-object-info "$HTTPD_URL/smart/http_parent" $hello_oid $tree_oid @@ -618,10 +636,10 @@ test_expect_success 'batch-command -Z remote-object-info http:// default filter' set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && cd "$HTTPD_DOCUMENT_ROOT_PATH/http_client_empty" && - printf "%s\0" "$hello_oid $hello_size" >expect && - printf "%s\0" "$tree_oid $tree_size" >>expect && - printf "%s\0" "$commit_oid $commit_size" >>expect && - printf "%s\0" "$tag_oid $tag_size" >>expect && + printf "%s\0" "$hello_oid $hello_type $hello_size" >expect && + printf "%s\0" "$tree_oid $tree_type $tree_size" >>expect && + printf "%s\0" "$commit_oid $commit_type $commit_size" >>expect && + printf "%s\0" "$tag_oid $tag_type $tag_size" >>expect && batch_input="remote-object-info $HTTPD_URL/smart/http_parent $hello_oid $tree_oid remote-object-info $HTTPD_URL/smart/http_parent $commit_oid $tag_oid diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 446c1776cb478b..05b54062b3bc85 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -1405,6 +1405,17 @@ do " done +test_expect_success 'relative path to a sparse directory' ' + init_repos && + + # A "::" argument whose path is relative is resolved + # into a heap-allocated buffer, and a sparse directory found at that + # path is reported through it. Cover that combination, so that the + # reporting does not read the buffer after it has been released. + test_sparse_match test_must_fail git show :0:./folder1/ && + test_sparse_match test_must_fail git rev-parse :0:./folder1/ +' + test_expect_success 'submodule handling' ' init_repos && diff --git a/t/t1401-symbolic-ref.sh b/t/t1401-symbolic-ref.sh index a2a7e947164c2a..602db6d0804a48 100755 --- a/t/t1401-symbolic-ref.sh +++ b/t/t1401-symbolic-ref.sh @@ -38,6 +38,16 @@ test_expect_success 'symbolic-ref refuses bare sha1' ' reset_to_sane +test_expect_success 'symbolic-ref reports a non-symbolic ref with exit code 128' ' + test_expect_code 128 git symbolic-ref refs/heads/foo 2>err && + test_grep "is not a symbolic ref" err +' + +test_expect_success 'symbolic-ref -q is silent and exits 1 on a non-symbolic ref' ' + test_expect_code 1 git symbolic-ref -q refs/heads/foo 2>err && + test_must_be_empty err +' + test_expect_success 'HEAD cannot be removed' ' test_must_fail git symbolic-ref -d HEAD ' diff --git a/t/t1402-check-ref-format.sh b/t/t1402-check-ref-format.sh index cabc516ae9a4fa..bc1e878a0fb4a1 100755 --- a/t/t1402-check-ref-format.sh +++ b/t/t1402-check-ref-format.sh @@ -51,12 +51,20 @@ invalid_ref '.refs/foo' invalid_ref 'refs/heads/foo.' invalid_ref 'heads/foo..bar' invalid_ref 'heads/foo?bar' +invalid_ref 'heads/foo~bar' +invalid_ref 'heads/foo^bar' +invalid_ref 'heads/foo:bar' +invalid_ref 'heads/foo*bar' +invalid_ref 'heads/foo[bar' +invalid_ref 'heads/foo bar' valid_ref 'foo./bar' invalid_ref 'heads/foo.lock' invalid_ref 'heads///foo.lock' invalid_ref 'foo.lock/bar' invalid_ref 'foo.lock///bar' valid_ref 'heads/foo@bar' +valid_ref 'refs/@' +invalid_ref '@' --allow-onelevel invalid_ref 'heads/v@{ation' invalid_ref 'heads/foo\bar' invalid_ref "$(printf 'heads/foo\t')" diff --git a/t/t1503-rev-parse-verify.sh b/t/t1503-rev-parse-verify.sh index 87638a4a2c71f9..f07b45de5a3fc1 100755 --- a/t/t1503-rev-parse-verify.sh +++ b/t/t1503-rev-parse-verify.sh @@ -60,6 +60,11 @@ test_expect_success 'works with one good rev' ' test "$rev_head" = "$HASH4" ' +test_expect_success WITH_BREAKING_CHANGES 'rejects uppercase revs' ' + UC_HASH=$(echo "$HASH1" | tr a-f A-F) && + test_must_fail git rev-parse --verify "$UC_HASH" +' + test_expect_success 'fails with any bad rev or many good revs' ' test_must_fail git rev-parse --verify 2>error && test_grep "single revision" error && diff --git a/t/t1900-repo-info.sh b/t/t1900-repo-info.sh index c85d390f438e80..b689445b7a0fb5 100755 --- a/t/t1900-repo-info.sh +++ b/t/t1900-repo-info.sh @@ -174,7 +174,11 @@ test_repo_info_path () { cd repo/sub && ROOT="$(test-tool path-utils real_path ..)" && export ROOT && eval "$init_command" && - echo "path.$field_name.absolute=$ROOT/$expected_dir" >expect && + case "$expected_dir" in + /*) EXPECT_ABS="$expected_dir" ;; + *) EXPECT_ABS="$ROOT/$expected_dir" ;; + esac && + echo "path.$field_name.absolute=$EXPECT_ABS" >expect && git repo info "path.$field_name.absolute" >actual && test_cmp expect actual ) @@ -188,7 +192,11 @@ test_repo_info_path () { cd repo/sub && ROOT="$(test-tool path-utils real_path ..)" && export ROOT && eval "$init_command" && - echo "path.$field_name.relative=../$expected_dir" >expect && + case "$expected_dir" in + /*) EXPECT_REL="$(test-tool path-utils relative_path "$expected_dir" "$PWD")" ;; + *) EXPECT_REL="../$expected_dir" ;; + esac && + echo "path.$field_name.relative=$EXPECT_REL" >expect && git repo info "path.$field_name.relative" >actual && test_cmp expect actual ) @@ -207,10 +215,152 @@ test_repo_info_path 'commondir with only GIT_DIR' 'commondir' \ '.git' \ 'GIT_DIR="../.git" && export GIT_DIR' +test_expect_success 'path.git-prefix at repository root' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + echo "path.git-prefix=" >expect && + git repo info path.git-prefix >actual && + test_cmp expect actual + ) +' + +test_expect_success 'path.git-prefix in subdirectory' ' + test_when_finished "rm -rf repo" && + git init repo && + mkdir -p repo/sub/dir && + ( + cd repo/sub/dir && + echo "path.git-prefix=sub/dir/" >expect && + git repo info path.git-prefix >actual && + test_cmp expect actual + ) +' + test_repo_info_path 'gitdir standard' 'gitdir' '.git' test_repo_info_path 'gitdir with explicit GIT_DIR' 'gitdir' \ '.git' \ 'GIT_DIR="../.git" && export GIT_DIR' +test_repo_info_path 'grafts standard' 'grafts' '.git/info/grafts' + +test_repo_info_path 'grafts with GIT_GRAFT_FILE override' 'grafts' \ + 'custom-graft-file' \ + 'GIT_GRAFT_FILE="$ROOT/custom-graft-file" && export GIT_GRAFT_FILE' + +test_repo_info_path 'hooks standard' 'hooks' '.git/hooks' + +test_repo_info_path 'hooks with core.hooksPath override' 'hooks' \ + 'custom-hooks' \ + 'git config core.hooksPath "$ROOT/custom-hooks" && mkdir -p "$ROOT/custom-hooks"' + +# /dev/null is not a real, canonicalizable filesystem path on Windows, +# so path resolution for core.hooksPath=/dev/null cannot be expected to +# produce a literal "/dev/null" the way it does on POSIX systems. +if ! test_have_prereq MINGW +then + test_repo_info_path 'hooks with core.hooksPath=/dev/null' 'hooks' \ + '/dev/null' \ + 'git config core.hooksPath /dev/null' +fi + +test_repo_info_path 'index standard' 'index' '.git/index' + +test_repo_info_path 'index with GIT_INDEX_FILE override' 'index' \ + 'custom-index-file' \ + 'GIT_INDEX_FILE="$ROOT/custom-index-file" && export GIT_INDEX_FILE' + +test_expect_success 'path.index in a bare repository returns default index location' ' + test_when_finished "rm -rf bare.git" && + git init --bare bare.git && + ( + cd bare.git && + ROOT="$(test-tool path-utils real_path .)" && + + echo "path.index.absolute=$ROOT/index" >expect.abs && + git repo info path.index.absolute >actual.abs && + test_cmp expect.abs actual.abs && + + echo "path.index.relative=index" >expect.rel && + git repo info path.index.relative >actual.rel && + test_cmp expect.rel actual.rel + ) +' + +test_expect_success 'path.superproject-root absolute and relative' ' + test_when_finished "rm -rf sub super" && + git init sub && + test_commit -C sub initial && + git init super && + ( + cd super && + git -c protocol.file.allow=always submodule add "../sub" sub && + git commit -m "add submodule" && + + cd sub && + ROOT="$(test-tool path-utils real_path ..)" && + + echo "path.superproject-root.absolute=$ROOT" >expect.abs && + git repo info path.superproject-root.absolute >actual.abs && + test_cmp expect.abs actual.abs && + + echo "path.superproject-root.relative=../" >expect.rel && + git repo info path.superproject-root.relative >actual.rel && + test_cmp expect.rel actual.rel + ) +' + +test_expect_success 'path.superproject-root returns empty when not in a submodule' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + + echo "path.superproject-root.absolute=" >expect.abs && + git repo info path.superproject-root.absolute >actual.abs && + test_cmp expect.abs actual.abs && + + echo "path.superproject-root.relative=" >expect.rel && + git repo info path.superproject-root.relative >actual.rel && + test_cmp expect.rel actual.rel + ) +' + +test_expect_success 'path.toplevel absolute and relative' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + mkdir -p repo/sub && + cd repo/sub && + + ROOT="$(test-tool path-utils real_path ..)" && + + echo "path.toplevel.absolute=$ROOT" >expect.abs && + git repo info path.toplevel.absolute >actual.abs && + test_cmp expect.abs actual.abs && + + echo "path.toplevel.relative=../" >expect.rel && + git repo info path.toplevel.relative >actual.rel && + test_cmp expect.rel actual.rel + ) +' + +test_expect_success 'path.toplevel absolute and relative in a bare repository' ' + test_when_finished "rm -rf bare.git" && + git init --bare bare.git && + ( + cd bare.git && + + echo "path.toplevel.absolute=" >expect.abs && + git repo info path.toplevel.absolute >actual.abs && + test_cmp expect.abs actual.abs && + + echo "path.toplevel.relative=" >expect.rel && + git repo info path.toplevel.relative >actual.rel && + test_cmp expect.rel actual.rel + ) +' + test_done diff --git a/t/t2400-worktree-add.sh b/t/t2400-worktree-add.sh index 87b926728ad8cf..5c105cf2529bfa 100755 --- a/t/t2400-worktree-add.sh +++ b/t/t2400-worktree-add.sh @@ -624,12 +624,12 @@ test_expect_success '"add" dwims' ' test_expect_success '"add" dwims with checkout.defaultRemote' ' test_when_finished rm -rf repo_upstream repo_dwim foo && setup_remote_repo repo_upstream repo_dwim && - git init repo_dwim && ( cd repo_dwim && git remote add repo_upstream2 ../repo_upstream && git fetch repo_upstream2 && - test_must_fail git worktree add ../foo foo && + test_must_fail git worktree add ../foo foo 2>error.actual && + test_grep "matched multiple (2) remote tracking branches" error.actual && git -c checkout.defaultRemote=repo_upstream worktree add ../foo foo && git status -uno --porcelain >status.actual && test_must_be_empty status.actual diff --git a/t/t3200-branch.sh b/t/t3200-branch.sh index b5a56ff25d871c..cdb6c6a6342a0c 100755 --- a/t/t3200-branch.sh +++ b/t/t3200-branch.sh @@ -1757,4 +1757,425 @@ test_expect_success 'errors if given a bad branch name' ' test_cmp expect actual ' +test_expect_success '--forked: setup' ' + test_create_repo forked-upstream && + ( + cd forked-upstream && + test_commit base && + git branch one base && + git branch two base + ) && + + test_create_repo forked-other && + ( + cd forked-other && + test_commit other-base && + git branch foreign other-base + ) && + + git clone forked-upstream forked && + ( + cd forked && + git remote add -f other ../forked-other && + git branch local-base && + git branch --track local-one origin/one && + git branch --track local-two origin/two && + git branch --track local-foreign other/foreign && + git branch --track local-onbase local-base && + + git checkout local-one && + test_commit --no-tag local-one-work local-one.t && + git checkout local-foreign && + test_commit --no-tag local-foreign-work local-foreign.t + ) +' + +test_expect_success '--forked filters by upstream' ' + git -C forked branch --forked origin/one \ + --format="%(refname:short)" >actual && + echo local-one >expect && + test_cmp expect actual +' + +test_expect_success '--forked filters by wildmatch' ' + git -C forked branch --forked "origin/*" \ + --format="%(refname:short)" >actual && + cat >expect <<-\EOF && + local-one + local-two + main + EOF + test_cmp expect actual +' + +test_expect_success '--forked matches branches with local upstream' ' + git -C forked branch --forked local-base \ + --format="%(refname:short)" >actual && + echo local-onbase >expect && + test_cmp expect actual +' + +test_expect_success '--forked can be repeated to widen the filter' ' + git -C forked branch --forked origin/one \ + --forked other/foreign \ + --format="%(refname:short)" >actual && + cat >expect <<-\EOF && + local-foreign + local-one + EOF + test_cmp expect actual +' + +test_expect_success '--forked combines literal and glob arguments' ' + git -C forked branch --forked local-base \ + --forked "other/*" \ + --format="%(refname:short)" >actual && + cat >expect <<-\EOF && + local-foreign + local-onbase + EOF + test_cmp expect actual +' + +test_expect_success '--forked "*/*" covers every remote-tracking upstream' ' + git -C forked branch --forked "*/*" \ + --format="%(refname:short)" >actual && + cat >expect <<-\EOF && + local-foreign + local-one + local-two + main + EOF + test_cmp expect actual +' + +test_expect_success '--forked composes with --no-merged' ' + git -C forked branch --forked "origin/*" \ + --no-merged origin/one \ + --format="%(refname:short)" >actual && + echo local-one >expect && + test_cmp expect actual +' + +test_expect_success '--forked uses the branch /HEAD points at' ' + git -C forked branch --forked origin \ + --format="%(refname:short)" >actual && + echo main >expect && + test_cmp expect actual +' + +test_expect_success '--forked narrows a argument' ' + git -C forked branch --forked "origin/*" "local-*" \ + --format="%(refname:short)" >actual && + cat >expect <<-\EOF && + local-one + local-two + EOF + test_cmp expect actual +' + +test_expect_success '--forked rejects unknown branch/pattern' ' + test_must_fail git -C forked branch --forked nope 2>err && + test_grep "not a valid branch or pattern" err +' + +test_expect_success '--forked requires a value' ' + test_must_fail git -C forked branch --forked 2>err && + test_grep "requires a value" err +' + +test_expect_success '--delete-merged: setup' ' + git init -b main upstream && + ( + cd upstream && + test_commit base && + git checkout -b next && + test_commit next-work && + git checkout main + ) && + git init -b main other && + test_commit -C other other-base && + git init -b main fork +' + +setup_repo_for_delete_merged () { + rm -rf repo && + git clone upstream repo && + ( + cd repo && + git remote add fork ../fork && + git remote add other ../other && + git config push.default current && + git fetch other + ) +} + +create_merged_branch () { + ( + cd repo && + git checkout -b "$1" --track origin/next && + git commit --allow-empty -m "$1 work" && + git push origin "$1:next" + ) +} + +check_branches () { + git for-each-ref --format="%(refname:short)" refs/heads/ >actual && + cat >expect && + test_cmp expect actual +} + +test_expect_success '--delete-merged keeps cloned main without explicit push configuration' ' + setup_repo_for_delete_merged && + ( + cd repo && + test_cmp_config origin branch.main.remote && + test_cmp_config refs/heads/main branch.main.merge && + git checkout --detach && + + git branch --delete-merged */* && + + check_branches <<-\EOF + main + EOF + ) +' + +test_expect_success '--delete-merged deletes only selected merged branches' ' + setup_repo_for_delete_merged && + create_merged_branch also-merged && + create_merged_branch merged && + ( + cd repo && + git checkout -b unmerged --track origin/next && + git commit --allow-empty -m "unmerged work" && + git checkout -b tracks-other --track other/main && + sha=$(git rev-parse --short merged) && + + git branch --dry-run --delete-merged origin/next merged \ + >actual 2>&1 && + echo "Would delete branch merged (was $sha)." >expect && + test_cmp expect actual && + git rev-parse --verify refs/heads/merged && + + check_branches <<-\EOF && + also-merged + main + merged + tracks-other + unmerged + EOF + + git branch --delete-merged origin/next merged >actual 2>&1 && + echo "Deleted branch merged (was $sha)." >expect && + test_cmp expect actual && + + check_branches <<-\EOF + also-merged + main + tracks-other + unmerged + EOF + ) +' + +test_expect_success '--delete-merged keeps main despite a different default push remote' ' + setup_repo_for_delete_merged && + create_merged_branch on-next && + create_merged_branch checked-out && + create_merged_branch upstream-gone && + ( + cd repo && + git config remote.pushDefault fork && + git checkout -b local-to-delete --track main && + git config branch.upstream-gone.merge refs/heads/topic && + git checkout -b tracks-other --track other/main && + git checkout checked-out && + + git branch --delete-merged origin/* --delete-merged main && + + check_branches <<-\EOF + checked-out + main + tracks-other + upstream-gone + EOF + ) +' + +test_expect_success '--delete-merged maps push refspecs to upstreams' ' + setup_repo_for_delete_merged && + ( + cd repo && + git checkout -b topic && + git config remote.origin.push \ + "refs/heads/topic:refs/heads/published" && + git push origin && + git branch --set-upstream-to=origin/published topic && + git checkout -b other-topic --track origin/published && + git checkout --detach && + + git branch --delete-merged origin/published && + + check_branches <<-\EOF + main + topic + EOF + ) +' + +test_expect_success '--delete-merged keeps the upstream of a surviving branch' ' + setup_repo_for_delete_merged && + create_merged_branch feature && + ( + cd repo && + git checkout -b topic --track feature && + git commit --allow-empty -m "topic work" && + + git branch --dry-run --delete-merged origin/next >out && + test_grep ! "feature" out && + + git branch --delete-merged origin/next 2>err && + test_must_be_empty err && + + check_branches <<-\EOF && + feature + main + topic + EOF + + pattern="branch\\.(feature|topic)\\.(merge|remote)" && + git config --local --get-regexp "$pattern" >actual && + cat >expect <<-\EOF && + branch.feature.remote origin + branch.feature.merge refs/heads/next + branch.topic.remote . + branch.topic.merge refs/heads/feature + EOF + test_cmp expect actual + ) +' + +test_expect_success '--delete-merged clears the deleted upstream of a protected branch' ' + setup_repo_for_delete_merged && + ( + cd repo && + git branch --track lower origin/next && + git branch --track mid lower && + git checkout -b tip --track mid && + git commit --allow-empty -m "tip work" && + sha=$(git rev-parse --short lower) && + + git branch --dry-run --delete-merged origin/next \ + --delete-merged lower >actual 2>&1 && + echo "Would delete branch lower (was $sha)." >expect && + test_cmp expect actual && + + pattern="branch\\.(lower|mid|tip)\\.(merge|remote)" && + git config --local --get-regexp "$pattern" >actual && + cat >expect <<-\EOF && + branch.lower.remote origin + branch.lower.merge refs/heads/next + branch.mid.remote . + branch.mid.merge refs/heads/lower + branch.tip.remote . + branch.tip.merge refs/heads/mid + EOF + test_cmp expect actual && + + git branch --delete-merged origin/next \ + --delete-merged lower >actual 2>&1 && + echo "Deleted branch lower (was $sha)." >expect && + test_cmp expect actual && + + check_branches <<-\EOF && + main + mid + tip + EOF + + pattern="branch\\.(mid|tip)\\.(merge|remote)" && + git config --local --get-regexp "$pattern" >actual && + cat >expect <<-\EOF && + branch.tip.remote . + branch.tip.merge refs/heads/mid + EOF + test_cmp expect actual + ) +' + +test_expect_success '--delete-merged result is independent of stacked branch names' ' + setup_repo_for_delete_merged && + ( + cd repo && + git branch --track c-lower origin/next && + git branch --track b-mid c-lower && + git checkout -b a-tip --track b-mid && + git commit --allow-empty -m "tip work" && + + git branch --delete-merged origin/next --delete-merged "c-*" && + + check_branches <<-\EOF && + a-tip + b-mid + main + EOF + + git branch --delete-merged origin/next \ + --delete-merged "c-*" >actual 2>&1 && + test_must_be_empty actual && + + check_branches <<-\EOF + a-tip + b-mid + main + EOF + ) +' + +test_expect_success '--delete-merged requires a value' ' + test_must_fail git -C forked branch --delete-merged 2>err && + test_grep "requires a value" err +' + +test_expect_success '--delete-merged honours branch..deleteMerged=false' ' + setup_repo_for_delete_merged && + create_merged_branch deleted && + create_merged_branch kept && + ( + cd repo && + git config branch.kept.deleteMerged false && + git checkout --detach && + + git branch --delete-merged origin/next 2>err && + + test_grep "Skipping .kept." err && + check_branches <<-\EOF + kept + main + EOF + ) +' + +test_expect_success "branch -d still deletes a deleteMerged=false branch" ' + setup_repo_for_delete_merged && + create_merged_branch kept && + ( + cd repo && + git config branch.kept.deleteMerged false && + git checkout --detach && + + git branch -d kept && + + check_branches <<-\EOF + main + EOF + ) +' + +test_expect_success '--dry-run without --delete-merged is rejected' ' + test_must_fail git -C forked branch --dry-run 2>err && + test_grep "requires --delete-merged" err +' + test_done diff --git a/t/t3400-rebase.sh b/t/t3400-rebase.sh index accaaba4d452d2..41d4f66588a6e3 100755 --- a/t/t3400-rebase.sh +++ b/t/t3400-rebase.sh @@ -483,7 +483,7 @@ test_expect_success 'git rebase --update-ref with core.commentChar and branch on GIT_SEQUENCE_EDITOR="cat >actual" git -c core.commentChar=% \ rebase -i --update-refs base && test_grep "% Ref refs/heads/wt-topic checked out at" actual && - test_grep "% Ref refs/heads/topic2 checked out at" actual + test_grep ! "% Ref refs/heads/topic2 checked out at" actual ' test_done diff --git a/t/t3404-rebase-interactive.sh b/t/t3404-rebase-interactive.sh index 7a3127e8a9b960..d110f0a513a699 100755 --- a/t/t3404-rebase-interactive.sh +++ b/t/t3404-rebase-interactive.sh @@ -65,6 +65,24 @@ test_expect_success 'setup' ' test_commit P fileP ' +test_expect_success MINGW 'rebase releases object database before committing' ' + test_when_finished "rm -f .git/hooks/post-commit repacked packs" && + git switch -C repack-rewrite primary && + git repack -ad && + write_script .git/hooks/post-commit <<-\EOF && + git repack -ad && + >repacked + EOF + ( + set_fake_editor && + FAKE_LINES="reword 1" GIT_TEST_LEGACY_DELETE=1 \ + git -c core.commitGraph=false rebase -i HEAD^ + ) && + test_path_is_file repacked && + ls .git/objects/pack/*.pack >packs && + test_line_count = 1 packs +' + # "exec" commands are run with the user shell by default, but this may # be non-POSIX. For example, if SHELL=zsh then ">file" doesn't work # to create a file. Unsetting SHELL avoids such non-portable behavior @@ -1986,15 +2004,23 @@ test_expect_success '--update-refs ignores non-branch decorations' ' ) && grep ^update-ref todo >actual && test_write_lines "update-ref refs/heads/no-conflict-branch" >expect && + test_grep ! "^# Ref refs/heads/update-refs checked out" todo && test_cmp expect actual ' test_expect_success '--update-refs updates refs correctly' ' + test_when_finished " + test_might_fail git symbolic-ref -d refs/heads/no-conflict-branch-alias && + test_might_fail git symbolic-ref -d refs/heads/second-alias + " && git checkout -B update-refs no-conflict-branch && git branch -f base HEAD~4 && git branch -f first HEAD~3 && git branch -f second HEAD~3 && git branch -f third HEAD~1 && + git symbolic-ref refs/heads/no-conflict-branch-alias \ + refs/heads/no-conflict-branch && + git symbolic-ref refs/heads/second-alias refs/heads/second && test_commit extra2 fileX && git commit --amend --fixup=L && @@ -2002,8 +2028,16 @@ test_expect_success '--update-refs updates refs correctly' ' test_cmp_rev HEAD~3 refs/heads/first && test_cmp_rev HEAD~3 refs/heads/second && + test_cmp_rev HEAD~3 refs/heads/second-alias && test_cmp_rev HEAD~1 refs/heads/third && test_cmp_rev HEAD refs/heads/no-conflict-branch && + test_cmp_rev HEAD refs/heads/no-conflict-branch-alias && + test_write_lines refs/heads/no-conflict-branch >expect && + git symbolic-ref refs/heads/no-conflict-branch-alias >actual && + test_cmp expect actual && + test_write_lines refs/heads/second >expect && + git symbolic-ref refs/heads/second-alias >actual && + test_cmp expect actual && q_to_tab >expect <<-\EOF && Successfully rebased and updated refs/heads/update-refs. @@ -2019,6 +2053,78 @@ test_expect_success '--update-refs updates refs correctly' ' test_cmp expect err.trimmed ' +test_expect_success '--update-refs checks resolved non-branch symref target' ' + test_when_finished " + git worktree remove --force checked-out-target-wt && + git symbolic-ref -d refs/heads/non-branch-alias && + git tag -d checked-out-target + " && + git tag checked-out-target HEAD~1 && + git symbolic-ref refs/heads/non-branch-alias refs/tags/checked-out-target && + git worktree add --detach checked-out-target-wt checked-out-target && + git -C checked-out-target-wt symbolic-ref HEAD refs/tags/checked-out-target && + + GIT_SEQUENCE_EDITOR="cat >todo" git rebase -i --update-refs HEAD~2 && + + test_grep "^# Ref refs/heads/non-branch-alias checked out at" todo && + test_write_lines refs/tags/checked-out-target >expect && + git symbolic-ref refs/heads/non-branch-alias >actual && + test_cmp expect actual +' + +test_expect_success '--update-refs deduplicates non-branch symref targets' ' + test_when_finished " + git symbolic-ref -d refs/heads/non-branch-alias-one && + git symbolic-ref -d refs/heads/non-branch-alias-two && + git tag -d shared-non-branch-target + " && + git tag shared-non-branch-target HEAD~1 && + git symbolic-ref refs/heads/non-branch-alias-one \ + refs/tags/shared-non-branch-target && + git symbolic-ref refs/heads/non-branch-alias-two \ + refs/tags/shared-non-branch-target && + + GIT_SEQUENCE_EDITOR=: git rebase -i --force-rebase --update-refs HEAD~2 && + + test_cmp_rev HEAD~1 refs/heads/non-branch-alias-one && + test_cmp_rev HEAD~1 refs/heads/non-branch-alias-two && + test_write_lines refs/tags/shared-non-branch-target >expect && + git symbolic-ref refs/heads/non-branch-alias-one >actual && + test_cmp expect actual && + git symbolic-ref refs/heads/non-branch-alias-two >actual && + test_cmp expect actual +' + +test_expect_success '--update-refs honors non-branch symref reservations' ' + test_when_finished " + test_might_fail git worktree remove --force reserved-target-wt && + test_might_fail git symbolic-ref -d \ + refs/heads/reserved-non-branch-alias-one && + test_might_fail git symbolic-ref -d \ + refs/heads/reserved-non-branch-alias-two && + test_might_fail git tag -d reserved-non-branch-target + " && + git tag reserved-non-branch-target HEAD~1 && + git symbolic-ref refs/heads/reserved-non-branch-alias-one \ + refs/tags/reserved-non-branch-target && + git symbolic-ref refs/heads/reserved-non-branch-alias-two \ + refs/tags/reserved-non-branch-target && + git worktree add --detach reserved-target-wt HEAD && + wt_gitdir=$(git -C reserved-target-wt rev-parse --absolute-git-dir) && + mkdir -p "$wt_gitdir/rebase-merge" && + old_oid=$(git rev-parse refs/heads/reserved-non-branch-alias-one) && + test_write_lines refs/heads/reserved-non-branch-alias-one \ + "$old_oid" "$old_oid" >"$wt_gitdir/rebase-merge/update-refs" && + + GIT_SEQUENCE_EDITOR="cat >todo" git rebase -i --update-refs HEAD~2 && + + test_grep "^# Ref refs/heads/reserved-non-branch-alias-one checked out at" \ + todo && + test_grep "^# Ref refs/heads/reserved-non-branch-alias-two checked out at" \ + todo && + test_grep ! "^update-ref refs/heads/reserved-non-branch-alias" todo +' + test_expect_success 'respect user edits to update-ref steps' ' git checkout -B update-refs-break no-conflict-branch && git branch -f base HEAD~4 && diff --git a/t/t3436-rebase-more-options.sh b/t/t3436-rebase-more-options.sh index 94671d3c465046..c84c6717ab41f8 100755 --- a/t/t3436-rebase-more-options.sh +++ b/t/t3436-rebase-more-options.sh @@ -201,6 +201,58 @@ test_expect_success '--ignore-date is an alias for --reset-author-date' ' test_atime_is_ignored -2 ' +test_expect_success '--no-edit on continue uses existing commit message' ' + git checkout commit2 && + test_must_fail git rebase -m --onto commit2^^ commit2^ && + echo resolved >foo && + git add foo && + write_script fail-if-editor-invoked <<-\EOF && + echo editor invoked >&2 + exit 1 + EOF + GIT_EDITOR=./fail-if-editor-invoked git rebase --continue --no-edit && + git log --format=%s -1 >actual && + echo commit2 >expect && + test_cmp expect actual +' + +test_expect_success '--no-edit cannot be used when starting a rebase' ' + test_must_fail git rebase --no-edit -m main side 2>err && + test_grep "only be used with --continue" err +' + +test_expect_success 'rebase.noEdit skips editor on continue' ' + git config rebase.noEdit true && + git checkout commit2 && + test_must_fail git rebase -m --onto commit2^^ commit2^ && + echo resolved >foo && + git add foo && + write_script fail-if-editor-invoked <<-\EOF && + echo editor invoked >&2 + exit 1 + EOF + GIT_EDITOR=./fail-if-editor-invoked git rebase --continue && + git log --format=%s -1 >actual && + echo commit2 >expect && + test_cmp expect actual +' + +test_expect_success '--edit on continue overrides rebase.noEdit' ' + git config rebase.noEdit true && + git checkout commit2 && + test_must_fail git rebase -m --onto commit2^^ commit2^ && + echo resolved >foo && + git add foo && + ( + set_fake_editor && + FAKE_COMMIT_MESSAGE="edited on continue" \ + git rebase --continue --edit + ) && + test_write_lines "edited on continue" "" >expect && + git log --format=%B -1 >actual && + test_cmp expect actual +' + # This must be the last test in this file test_expect_success '$EDITOR and friends are unchanged' ' test_editor_unchanged diff --git a/t/t3455-history-squash.sh b/t/t3455-history-squash.sh new file mode 100755 index 00000000000000..d21e9d9fc468fd --- /dev/null +++ b/t/t3455-history-squash.sh @@ -0,0 +1,886 @@ +#!/bin/sh + +test_description='tests for git-history squash subcommand' + +. ./test-lib.sh + +stage_file () { + printf "%s\n" "$1" >file && + git add file +} + +commit_with_message () { + printf "%b" "$1" >msg && + git commit --allow-empty -qF msg +} + +check_commit_count () { + git rev-list --count "$1" >actual && + echo "$2" >expect && + test_cmp expect actual +} + +check_log_subjects () { + git log --format="%s" "$1" >actual && + cat >expect && + test_cmp expect actual +} + +check_log_messages () { + git log --format="%B" "$1" >actual && + cat >expect && + test_cmp expect actual +} + +# Checks that the author data of two commits matches +# Usage: check_commit_author +check_commit_author () { + git show -s --format="%an <%ae> %ad" "$1" >expect && + git show -s --format="%an <%ae> %ad" "$2" >actual && + test_cmp expect actual +} + +test_expect_success 'setup linear history touching two files' ' + test_commit base file a start && + GIT_AUTHOR_NAME=One GIT_AUTHOR_EMAIL=one@example.com \ + test_commit one other x && + GIT_AUTHOR_NAME=Two GIT_AUTHOR_EMAIL=two@example.com \ + test_commit two file c && + GIT_AUTHOR_NAME=Three GIT_AUTHOR_EMAIL=three@example.com \ + test_commit three file d +' + +test_expect_success 'errors on missing range argument' ' + test_must_fail git history squash 2>err && + test_grep "expects a revision range" err +' + +test_expect_success 'errors on an empty range' ' + test_must_fail git history squash HEAD..HEAD 2>err && + test_grep "the revision range is empty" err +' + +test_expect_success 'errors on a single revision that is not a range' ' + test_must_fail git history squash HEAD 2>err && + test_grep "not a .*range" err && + test_must_fail git history squash HEAD~1 2>err && + test_grep "not a .*range" err +' + +test_expect_success 'errors on a range holding a single commit' ' + git reset --hard three && + head_before=$(git rev-parse HEAD) && + + test_must_fail git history squash "HEAD^!" 2>err && + test_grep "single commit; nothing to squash" err && + test_cmp_rev "$head_before" HEAD +' + +test_expect_success 'rejects root commit' ' + # create a disconnected root commit + oid=$(git commit-tree -m root three^{tree}) && + # because we pass --ancestry-path when calling setup_revs() it the + # revision walk will only include commits decended from $oid so + # we need to give it another --ancestry-path commit to actually walk + # any commits. + test_must_fail git history squash --ancestry-path=start $oid..three 2>err && + echo "error: cannot squash down to root commit" >expect && + test_cmp expect err +' + +test_expect_success 'rejects multiple tips' ' + oid=$(git commit-tree -m tip -p start^0 three^{tree}) && + test_must_fail git history squash ^start $oid three~1 2>err && + echo "error: the revision range contains more than one tip commit" >expect && + test_cmp expect err && + + git reset --hard three && + git history squash --no-edit ^start three~1 three && + test_cmp_rev HEAD~1 start^0 && + test_cmp_rev HEAD^{tree} three^{tree} +' + +test_expect_success 'accepts multiple revision arguments with an exclusion' ' + git reset --hard three && + git branch -f keep HEAD~2 && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start..HEAD ^keep && + + git reflog -1 --format=%gs >actual && + echo "squash: updating start..HEAD ^keep" >expect && + test_cmp expect actual && + + check_log_subjects start..HEAD <<-\EOF && + two + one + EOF + test_cmp_rev keep HEAD~1 && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + + git branch -D keep +' + +test_expect_success 'squashes a branch the current branch is not on' ' + git reset --hard three && + main=$(git symbolic-ref --short HEAD) && + head_before=$(git rev-parse HEAD) && + git checkout -b off-history start && + test_commit --no-tag off-one off a && + test_commit --no-tag off-two off b && + git checkout "$main" && + + git history squash --no-edit start..off-history && + + check_commit_count start..off-history 1 && + test_cmp_rev "$head_before" HEAD && + + git branch -D off-history +' + +test_expect_success 'squashes a range into a single commit without changing the tree' ' + git reset --hard three && + head_before=$(git rev-parse HEAD) && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit --dry-run start.. >out && + predicted=$(awk "/^update refs\/heads\// {print \$3}" out) && + test_cmp_rev "$head_before" HEAD && + + git history squash --no-edit start.. && + + test "$predicted" = "$(git rev-parse HEAD)" && + check_commit_count start..HEAD 1 && + test_cmp_rev start HEAD^ && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + check_log_subjects -1 <<-\EOF && + one + EOF + git reflog >reflog && + test_grep "squash: updating" reflog +' + +test_expect_success 'sanitizes rev-list walk options, before and after --' ' + git reset --hard three && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit --date-order start.. 2>err && + test_grep "ignoring rev-list options" err && + test_cmp_rev start HEAD^ && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + + git reset --hard three && + git history squash --no-edit -- --reverse start.. 2>err && + test_grep "ignoring rev-list options" err && + test_cmp_rev start HEAD^ && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" +' + +test_expect_success 'squashes an interior range and replays descendants verbatim' ' + git reset --hard three && + final_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start..@~1 && + + check_log_subjects start..HEAD <<-\EOF && + three + one + EOF + + test_cmp_rev start HEAD~2 && + test "$final_tree" = "$(git rev-parse HEAD^{tree})" +' + +test_expect_success 'squashes when the base is the root commit' ' + git reset --hard three && + root=$(git rev-list --max-parents=0 HEAD) && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit "$root.." && + + check_commit_count "$root..HEAD" 1 && + test_cmp_rev "$root" HEAD^ && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" +' + +test_expect_success 'squashing a mix of fixups' ' + git reset --hard three && + echo fix >file && + git commit --fixup=two -a && + echo really fix >file && + git commit --fixup=one -a && + echo really really fix >file && + git commit --fixup=HEAD~1 -a && # fixup! two + echo really really really fix >file && + git commit --fixup=HEAD~1 -a && # fixup! one + + # squashing fixup! with a target that is not being squashed fails + test_must_fail git history squash one.. 2>err && + test_grep "^error: cannot squash .* (fixup! one): its target is not being squashed" err && + + # squashing fixup! into fixup! with a different target fails + test_must_fail git history squash HEAD~4.. 2>err && # HEAD~4 is fixup! two + test_grep "^error: cannot squash .* (fixup! one): its target is not being squashed" err && + + # squashing a sequence of fixup! commits into their targets + git history squash --no-edit start..HEAD~1 && + test_cmp_rev start HEAD~2 && + check_commit_author one HEAD~1 && + test_commit_message HEAD~1 -m one && + + # squashing "fixup! fixup! " into "" + git history squash --no-edit start.. && + test_cmp_rev start HEAD~1 && + check_commit_author one HEAD && + test_commit_message HEAD -m one +' + +test_expect_success 'squashing "squash!" messages' ' + git reset --hard two && + echo fix >file && + git commit --fixup=HEAD -a && + oldest=$(git rev-parse HEAD) && + echo better fix >file && + git commit -a -F - <<-EOF && + squash! $(git rev-parse two) + + Append this + EOF + + echo an even better fix >file && + git commit -a -F - <<-EOF && + squash! squash! two + + Append this as well + EOF + + # must edit when squashing "squash!" into its target + test_must_fail git history squash --no-edit two^.. 2>err && + test_grep "^error: squashing .* (squash! [a-f0-9]*) would discard its message" err && + + # squashing "squash!" into "fixup!" appends messages and changes + # subject prefix + git history squash --no-edit two.. && + test_cmp_rev HEAD^ two && + test_commit_message HEAD <<-\EOF && + squash! two + + Append this + + Append this as well + EOF + check_commit_author "$oldest" HEAD && + + git commit --allow-empty -F - <<-\EOF && + amend! two + + A new message + EOF + + # "amend!" does not replace "squash!" + test_must_fail git history squash --no-edit HEAD~2.. 2>err && + test_grep "^error: squashing .* (amend! two) would overwrite .squash!. message" err +' + +test_expect_success '--no-edit uses last "amend!" message without an editor' ' + git reset --hard three && + write_script editor <<-\EOF && + exit 1 + EOF + test_set_editor "$(pwd)/editor" && + echo fix >file && + git commit --author="Fix Me " --fixup=HEAD -a && + git commit --allow-empty -F - <<-EOF && + amend! $(git rev-parse --short HEAD) + + The first reword + + More detail + EOF + + git commit --allow-empty -F - <<-\EOF && + amend! three + + The second reword + + Extra detail + EOF + + test_commit WIP && + + cat >msg <<-EOF && + amend! $(git rev-parse --short HEAD^) + + The third reword + + Excruciating detail + EOF + + git commit --author="Someone Else " --allow-empty \ + -F msg && + + # squashing amend! updates the commit message + git history squash --no-edit three^.. && + sed -e 1,2d msg | test_commit_message HEAD && + check_commit_author three HEAD && + test_cmp_rev HEAD^ three^ && + + # squashing amend! into fixup! updates subject prefix + git reset --hard HEAD@{1} && + git history squash --no-edit three.. && + sed "1s/.*/amend! three/" msg | test_commit_message HEAD && + check_commit_author HEAD@{1}~4 HEAD && + test_cmp_rev HEAD^ three && + + # squashing amend! into amend! keeps original subject line + git reset --hard HEAD@{1} && + git history squash --no-edit HEAD~3.. && + sed "1s/.*/amend! three/" msg | test_commit_message HEAD && + test_cmp_rev HEAD~3 three && + + # all amend! messages must target the first commit + git reset --hard HEAD@{1} && + git commit --allow-empty -F - <<-\EOF && + amend! WIP + + The real message + EOF + + test_must_fail git history squash --no-edit HEAD~4.. 2>err && + test_grep "^error: cannot squash .* that does not target" err && + + # amend! message that targets commit that is not in range is rejected + test_must_fail git history squash --no-edit HEAD~3.. 2>err && + test_grep "^error: cannot squash .* target is not being squashed" err && + test_set_editor : +' + +test_expect_success 'squashing fixups into a merge' ' + test_when_finished \ + "git switch -f $GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME; \ + git branch -D feature" && + git checkout -f start && + test_commit F1 && + git checkout -b feature start && + test_commit F2 && + git merge F1 && + echo fixed >F1.t && + cat >msg <<-EOF && + amend! $(git rev-parse HEAD) + + merge F1 and F2 + + reworded + EOF + + git commit -a -F msg && + git history squash --no-edit HEAD^^! HEAD && + test_cmp_rev HEAD^1 F2 && + test_cmp_rev HEAD^2 F1 && + test_cmp_rev HEAD@{1}^{tree} HEAD^{tree} && + sed 1,2d msg | test_commit_message HEAD +' + +test_expect_success 'edits every message and aborts on an empty result' ' + git reset --hard start && + stage_file b && + git commit -m "re-one subject" -m "re-one body line" && + test_commit --no-tag re-two file c && + test_commit re-three file d && + head_before=$(git rev-parse HEAD) && + + write_script empty-editor <<-\EOF && + >"$1" + EOF + test_set_editor "$(pwd)/empty-editor" && + test_must_fail git history squash start.. 2>err && + test_grep "Aborting commit due to empty commit message" err && + test_cmp_rev "$head_before" HEAD && + + write_script editor <<-\EOF && + cat "$1" >edited && + echo combined >"$1" + EOF + test_set_editor "$(pwd)/editor" && + git history squash start.. && + + cat >expect <<-EOF && + # This is a combination of 3 commits. + # This is the 1st commit message: + + re-one subject + + re-one body line + + # This is the commit message #2: + + re-two + + # This is the commit message #3: + + re-three + + # Please enter the commit message for the squash changes. Lines starting + # with ${SQ}#${SQ} will be ignored, and an empty message aborts the commit. + # Changes to be committed: + # modified: file + # + EOF + test_cmp expect edited && + check_log_subjects -1 <<-\EOF + combined + EOF +' + +test_expect_success 'handles fixup!, squash! and amend! like rebase' ' + git reset --hard start && + test_commit --no-tag mark-base file b && + stage_file c && + commit_with_message "fixup! mark-base\n\nfixup body\n" && + stage_file d && + commit_with_message "squash! mark-base\n\nsquash remark\n" && + stage_file e && + commit_with_message "amend! mark-base\n\namended message\n" && + + write_script editor <<-\EOF && + cat "$1" >edited + EOF + test_set_editor "$(pwd)/editor" && + git history squash start.. && + + cat >expect <<-EOF && + # This is a combination of 4 commits. + # This is the 1st commit message: + + mark-base + + # The commit message #2 will be skipped: + + # fixup! mark-base + # + # fixup body + + # This is the commit message #3: + + # squash! mark-base + + squash remark + + # This is the commit message #4: + + # amend! mark-base + + amended message + + # Please enter the commit message for the squash changes. Lines starting + # with ${SQ}#${SQ} will be ignored, and an empty message aborts the commit. + # Changes to be committed: + # modified: file + # + EOF + test_cmp expect edited && + check_log_messages -1 <<-\EOF + mark-base + + squash remark + + amended message + + EOF +' + +test_expect_success 'groups fixups under their targets in the editor' ' + git reset --hard start && + test_commit --no-tag alpha file a1 && + test_commit --no-tag beta file b1 && + stage_file a2 && + commit_with_message "fixup! alpha\n" && + stage_file b2 && + commit_with_message "fixup! beta\n" && + + write_script editor <<-\EOF && + cat "$1" >edited + EOF + test_set_editor "$(pwd)/editor" && + git history squash start.. && + + cat >expect <<-EOF && + # This is a combination of 4 commits. + # This is the 1st commit message: + + alpha + + # The commit message #2 will be skipped: + + # fixup! alpha + + # This is the commit message #3: + + beta + + # The commit message #4 will be skipped: + + # fixup! beta + + # Please enter the commit message for the squash changes. Lines starting + # with ${SQ}#${SQ} will be ignored, and an empty message aborts the commit. + # Changes to be committed: + # modified: file + # + EOF + test_cmp expect edited +' + +test_expect_success 'lets amend! replace its target message in the editor' ' + git reset --hard start && + test_commit --no-tag mark-base file b && + stage_file c && + commit_with_message "amend! mark-base\n\namended message\n" && + stage_file d && + commit_with_message "squash! mark-base\n\nsquash remark\n" && + + write_script editor <<-\EOF && + cat "$1" >edited + EOF + test_set_editor "$(pwd)/editor" && + git history squash start.. && + + cat >expect <<-EOF && + # This is a combination of 3 commits. + # The 1st commit message will be skipped: + + # mark-base + + # This is the commit message #2: + + # amend! mark-base + + amended message + + # This is the commit message #3: + + # squash! mark-base + + squash remark + + # Please enter the commit message for the squash changes. Lines starting + # with ${SQ}#${SQ} will be ignored, and an empty message aborts the commit. + # Changes to be committed: + # modified: file + # + EOF + test_cmp expect edited && + check_log_messages -1 <<-\EOF + amended message + + squash remark + + EOF +' + +test_expect_success '--update-refs=head only moves HEAD' ' + git reset --hard three && + git branch -f other HEAD && + other_before=$(git rev-parse other) && + + git history squash --no-edit --update-refs=head start.. && + + check_commit_count start..HEAD 1 && + test_cmp_rev "$other_before" other +' + +test_expect_success 'refuses to fold a range a branch points into' ' + test_when_finished \ + "git switch -f $GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME; \ + git branch -D feature" && + git checkout -f -b feature start && + test_commit C1 && + test_commit C2 && + git checkout -b topic-1 start && + test_commit C3 && + test_commit C4 && + git checkout C3 && + test_commit C5 && + git checkout feature && + git merge C5 && + test_commit C6 && + git checkout -b topic-2 C2 && + test_commit C7 && + git checkout feature && + + test_must_fail git history squash start.. 2>err && + test_grep "^error: the following branches cannot be rewritten" err && + test_grep "^ topic-1$" err && + test_grep "^ topic-2$" err && + test_grep "^hint: .* --update-refs=head" err && + test_cmp_rev C6 HEAD && + + # squash succeeds with --update-refs=head + git history squash --no-edit --update-refs=head start.. && + test_cmp_rev start HEAD^ && + test_cmp_rev C6^{tree} HEAD^{tree} && + test_cmp_rev C6 HEAD@{1} +' + +test_expect_success 'advice.historyUpdateRefs silences the hint' ' + git reset --hard three && + git branch -f mid HEAD~1 && + head_before=$(git rev-parse HEAD) && + + test_must_fail git -c advice.historyUpdateRefs=false \ + history squash start.. 2>err && + test_grep "^error: the following branches cannot be rewritten" err && + test_grep "^ mid$" err && + test_grep ! "hint:" err && + test_cmp_rev "$head_before" HEAD && + + git branch -D mid +' + +test_expect_success 'leaves tags and remote-tracking refs unchanged' ' + git reset --hard three && + git tag -f mark HEAD~1 && + git update-ref refs/remotes/origin/mark HEAD~1 && + mark_before=$(git rev-parse mark) && + + git history squash --no-edit start.. && + + test_cmp_rev "$mark_before" mark && + test_cmp_rev "$mark_before" refs/remotes/origin/mark && + + git tag -d mark && + git update-ref -d refs/remotes/origin/mark +' + +test_expect_success 'squashes a range whose internal merge has a single base' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + test_commit --no-tag before-side file b && + git checkout -b inner-side && + test_commit --no-tag on-inner-side inner x && + git checkout "$main" && + test_commit --no-tag after-side file c && + git merge --no-ff -m merge inner-side && + git branch -D inner-side && + test_commit --no-tag after-merge file d && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start.. && + + check_commit_count start..HEAD 1 && + check_log_subjects -1 <<-\EOF && + before-side + EOF + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + test_path_is_file inner +' + +test_expect_success 'folds a merge of a branch that forked at the base' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + git checkout -b base-fork-side && + test_commit --no-tag base-fork-side side x && + git checkout "$main" && + test_commit --no-tag base-fork-main file b && + git merge --no-ff -m "merge base-fork-side" base-fork-side && + git branch -D base-fork-side && + test_commit --no-tag base-fork-tail file c && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start.. && + + check_commit_count start..HEAD 1 && + test_cmp_rev start HEAD^ && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + test_path_is_file side +' + +test_expect_success 'refuses a merge whose other parent is outside the range' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + git checkout -b outside-parent && + test_commit --no-tag outside-parent outside x && + git checkout "$main" && + test_commit --no-tag outside-main file b && + base=$(git rev-parse HEAD) && + test_commit --no-tag outside-mid file c && + git merge --no-ff -m "merge outside-parent" outside-parent && + git branch -D outside-parent && + merged=$(git rev-parse HEAD) && + + test_must_fail git history squash "$base.." 2>err && + test_grep "parent .* of commit .* is outside the revision range" err && + test_cmp_rev "$merged" HEAD +' + +test_expect_success 'folds a range whose tip is a merge commit' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + test_commit --no-tag tipmerge-base file b && + git checkout -b tipmerge-side && + test_commit --no-tag tipmerge-side side x && + git checkout "$main" && + test_commit --no-tag tipmerge-main file c && + git merge --no-ff -m "merge tipmerge-side" tipmerge-side && + git branch -D tipmerge-side && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start.. && + + check_commit_count start..HEAD 1 && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + test_path_is_file side +' + +test_expect_success 'folds a range whose base is a merge commit' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + git checkout -b basemerge-side && + test_commit --no-tag basemerge-side side x && + git checkout "$main" && + test_commit --no-tag basemerge-main file b && + git merge --no-ff -m "merge basemerge-side" basemerge-side && + git branch -D basemerge-side && + base=$(git rev-parse HEAD) && + test_commit --no-tag basemerge-one file c && + test_commit --no-tag basemerge-two file d && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit "$base.." && + + check_commit_count "$base..HEAD" 1 && + test_cmp_rev "$base" HEAD^ && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" +' + +test_expect_success 'folds a range with two interior merges' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + test_commit --no-tag two-merge-a file a1 && + git checkout -b two-merge-s1 && + test_commit --no-tag two-merge-s1 s1 x && + git checkout "$main" && + git merge --no-ff -m "merge s1" two-merge-s1 && + test_commit --no-tag two-merge-b file b1 && + git checkout -b two-merge-s2 && + test_commit --no-tag two-merge-s2 s2 y && + git checkout "$main" && + git merge --no-ff -m "merge s2" two-merge-s2 && + git branch -D two-merge-s1 two-merge-s2 && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start.. && + + check_commit_count start..HEAD 1 && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + test_path_is_file s1 && + test_path_is_file s2 +' + +test_expect_success 'folds a range with a nested merge' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + git checkout -b nested-outer && + test_commit --no-tag nested-outer outer x && + git checkout -b nested-inner && + test_commit --no-tag nested-inner inner y && + git checkout nested-outer && + git merge --no-ff -m "merge inner" nested-inner && + git checkout "$main" && + test_commit --no-tag nested-main file b1 && + git merge --no-ff -m "merge outer" nested-outer && + git branch -D nested-outer nested-inner && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start.. && + + check_commit_count start..HEAD 1 && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + test_path_is_file outer && + test_path_is_file inner +' + +test_expect_success 'folds a range with an octopus merge' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + test_commit --no-tag octo-base file a1 && + git checkout -b octo-1 && + test_commit --no-tag octo-1 o1 x && + git checkout "$main" && + git checkout -b octo-2 && + test_commit --no-tag octo-2 o2 y && + git checkout "$main" && + git merge --no-ff -m octopus octo-1 octo-2 && + git branch -D octo-1 octo-2 && + tip_tree=$(git rev-parse HEAD^{tree}) && + + git history squash --no-edit start.. && + + check_commit_count start..HEAD 1 && + test "$tip_tree" = "$(git rev-parse HEAD^{tree})" && + test_path_is_file o1 && + test_path_is_file o2 +' + +test_expect_success 'refuses an octopus merge with an arm forked before the base' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + git checkout -b octo-pre && + test_commit octo-pre-side pside x && + git checkout "$main" && + test_commit octo-pre-main file b1 && + octo_base=$(git rev-parse HEAD) && + git checkout -b octo-within && + test_commit --no-tag octo-within wside y && + git checkout "$main" && + git merge --no-ff -m octopus octo-pre octo-within && + merged=$(git rev-parse HEAD) && + git branch -D octo-pre octo-within && + + test_must_fail git history squash "$octo_base.." 2>err && + test_grep "parent .* of commit .* is outside the revision range" err && + test_cmp_rev "$merged" HEAD +' + +test_expect_success 'refuses when a descendant above the range is a merge' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + test_commit --no-tag desc-one file b && + test_commit --no-tag desc-two file c && + git tag desc-tip && + git checkout -b desc-above && + test_commit --no-tag desc-above above x && + git checkout "$main" && + test_commit --no-tag desc-main file d && + git merge --no-ff -m "merge desc-above" desc-above && + git branch -D desc-above && + head_before=$(git rev-parse HEAD) && + + test_must_fail git history squash --no-edit start..desc-tip 2>err && + test_grep "merge commits is not supported" err && + test_cmp_rev "$head_before" HEAD +' + +test_expect_success 'refuses to fold a range a ref points into at a merge' ' + git reset --hard start && + main=$(git symbolic-ref --short HEAD) && + test_commit --no-tag refmerge-base file b && + git checkout -b refmerge-side && + test_commit --no-tag refmerge-side side x && + git checkout "$main" && + test_commit --no-tag refmerge-main file c && + git merge --no-ff -m "interior merge" refmerge-side && + git branch -D refmerge-side && + git branch at-merge HEAD && + test_commit --no-tag refmerge-tail file d && + head_before=$(git rev-parse HEAD) && + + test_must_fail git history squash start.. 2>err && + test_grep "^error: the following branches cannot be rewritten" err && + test_grep "^ at-merge$" err && + test_cmp_rev "$head_before" HEAD && + + git branch -D at-merge +' + +test_done diff --git a/t/t3650-replay-basics.sh b/t/t3650-replay-basics.sh index 3353bc4a4dc6ed..255bae58460297 100755 --- a/t/t3650-replay-basics.sh +++ b/t/t3650-replay-basics.sh @@ -52,8 +52,19 @@ test_expect_success 'setup' ' test_merge P O --no-ff && git switch main && + git switch --orphan unrelated && + test_commit unrelated-root && + git switch -c conflict B && - test_commit C.conflict C.t conflict + test_commit C.conflict C.t conflict && + git branch -D unrelated && + + git switch -c divergent-x main && + test_commit X && + git switch -c divergent-y main && + test_commit Y && + git switch divergent-x && + test_merge Z divergent-y --no-ff ' test_expect_success 'setup bare' ' @@ -565,4 +576,100 @@ test_expect_success '--onto with --ref rejects multiple revision ranges' ' test_grep "cannot be used with multiple revision ranges" err ' +test_expect_success 'replay to rebase merge commit with --linearize' ' + git replay --ref-action=print --linearize \ + --onto main I..topic-with-merge >result && + + test_line_count = 1 result && + + git log --format=%s $(cut -f 3 -d " " result) >actual && + test_write_lines O N J M L B A >expect && + test_cmp expect actual +' + +test_expect_success 'replay to rebase merge commit with --linearize down to the root commit' ' + git replay --ref-action=print --linearize \ + --onto unrelated-root topic-with-merge >result && + + test_line_count = 1 result && + + git log --format=%s $(cut -f 3 -d " " result) >actual && + test_write_lines O N J I B A unrelated-root >expect && + test_cmp expect actual +' + +test_expect_success 'replay to cherry-pick merge commit with --linearize' ' + git replay --ref-action=print --linearize \ + --advance main I..topic-with-merge >result && + + test_line_count = 1 result && + + git log --format=%s $(cut -f 3 -d " " result) >actual && + test_write_lines O N J M L B A >expect && + test_cmp expect actual && + + printf "update refs/heads/main " >expect && + printf "%s " $(cut -f 3 -d " " result) >>expect && + git rev-parse main >>expect && + test_cmp expect result +' + +test_expect_success 'replay --linearize produces the same patches' ' + git replay --ref-action=print --linearize \ + --onto main I..topic-with-merge >result && + + test_line_count = 1 result && + tip=$(cut -f 3 -d " " result) && + + # range-diff does not care about the dropped merge, + # so the original commits (I..topic-with-merge) + # and the replayed chain (main..tip) must produce identical patches. + git range-diff I..topic-with-merge main..$tip >out && + test_file_not_empty out && + test_grep ! -v "=" out && + + git log --oneline main..$tip >out && + test_line_count = 3 out +' + +test_expect_success '--linearize rejects multiple revision ranges' ' + test_must_fail git replay --ref-action=print --linearize \ + --onto main ^B topic2 topic3 topic4 2>err && + test_grep "cannot be used with multiple revision ranges" err +' + +test_expect_success 'replay with --linearize of a divergent merge keeps both sides' ' + git replay --ref-action=print --linearize \ + --onto main main..divergent-x >result && + test_line_count = 1 result && + tip=$(cut -f 3 -d " " result) && + + # The merge Z is dropped, but both X and Y are linearized onto main; + # neither side is lost. + git log --format=%s main..$tip >actual && + test_write_lines Y X >expect && + test_cmp expect actual +' + +test_expect_success '--linearize and --contained cannot be used together' ' + test_must_fail git replay --ref-action=print --linearize --contained \ + --onto main ^B topic-with-merge 2>err && + test_grep "cannot be used together" err +' + +test_expect_success 'replay --revert with --linearize reverts a range containing a merge' ' + git replay --ref-action=print --revert=divergent-x --linearize \ + main..divergent-x >result && + test_line_count = 1 result && + tip=$(cut -f 3 -d " " result) && + + git log --format=%s $tip >actual && + test_write_lines \ + "Revert \"X\"" "Revert \"Y\"" Z Y X M L B A >expect && + test_cmp expect actual && + + test_must_fail git cat-file -e $tip:X.t && + test_must_fail git cat-file -e $tip:Y.t +' + test_done diff --git a/t/t3702-add-edit.sh b/t/t3702-add-edit.sh index 8bacacbac6807c..f6285640051e59 100755 --- a/t/t3702-add-edit.sh +++ b/t/t3702-add-edit.sh @@ -124,5 +124,15 @@ test_expect_success 'add -e notices editor failure' ' test_must_fail env GIT_EDITOR=false git add -e && test_expect_code 1 git diff --exit-code ' +test_expect_success 'add -e works from a subdirectory' ' + git reset --hard && + echo change >>file && + mkdir -p subdir && + ( + cd subdir && + GIT_EDITOR=cat git add -e ../file + ) && + git diff --cached | grep -q "^+change" +' test_done diff --git a/t/t4080-diff-process.sh b/t/t4080-diff-process.sh new file mode 100755 index 00000000000000..a8efa19416b675 --- /dev/null +++ b/t/t4080-diff-process.sh @@ -0,0 +1,593 @@ +#!/bin/sh + +test_description='diff..process: oid-only hunk requests' + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh + +# See t/helper/test-diff-process-backend.c for the process implementation +# and available --mode= options. + +BACKEND="test-tool diff-process-backend" + +test_expect_success 'setup' ' + echo "*.c diff=cdiff" >.gitattributes && + git add .gitattributes && + + # 10 lines, changes at 5-6 and 9-10 between the two commits. + cat >pair.c <<-\EOF && + line1 + line2 + line3 + line4 + original5 + original6 + line7 + line8 + line9 + line10 + EOF + git add pair.c && + git commit -m "add pair.c" && + + cat >pair.c <<-\EOF && + line1 + line2 + line3 + line4 + changed5 + changed6 + line7 + line8 + changed9 + changed10 + EOF + git add pair.c && + git commit -m "change pair.c" +' + +test_expect_success 'an oid-capable process answers blame by object names alone' ' + test_when_finished "rm -f backend.log" && + ORIG=$(git rev-parse --short HEAD~1) && + CHANGE=$(git rev-parse --short HEAD) && + # The process reports only lines 5-6 as changed, so blame attributes + # lines 9-10 to the original commit even though the builtin diff + # would show them as changed. + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + blame pair.c >actual && + sed -n "9p" actual >line9 && + sed -n "10p" actual >line10 && + test_grep "$ORIG" line9 && + test_grep "$ORIG" line10 && + sed -n "5p" actual >line5 && + test_grep "$CHANGE" line5 && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'an oid-capable process answers --numstat by object names alone' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --format= --numstat -- pair.c >actual && + printf "2\t2\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'need-content falls through to the builtin diff' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-need-content --log=backend.log" \ + log -1 --format= --numstat -- pair.c >actual && + printf "4\t4\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'a warmed hunk store does not override process hunks' ' + test_when_finished "git diff-hunks clear" && + ORIG=$(git rev-parse --short HEAD~1) && + GIT_DIFF_HUNKS_WRITE=1 git log -2 --stat -- pair.c >/dev/null && + + # Control: without a process, blame is served from the store. + git blame --show-stats pair.c >stats && + test_grep "num precomputed hits: 1" stats && + + # The store holds the builtin hunks, but a process-capable driver + # makes the process authoritative, so blame must reflect the + # process hunks (only lines 5-6), not a store hit. + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed" \ + blame pair.c >actual && + sed -n "9p" actual >line9 && + test_grep "$ORIG" line9 +' + +test_expect_success 'a worktree side is not asked by object names' ' + test_when_finished "rm -f backend.log && git checkout -- pair.c" && + echo "worktree edit" >>pair.c && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff --numstat -- pair.c >actual && + printf "1\t0\tpair.c\n" >expect && + test_cmp expect actual && + test_path_is_missing backend.log +' + +test_expect_success 'diff process bypassed by --no-ext-diff' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --format= --numstat --no-ext-diff -- pair.c >actual && + printf "4\t4\tpair.c\n" >expect && + test_cmp expect actual && + test_path_is_missing backend.log +' + +test_expect_success 'format-patch keeps its diffstat off the process' ' + test_when_finished "rm -f backend.log" && + # format-patch emits a diffstat, and a diffstat consults the + # process, but the gate keeps it builtin so a generated patch + # applies for recipients without the process. + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + format-patch -1 --stdout --stat -- pair.c >actual && + test_grep "^+changed9" actual && + test_path_is_missing backend.log +' + +test_expect_success 'format-patch --ext-diff keeps its diffstat off the process' ' + test_when_finished "rm -f backend.log" && + # The gate holds even when --ext-diff enables the external command. + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + format-patch -1 --stdout --ext-diff --stat -- pair.c >actual && + test_grep "^+changed9" actual && + test_path_is_missing backend.log +' + +test_expect_success 'diff process not consulted by plumbing diff commands' ' + test_when_finished "rm -f backend.log && git checkout -f HEAD -- pair.c" && + # diff-tree diffs the two commits, a real pair a defeated gate would + # consult the process for. + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff-tree --numstat HEAD >actual && + test_grep "pair.c" actual && + # diff-index needs a change to diff, or there is no pair and a + # missing log proves nothing; stage one and diff it against HEAD. + echo "staged change" >>pair.c && + git add pair.c && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff-index --cached --numstat HEAD -- pair.c >actual && + test_grep "pair.c" actual && + test_path_is_missing backend.log +' + +test_expect_success 'add -p stages from the builtin diff with a process configured' ' + test_when_finished "rm -f backend.log" && + cat >gate.c <<-\EOF && + int gate(void) { return 1; } + EOF + git add gate.c && + git commit -m "add gate.c" && + cat >gate.c <<-\EOF && + int gate(void) { return 2; } + EOF + # add -p builds its hunks from patch text, which is not a provider + # consumer today, so a configured process cannot shape what it + # offers. This pins that interactive patch stays builtin for the + # current consumers, rather than exercising the plumbing gate. + test_write_lines y | + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + add -p gate.c && + git diff --cached -- gate.c >staged && + test_grep "return 2" staged && + test_path_is_missing backend.log && + git commit -m "gate.c v2" +' + +test_expect_success 'blame withholds identity for the working-tree pair' ' + test_when_finished "rm -f backend.log && git checkout -- pair.c" && + echo "uncommitted" >>pair.c && + wt=$(git hash-object pair.c) && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + blame pair.c >actual && + # The dirty working-tree side is not a stored blob: no request + # may name its bytes by object id. + test_grep ! "new-oid=$wt" backend.log +' + +test_expect_success 'a replaced blob makes the process step aside' ' + new_blob=$(git rev-parse HEAD:pair.c) && + test_when_finished "rm -f backend.log && git replace -d $new_blob" && + # Replacing the new-side blob redirects the content the diff reads + # under the id the process would be sent, so a raw-id request would + # name bytes other than the ones diffed. Identity is withheld: the + # process is not consulted and the builtin computes the pair from + # the replaced content. + repl=$(printf "just one line\n" | git hash-object -w --stdin) && + git replace "$new_blob" "$repl" && + git log -1 --format= --numstat -- pair.c >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --format= --numstat -- pair.c >actual && + test_cmp expect actual && + test_path_is_missing backend.log +' + +test_expect_success 'an equivalence answer omits the pair from the stat' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-equal --log=backend.log" \ + log -1 --format= --numstat -- pair.c >actual && + # An equivalent pair sums to a zero-count entry, and the stat + # code omits zero-count modified entries, the same way a + # whitespace-only pair prints nothing under -w. The builtin + # diff would print nonzero counts here, and the log proves the + # process was consulted. + test_must_be_empty actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'blame passes equivalent pairs through to the boundary' ' + ORIG=$(git rev-parse --short ":/add pair.c") && + # The process asserts every consulted pair equal, so no line is + # ever treated as changed: every line passes through to the + # commit that added the file, marked as the blame boundary. + git -c diff.cdiff.process="$BACKEND --mode=oid-equal" \ + blame pair.c >actual && + sed -n "5p" actual >line5 && + test_grep "^\^$ORIG" line5 && + sed -n "10p" actual >line10 && + test_grep "^\^$ORIG" line10 +' + +test_expect_success 'a warming run records a pair the process defers' ' + test_when_finished "git diff-hunks clear" && + git diff-hunks clear && + # The process owns the path but defers this pair with + # need-content, so the pair gets the builtin diff; that is the + # result the store holds, so the warming run records it and a + # later read is served from the store. + GIT_DIFF_HUNKS_WRITE=1 git -c core.diffHunks=true \ + -c diff.cdiff.process="$BACKEND --mode=oid-need-content" \ + log -1 --format= --stat -- pair.c >/dev/null && + git -c core.diffHunks=true blame --show-stats pair.c >stats 2>&1 && + test_grep "num precomputed hits: 1" stats +' + +# The protocol error paths: each adversarial response shape must warn, +# fall back to the builtin output, and either keep the process alive +# (a per-pair rejection) or disable it for the rest of the command (a +# protocol error). The request log tells the two apart: the log walk +# below consults two pairs (gate.c first, then pair.c), so a disabled +# process shows one logged request and a live one shows two. + +test_expect_success 'a malformed hunk line disables the process for the command' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-malformed --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "disabling it for the remainder" err && + test_line_count = 1 backend.log +' + +test_expect_success 'coordinates past the blob size skip the pair, process stays alive' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-huge --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "past the end" err && + test_line_count = 2 backend.log +' + +test_expect_success 'a count that overflows long skips the pair, process stays alive' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-erange --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "out-of-range coordinates" err && + test_line_count = 2 backend.log +' + +test_expect_success 'overlapping hunks are rejected per pair' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-overlap --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "overlapping hunks" err && + test_line_count = 2 backend.log +' + +test_expect_success 'misaligned hunks are rejected per pair' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-misaligned --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "misaligned" err && + test_line_count = 2 backend.log +' + +test_expect_success 'a start of zero with a nonzero count is rejected per pair' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + # A start of 0 names an empty side, so a nonzero count beside it + # names no line; the coordinate is rejected and the pair falls back + # to the builtin diff while the process stays alive. + git -c diff.cdiff.process="$BACKEND --mode=oid-badstart --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "out-of-range coordinates" err && + test_line_count = 2 backend.log +' + +test_expect_success 'an unrecognized status disables the process for the command' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-unknown-status --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "unrecognized status .frobnicate." err && + test_line_count = 1 backend.log +' + +test_expect_success 'status=abort withdraws the capability without a warning' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-abort --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep ! "disabling" err && + test_line_count = 1 backend.log +' + +test_expect_success 'a bare status without the hunk-section flush is a protocol error' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-bare-status --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "disabling it for the remainder" err && + test_line_count = 1 backend.log +' + +test_expect_success 'an empty packet in the hunk section is a protocol error' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-empty-packet --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "disabling it for the remainder" err && + test_line_count = 1 backend.log +' + +test_expect_success 'a process that dies mid-response fails the command over to builtin' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-crash --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "disabling it for the remainder" err && + test_line_count = 1 backend.log +' + +test_expect_success 'garbage bytes on stdout fail the command over to builtin' ' + test_when_finished "rm -f backend.log err" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=oid-garbage --log=backend.log" \ + log --format= --numstat -- "*.c" >actual 2>err && + test_cmp expect actual && + test_grep "disabling it for the remainder" err && + test_line_count = 1 backend.log +' + +test_expect_success 'a process announcing no capability is never asked' ' + test_when_finished "rm -f backend.log" && + git log --format= --numstat -- "*.c" >expect && + git -c diff.cdiff.process="$BACKEND --mode=cap-none --log=backend.log" \ + log --format= --numstat -- "*.c" >actual && + test_cmp expect actual && + test_must_be_empty backend.log +' + +test_expect_success 'a trailing token on a hunk line is ignored' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-trailing --log=backend.log" \ + log -1 --format= --numstat -- pair.c >actual && + printf "2\t2\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'a failed start warns once and the store may serve the path' ' + git init failrepo && + ( + cd failrepo && + echo "*.c diff=cdiff" >.gitattributes && + git add .gitattributes && + test_commit f1 f.c "one" && + test_commit f2 f.c "one +two" && + test_commit f3 f.c "one +two +three" && + GIT_DIFF_HUNKS_WRITE=1 git log --format= --stat -- f.c >/dev/null && + # The command has no shell metacharacters, so it fails at + # exec time; a shell-wrapped command would fail at the + # handshake, which the gentle handshake in the base + # (061a68e443) likewise degrades to the builtin diff. + git blame f.c >expect && + git -c diff.cdiff.process=/does-not-exist-diff-backend \ + blame f.c >actual 2>err && + test_cmp expect actual && + # One warning even though the blame consults two pairs. + test $(grep -c "failed to start" err) = 1 && + # A failure is a non-answer like any other: the request + # that observes it and every later request pass to the + # store, so the warmed store serves both pairs. + git -c diff.cdiff.process=/does-not-exist-diff-backend \ + blame --show-stats f.c >stats 2>&1 && + test_grep "num precomputed hits: 2" stats + ) +' + +test_expect_success 'the store serves a pair the process defers' ' + test_when_finished "git diff-hunks clear" && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git log --format= --stat -- pair.c >/dev/null && + git blame pair.c >expect && + # need-content defers the pair to the builtin diff, which is + # what the store holds, so the walk continues past the process + # and the store serves the pair. + git -c diff.cdiff.process="$BACKEND --mode=oid-need-content" \ + blame pair.c >actual && + test_cmp expect actual && + git -c diff.cdiff.process="$BACKEND --mode=oid-need-content" \ + blame --show-stats pair.c >stats 2>&1 && + test_grep "num precomputed hits: 1" stats +' + +test_expect_success 'git diff between commits consults the process' ' + test_when_finished "rm -f backend.log" && + ORIG=$(git rev-parse ":/add pair.c") && + CHANGE=$(git rev-parse ":/change pair.c") && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff --numstat $ORIG $CHANGE -- pair.c >actual && + printf "2\t2\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'git show consults the process' ' + test_when_finished "rm -f backend.log" && + CHANGE=$(git rev-parse ":/change pair.c") && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + show --format= --numstat $CHANGE -- pair.c >actual && + printf "2\t2\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'diff-tree --ext-diff consults the process' ' + test_when_finished "rm -f backend.log" && + CHANGE=$(git rev-parse ":/change pair.c") && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff-tree --ext-diff --no-commit-id --numstat $CHANGE >actual && + printf "2\t2\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success '--no-diff-process forbids consulting alone' ' + test_when_finished "rm -f backend.log" && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --no-diff-process --format= --numstat -- pair.c >actual && + printf "4\t4\tpair.c\n" >expect && + test_cmp expect actual && + test_path_is_missing backend.log +' + +test_expect_success '--diff-process allows plumbing to consult' ' + test_when_finished "rm -f backend.log" && + CHANGE=$(git rev-parse ":/change pair.c") && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + diff-tree --diff-process --no-commit-id --numstat $CHANGE >actual && + printf "2\t2\tpair.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=pair.c" backend.log +' + +test_expect_success 'a forced blame diff algorithm bypasses the process' ' + test_when_finished "rm -f backend.log" && + CHANGE=$(git rev-parse --short ":/change pair.c") && + # The process would attribute lines 9-10 to the original commit + # (see the oid-fixed blame test above); a forced builtin + # algorithm must produce the builtin attribution and never start + # the process. + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + blame --histogram pair.c >actual && + sed -n "9p" actual >line9 && + test_grep "$CHANGE" line9 && + test_path_is_missing backend.log +' + +test_expect_success 'textconv output is never identified to the process' ' + test_when_finished "rm -f backend.log" && + echo "*.tcv diff=tcv" >>.gitattributes && + git add .gitattributes && + git commit -m tcv-attr && + test_config diff.tcv.textconv cat && + test_commit tcv1 file.tcv "alpha" && + test_commit tcv2 file.tcv "alpha +beta" && + git blame file.tcv >expect && + git -c diff.tcv.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + blame file.tcv >actual && + test_cmp expect actual && + test_path_is_missing backend.log +' + +test_expect_success 'a gitlink side is never identified to the process' ' + test_when_finished "rm -f backend.log" && + echo "sub diff=cdiff" >>.gitattributes && + git add .gitattributes && + git commit -m sub-attr && + C1=$(git rev-parse HEAD) && + C2=$(git rev-parse HEAD~1) && + git update-index --add --cacheinfo 160000,$C1,sub && + git commit -m sub-1 && + git update-index --add --cacheinfo 160000,$C2,sub && + git commit -m sub-2 && + git -c diff.cdiff.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --format= --numstat -- sub >actual && + printf "1\t1\tsub\n" >expect && + test_cmp expect actual && + test_path_is_missing backend.log +' + +test_expect_success 'a relative diff consults by the repo-relative path' ' + test_when_finished "rm -f backend.log" && + echo "reldir/*.rel diff=rdrv" >>.gitattributes && + git add .gitattributes && + git commit -m rel-attr && + mkdir reldir && + test_write_lines line1 line2 line3 line4 original5 original6 \ + line7 line8 line9 line10 >reldir/x.rel && + git add reldir/x.rel && + git commit -m "add x.rel" && + test_write_lines line1 line2 line3 line4 changed5 changed6 \ + line7 line8 changed9 changed10 >reldir/x.rel && + git add reldir/x.rel && + git commit -m "change x.rel" && + # diff.relative strips the prefix from the displayed name; the + # driver lookup and the request pathname must still use the + # repo-relative path, or the directory-scoped attribute above + # would not match and the process would never be consulted. + # The process runs at the repository root, so its log lands there. + ( + cd reldir && + git -c diff.relative=true \ + -c diff.rdrv.process="$BACKEND --mode=oid-fixed --log=backend.log" \ + log -1 --format= --numstat -- x.rel + ) >actual && + printf "2\t2\tx.rel\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=reldir/x.rel" backend.log +' + +test_expect_success 'an empty file side is answered with a start of zero' ' + test_when_finished "rm -f backend.log" && + >empty.c && + git add empty.c && + git commit -m "add empty.c" && + printf "x\ny\nz\n" >empty.c && + git add empty.c && + git commit -m "fill empty.c" && + # The process addresses the empty old side with a start of 0 and a + # count of 0, and claims two of the three added lines. The answer is + # used as sent, so the stat shows the two lines it named, not the + # three the builtin would. + git -c diff.cdiff.process="$BACKEND --mode=oid-empty --log=backend.log" \ + log -1 --format= --numstat -- empty.c >actual && + printf "2\t0\tempty.c\n" >expect && + test_cmp expect actual && + test_grep "command=hunks-by-oid pathname=empty.c" backend.log +' + +test_done diff --git a/t/t4100-apply-stat.sh b/t/t4100-apply-stat.sh index 8393076469e083..d3406edaddbe11 100755 --- a/t/t4100-apply-stat.sh +++ b/t/t4100-apply-stat.sh @@ -113,6 +113,31 @@ test_expect_success 'applying a patch with a missing filename reports the input' test_cmp expect err ' +test_expect_success 'empty default filename reports the input' ' + cat >empty-name.patch <<-\EOF && + diff --git "a/""b/" + + --- /dev/null + +++ " + @@ -0,0 +1 @@ + + + EOF + test_must_fail git apply empty-name.patch 2>err && + test_grep "git diff header lacks filename information" err +' + +test_expect_success 'abandoned git header does not reuse names' ' + cat >abandoned-git-header.patch <<-\EOF && + diff --git a/foo b/foo + + --- /dev/null + +++ b/foo + @@ -0,0 +1 @@ + +x + EOF + git apply --check abandoned-git-header.patch +' + test_expect_success 'applying a patch with an invalid mode reports the input' ' cat >mode.patch <<-\EOF && diff --git a/f b/f diff --git a/t/t4211-line-log.sh b/t/t4211-line-log.sh index d0a834ed8f5bba..429ecd3a51c19b 100755 --- a/t/t4211-line-log.sh +++ b/t/t4211-line-log.sh @@ -176,24 +176,15 @@ test_expect_success '--name-status shows status and path' ' test_grep ! "^@@" actual ' -test_expect_success '--stat is not yet supported with -L' ' - test_must_fail git log -L1,24:b.c --stat 2>err && - test_grep "does not yet support" err -' - -test_expect_success '--numstat is not yet supported with -L' ' - test_must_fail git log -L1,24:b.c --numstat 2>err && - test_grep "does not yet support" err -' - -test_expect_success '--shortstat is not yet supported with -L' ' - test_must_fail git log -L1,24:b.c --shortstat 2>err && - test_grep "does not yet support" err -' - -test_expect_success '--dirstat is not yet supported with -L' ' +test_expect_success '--dirstat is not supported with -L' ' + # --dirstat is not supported with -L: its default mode measures + # whole-file change, not the tracked lines, and the + # --dirstat=lines variant is deferred too, so both forms are + # rejected like any other unsupported format. test_must_fail git log -L1,24:b.c --dirstat 2>err && - test_grep "does not yet support" err + test_grep "does not support" err && + test_must_fail git log -L1,24:b.c --dirstat=lines 2>err && + test_grep "does not support" err ' test_expect_success 'setup for checking fancy rename following' ' @@ -731,13 +722,138 @@ test_expect_success '-L with -S filters to string-count changes' ' test_expect_success '-L with -G filters to diff-text matches' ' git checkout parent-oids && git log -L:func2:file.c -G "F2 [+] 2" --format= >actual && - # -G greps the whole-file diff text, not just the tracked range; - # combined with -L, this selects commits that both touch func2 - # and have "F2 + 2" in their diff. + # -G greps the diff text, and under -L only the lines in the + # tracked range (unlike -S above, which searches the whole file); + # this selects commits whose change to func2 contains "F2 + 2". test $(grep -c "^diff --git" actual) = 1 && test_grep "F2 + 2" actual ' +test_expect_success 'setup for trailing deletion test' ' + git checkout --orphan trailing-del && + git reset --hard && + cat >file.c <<-\EOF && + void tracked() + { + return 1; + } + // trailing comment + EOF + git add file.c && + test_tick && + git commit -m "add file with trailing comment" && + # Modify tracked() AND delete the trailing comment in + # one commit, so the commit touches the tracked range + # and is not filtered out by the revision walker. + cat >file.c <<-\EOF && + void tracked() + { + return 2; + } + EOF + git commit -a -m "modify tracked and delete trailing comment" +' + +test_expect_success '-L does not include deletions past end of tracked range' ' + git log -L:tracked:file.c --format= -1 -p >actual && + # The trailing comment deletion is outside the tracked + # range and should not appear in the patch output. + test_grep "return 2" actual && + test_grep ! "trailing comment" actual +' + +test_expect_success '-L includes leading deletions resolved by in-range line' ' + git checkout --orphan leading-del && + git reset --hard && + cat >file.c <<-\EOF && + // leading comment + void tracked() + { + return 1; + } + EOF + git add file.c && + test_tick && + git commit -m "add file with leading comment" && + cat >file.c <<-\EOF && + void tracked() + { + return 2; + } + EOF + git commit -a -m "modify tracked and delete leading comment" && + git log -L:tracked:file.c --format= -1 -p >actual && + # The leading comment deletion is resolved by the next + # non-removal line (void tracked), which is in range: a + # removal is classified by the position of the following + # line, so it joins the range that line falls in. + test_grep "return 2" actual && + test_grep "leading comment" actual +' + +test_expect_success 'setup for line-range filter edge cases' ' + git checkout --orphan filter-edge && + git reset --hard && + cat >file.c <<-\EOF && + void before() + { + return 0; + } + + void tracked() + { + int a = 1; + int b = 2; + int c = 3; + return a + b + c; + } + + void after() + { + return 9; + } + EOF + git add file.c && + test_tick && + git commit -m "initial" +' + +test_expect_success '-L change at exact first line of range' ' + git checkout filter-edge && + # Change the function signature (first line of range) + sed "s/void tracked/int tracked/" file.c >tmp && + mv tmp file.c && + git commit -a -m "change first line" && + git log -L:tracked:file.c -p --format=%s -1 >actual && + test_grep "change first line" actual && + test_grep "+int tracked" actual && + test_grep "\\-void tracked" actual +' + +test_expect_success '-L change at exact last line of range' ' + git checkout filter-edge && + git reset --hard HEAD~1 && + # Change the closing brace line (last line of range) + sed "s/^}$/} \/\/ end tracked/" file.c >tmp && + mv tmp file.c && + git commit -a -m "change last line" && + git log -L:tracked:file.c -p --format=%s -1 >actual && + test_grep "change last line" actual && + test_grep "end tracked" actual +' + +test_expect_success '-L pure deletion in range (no additions)' ' + git checkout filter-edge && + git reset --hard HEAD~1 && + # Delete a line inside tracked() without adding anything + sed "/int c/d" file.c >tmp && + mv tmp file.c && + git commit -a -m "pure deletion" && + git log -L:tracked:file.c -p --format=%s -1 >actual && + test_grep "pure deletion" actual && + test_grep "\\-.*int c" actual +' + test_expect_success '-L with --diff-filter=M excludes root commit' ' git checkout parent-oids && git log -L:func2:file.c --diff-filter=M --format=%s --no-patch >actual && @@ -762,9 +878,9 @@ test_expect_success '-L with -S suppresses non-matching commits' ' test_cmp expect actual ' -test_expect_success '--full-diff is not yet supported with -L' ' +test_expect_success '--full-diff is not supported with -L' ' test_must_fail git log -L1,24:b.c --full-diff 2>err && - test_grep "does not yet support" err + test_grep "does not support" err ' test_expect_success '-L --oneline has no extra blank line before diff' ' @@ -775,12 +891,291 @@ test_expect_success '-L --oneline has no extra blank line before diff' ' test_grep "^diff --git" line2 ' +test_expect_success 'setup for stat range-scoping tests' ' + git checkout --orphan stat-scoping && + git reset --hard && + cat >file.c <<-\EOF && + int func1() + { + return F1; + } + + int func2() + { + return F2; + } + EOF + git add file.c && + test_tick && + git commit -m "Add func1() and func2()" && + + # Modify both functions in a single commit so that + # whole-file stats differ from the counts for the tracked range. + sed -e "s/F1/F1 + 1/" -e "s/F2/F2 + 2/" file.c >tmp && + mv tmp file.c && + git commit -a -m "Modify both functions" +' + +test_expect_success '--numstat counts only lines in tracked range' ' + # "Modify both functions" changes one line in func1 and one in + # func2. Whole-file numstat would show 2 added, 2 deleted. + # numstat for func2 within the tracked range should show only 1 and 1. + git log -L:func2:file.c --numstat --format=%s -1 >actual && + test_grep "Modify both functions" actual && + test_grep "^1 1 file.c$" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--numstat counts only additions for root commit' ' + # Root commit creates both func1 (4 lines) and func2 (4 lines). + # Whole-file numstat would show 9 lines added. numstat for func2 + # within the tracked range should show only 4. + git log -L:func2:file.c --numstat --format=%s >actual && + test_grep "Add func1() and func2()" actual && + test_grep "^4 0 file.c$" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--stat counts only lines in tracked range' ' + git log -L:func2:file.c --stat --format=%s -1 >actual && + test_grep "Modify both functions" actual && + test_grep "file.c |" actual && + test_grep "1 insertion" actual && + test_grep "1 deletion" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--shortstat counts only lines in tracked range' ' + # --shortstat prints only the summary line: no per-file "file.c |" + # line. Counts cover only the tracked range, as for --numstat above. + git log -L:func2:file.c --shortstat --format=%s -1 >actual && + test_grep "Modify both functions" actual && + test_grep "1 insertion" actual && + test_grep "1 deletion" actual && + test_grep ! "file.c |" actual && + test_grep ! "^diff --git" actual +' + +test_expect_success '--numstat across renames and multiple commits' ' + # parallel-change carries the tracked function f across an a.c -> b.c + # rename and a merge of two parallel histories. With -M, --numstat + # follows the rename and reports added/removed counts for f within + # the tracked range (not whole-file) per commit; the file column flips from + # b.c to a.c at the rename as the walk goes back in time. Commits + # that do not change the range of f emit no row (the merge and the + # pure file-move produce nothing), so there are fewer rows than + # commits. + git checkout parallel-change && + git log -M -L ":f:b.c" --format= --numstat >actual && + cat >expect <<-\EOF && + 1 1 b.c + 1 1 a.c + 1 1 a.c + 1 1 a.c + 1 0 a.c + 13 0 a.c + EOF + test_cmp expect actual +' + +test_expect_success '-L multiple ranges with --numstat excludes untracked change' ' + git checkout --orphan multi-range && + git reset --hard && + cat >m.c <<-\EOF && + int func1() + { + return F1; + } + + int func2() + { + return F2; + } + + int func3() + { + return F3; + } + EOF + git add m.c && + test_tick && + git commit -m "add m.c" && + # Change all three functions but track only func1 and func2. + # Whole-file numstat would be 3 3; a 2 2 result proves the + # untracked func3 change is excluded and the two ranges just sum. + sed -e "s/F1/F1 + 1/" -e "s/F2/F2 + 2/" -e "s/F3/F3 + 3/" m.c >tmp && + mv tmp m.c && + git commit -a -m "Modify all three functions" && + git log -L:func1:m.c -L:func2:m.c --numstat --format=%s -1 >actual && + test_grep "Modify all three functions" actual && + test_grep "^2 2 m.c$" actual && + test_grep ! "^3 3 m.c$" actual +' + test_expect_success '--summary shows new file on root commit' ' git checkout parent-oids && git log -L:func2:file.c --summary --format= >actual && test_grep "create mode 100644 file.c" actual ' +test_expect_success 'setup for --check test' ' + git checkout --orphan check-test && + git reset --hard && + cat >check.c <<-\EOF && + void tracked() + { + return; + } + + void other() + { + return; + } + EOF + git add check.c && + test_tick && + git commit -m "add check.c" && + # Introduce trailing whitespace errors in both functions + sed "s/return;/return; /" check.c >check.c.tmp && + mv check.c.tmp check.c && + git commit -a -m "introduce trailing whitespace" +' + +test_expect_success '--check scoped to tracked range with correct file line' ' + # tracked() trailing whitespace is at check.c:3; report it with the + # real file line number, not a count from the start of the range + # hunk. other() at check.c:8 is outside the range and is excluded. + test_must_fail git log -L:tracked:check.c --check --format= >actual && + test_grep "check.c:3: trailing whitespace" actual && + test_grep ! "check.c:8:" actual +' + +test_expect_success '--check reports each of several tracked ranges' ' + # Track both functions as separate ranges. Each range is flushed + # as its own hunk, so the second error must report its real file + # line (check.c:8), not continue the numbering from the first + # range (check.c:3). + test_must_fail git log -L:tracked:check.c -L:other:check.c \ + --check --format= >actual && + test_grep "check.c:3: trailing whitespace" actual && + test_grep "check.c:8: trailing whitespace" actual +' + +test_expect_success '--check line numbers stay correct across a gap in one range' ' + git checkout --orphan check-gap && + git reset --hard && + cat >gap.c <<-\EOF && + void tracked() + { + int a = 1; + int b = 2; + int c = 3; + int d = 4; + int e = 5; + int g = 7; + return; + } + EOF + git add gap.c && + test_tick && + git commit -m "add gap.c" && + # Two trailing-whitespace errors within one tracked range, + # separated by clean lines. ctxlen is inflated to the range span, + # so they land in a single xdiff hunk with the gap as context; + # both must report their real file line number, with the context + # lines between them counted. + sed -e "s/int a = 1;/int a = 1; /" -e "s/int g = 7;/int g = 7; /" gap.c >tmp && + mv tmp gap.c && + git commit -a -m "ws errors with a gap" && + test_must_fail git log -L:tracked:gap.c --check --format= >actual && + test_grep "gap.c:3: trailing whitespace" actual && + test_grep "gap.c:8: trailing whitespace" actual +' + +test_expect_success '--check does not report blank-at-eof outside the range' ' + git checkout --orphan check-eof && + git reset --hard && + printf "void tracked()\n{\n return;\n}\n\nint tail = 1;\n" >eof.c && + git add eof.c && + test_tick && + git commit -m "add eof.c" && + # One commit introduces a trailing-whitespace error inside tracked() + # (line 3) and a blank line at end of file (line 7, outside the + # range). The blank-at-eof check scans the whole file, so it must be + # scoped: report the in-range error, not the out-of-range EOF blank. + printf "void tracked()\n{\n return; \n}\n\nint tail = 1;\n\n" >eof.c && + git commit -a -m "ws in range, blank at eof out of range" && + test_must_fail git log -L:tracked:eof.c --check --format= >actual && + test_grep "eof.c:3: trailing whitespace" actual && + test_grep ! "blank line at EOF" actual +' + +test_expect_success '-L -G is scoped to the tracked range' ' + git checkout --orphan grep-scope && + git reset --hard && + cat >gp.c <<-\EOF && + int func1() + { + return ALPHA; + } + + int func2() + { + return BETA; + } + EOF + git add gp.c && + test_tick && + git commit -m "add gp.c" && + sed -e "s/ALPHA/ALPHA2/" -e "s/BETA/BETA2/" gp.c >tmp && + mv tmp gp.c && + git commit -a -m "touch both functions" && + # The commit changes ALPHA (func1) and BETA (func2). Tracking func2, + # -G BETA matches its in-range change; -G ALPHA must not, since ALPHA + # changes only outside the tracked range. + git log -L:func2:gp.c -G BETA --format=%s >actual && + test_grep "touch both functions" actual && + git log -L:func2:gp.c -G ALPHA --format=%s >actual && + test_grep ! "touch both functions" actual +' + +test_expect_success '-L -G searches the whole file under textconv' ' + git checkout --orphan grep-textconv && + git reset --hard && + cat >tc.c <<-\EOF && + int func1() + { + return F1; + } + + int func2() + { + return F2; + } + EOF + git add tc.c && + test_tick && + git commit -m "add tc.c" && + # One commit changes func1 and func2; MAGIC lands only in the + # func2 change, outside func1. + sed -e "s/F1/F1 + 1/" -e "s/return F2/return MAGIC/" tc.c >tmp && + mv tmp tc.c && + git commit -a -m "change both funcs" && + echo "tc.c diff=tc" >.gitattributes && + + # Without a textconv driver, -G is scoped to func1, so MAGIC (only + # in the func2 change) does not select the commit. + git log -L:func1:tc.c -G MAGIC --format=%s --no-patch >actual && + test_must_be_empty actual && + + # A textconv driver makes the range (original-file line numbers) + # meaningless against the driver output, so -G falls back to the + # whole file and MAGIC now selects the commit. + git config diff.tc.textconv cat && + git log -L:func1:tc.c -G MAGIC --format=%s --no-patch >actual && + test_grep "change both funcs" actual +' + test_expect_success 'get_commit_action() does not mutate a not-yet-walked commit' ' git init peek && ( diff --git a/t/t4211/sha1/expect.no-assertion-error b/t/t4211/sha1/expect.no-assertion-error index 54c568f273a6d2..95faf51a7b46d9 100644 --- a/t/t4211/sha1/expect.no-assertion-error +++ b/t/t4211/sha1/expect.no-assertion-error @@ -8,7 +8,7 @@ diff --git a/b.c b/b.c index bf79c2f..27c829c 100644 --- a/b.c +++ b/b.c -@@ -25,0 +18,9 @@ +@@ -24,0 +18,9 @@ +long f(long x) +{ + int s = 0; diff --git a/t/t4211/sha1/expect.vanishes-early b/t/t4211/sha1/expect.vanishes-early index a413ad36598ddf..e4b1a201d5d254 100644 --- a/t/t4211/sha1/expect.vanishes-early +++ b/t/t4211/sha1/expect.vanishes-early @@ -8,7 +8,7 @@ diff --git a/a.c b/a.c index 0b9cae5..5de3ea4 100644 --- a/a.c +++ b/a.c -@@ -23,0 +24,1 @@ int main () +@@ -22,0 +24 @@ int main () +/* incomplete lines are bad! */ commit 100b61a6f2f720f812620a9d10afb3a960ccb73c @@ -21,7 +21,7 @@ diff --git a/a.c b/a.c index 5e709a1..0b9cae5 100644 --- a/a.c +++ b/a.c -@@ -22,1 +22,1 @@ int main () +@@ -22 +22 @@ int main () -} +} \ No newline at end of file @@ -37,5 +37,5 @@ new file mode 100644 index 0000000..444e415 --- /dev/null +++ b/a.c -@@ -0,0 +20,1 @@ +@@ -0,0 +20 @@ +} diff --git a/t/t4211/sha256/expect.no-assertion-error b/t/t4211/sha256/expect.no-assertion-error index c25f2ce19c05d9..815d27f7f17b7e 100644 --- a/t/t4211/sha256/expect.no-assertion-error +++ b/t/t4211/sha256/expect.no-assertion-error @@ -8,7 +8,7 @@ diff --git a/b.c b/b.c index 69cb69c..a0d566e 100644 --- a/b.c +++ b/b.c -@@ -25,0 +18,9 @@ +@@ -24,0 +18,9 @@ +long f(long x) +{ + int s = 0; diff --git a/t/t4211/sha256/expect.vanishes-early b/t/t4211/sha256/expect.vanishes-early index bc33b963dc8570..263fc9eaace442 100644 --- a/t/t4211/sha256/expect.vanishes-early +++ b/t/t4211/sha256/expect.vanishes-early @@ -8,7 +8,7 @@ diff --git a/a.c b/a.c index e4fa1d8..62c1fc2 100644 --- a/a.c +++ b/a.c -@@ -23,0 +24,1 @@ int main () +@@ -22,0 +24 @@ int main () +/* incomplete lines are bad! */ commit 29f32ac3141c48b22803e5c4127b719917b67d0f8ca8c5248bebfa2a19f7da10 @@ -21,7 +21,7 @@ diff --git a/a.c b/a.c index d325124..e4fa1d8 100644 --- a/a.c +++ b/a.c -@@ -22,1 +22,1 @@ int main () +@@ -22 +22 @@ int main () -} +} \ No newline at end of file @@ -37,5 +37,5 @@ new file mode 100644 index 0000000..9f550c3 --- /dev/null +++ b/a.c -@@ -0,0 +20,1 @@ +@@ -0,0 +20 @@ +} diff --git a/t/t4220-diff-hunks.sh b/t/t4220-diff-hunks.sh new file mode 100755 index 00000000000000..086c53f651438f --- /dev/null +++ b/t/t4220-diff-hunks.sh @@ -0,0 +1,819 @@ +#!/bin/sh + +test_description='precomputed diff hunks store (git diff-hunks) + +The store maps an (old blob, new blob, diff settings) key to the hunks of +diffing the pair. It is a cache: reading is on by default +(core.diffHunks), while writing is +off by default and enabled per run by GIT_DIFF_HUNKS_WRITE (or the +diffHunks.write config), so a diff or log warms the store only when the +owner opts in. These tests check that a warmed store never changes +output, that lookups honor the diff settings, and that a corrupt store is +read as absent while verify reports the corruption.' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +STORE=.git/objects/info/diff-hunks + +# Warm the store the way a repository owner would: a stat walk with +# writing enabled. A --stat walk records one entry per trim-stable blob +# pair, serving blame and the summary formats alike. Extra arguments +# (e.g. -c options) are passed to git before "log". +warm () { + GIT_DIFF_HUNKS_WRITE=1 git "$@" log --all --stat >/dev/null +} + +# Run a command with the store disabled, for ground truth. +no_store () { + git -c core.diffhunks=false "$@" +} + +test_expect_success 'setup' ' + test_commit initial file.txt "line 1" && + test_commit second file.txt "line 1 +line 2" && + test_commit third file.txt "line 1 +line 2 +line 3" && + test_commit fourth file.txt "changed line 1 +line 2 +line 3 +line 4" +' + +test_expect_success 'ordinary commands do not create the store' ' + git log --stat >/dev/null && + git blame file.txt >/dev/null && + git diff --stat second third >/dev/null && + test_path_is_missing $STORE +' + +test_expect_success 'writing is gated by env and config, env wins' ' + test_when_finished "git diff-hunks clear" && + # The diffHunks.write config enables writing. + git -c diffHunks.write=true log --all --stat >/dev/null && + test_path_is_file $STORE && + git diff-hunks clear && + # GIT_DIFF_HUNKS_WRITE overrides the config: 0 disables it. + GIT_DIFF_HUNKS_WRITE=0 git -c diffHunks.write=true log --all --stat >/dev/null && + test_path_is_missing $STORE && + # and enables it without any config. + GIT_DIFF_HUNKS_WRITE=1 git log --all --stat >/dev/null && + test_path_is_file $STORE +' + +test_expect_success 'a warm builds a store that verifies' ' + warm && + test_path_is_file $STORE && + git diff-hunks verify +' + +test_expect_success 'a second warming run refreshes the store in place' ' + warm && + test_commit fifth file.txt "brand new line" && + warm && + git diff-hunks verify && + no_store log --stat >expect && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'core.diffhunks=false disables lookups' ' + warm && + git -c core.diffhunks=false blame --show-stats file.txt >out 2>&1 && + test_grep "num precomputed hits: 0" out +' + +# Writing seeds from the current store and merges into it, so a later +# warming run keeps the entries an earlier one recorded rather than +# rebuilding. Warm one pair, then a different pair, and confirm the first +# is still served. +test_expect_success 'a later warming run preserves earlier entries' ' + git init incr && + ( + cd incr && + test_commit a1 f.txt "1" && + test_commit a2 f.txt "1 +2" && + test_commit a3 f.txt "1 +2 +3" && + GIT_DIFF_HUNKS_WRITE=1 git diff --stat a1 a2 >/dev/null && + git diff-hunks verify && + GIT_DIFF_HUNKS_WRITE=1 git diff --stat a2 a3 >/dev/null && + git diff-hunks verify && + + # Blaming as of a2 diffs the a1..a2 pair. If seeding had + # dropped it when the a2..a3 pair was warmed, this would + # report zero precomputed hits. + git blame --show-stats a2 -- f.txt >out 2>&1 && + test_grep "num precomputed hits: [1-9]" out && + + # The second warm ADDED the a2..a3 pair; blaming a3 diffs + # both a2..a3 and a1..a2, so a hit on each shows the store + # gained the new pair while keeping the earlier one. + git blame --show-stats a3 -- f.txt >out3 2>&1 && + test_grep "num precomputed hits: 2" out3 && + + no_store log --stat >expect && + git log --stat >actual && + test_cmp expect actual + ) +' + +test_expect_success 'log --stat matches with and without the store' ' + no_store log --stat >expect && + warm && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'log --numstat and --shortstat match' ' + no_store log --numstat >expect_num && + no_store log --shortstat >expect_short && + warm && + git log --numstat >actual_num && + git log --shortstat >actual_short && + test_cmp expect_num actual_num && + test_cmp expect_short actual_short +' + +# A built store must reproduce diffstat output at every context +# length. Only trim-stable pairs are recorded, so one entry serves +# every context; a trim-divergent pair is never recorded and always +# computed. Zero context is where trim_common_tail runs, which is +# what makes the two diffs differ. +test_expect_success 'diffstat matches at several context lengths' ' + no_store log --stat >expect_def && + no_store log -U0 --stat >expect_u0 && + no_store log -U7 --stat >expect_u7 && + warm && + git log --stat >got_def && + git log -U0 --stat >got_u0 && + git log -U7 --stat >got_u7 && + test_cmp expect_def got_def && + test_cmp expect_u0 got_u0 && + test_cmp expect_u7 got_u7 +' + +test_expect_success 'store built at a nonzero context stays correct at that context' ' + no_store -c diff.context=5 log --stat >expect && + warm -c diff.context=5 && + git -c diff.context=5 log --stat >actual && + test_cmp expect actual +' + +# This blob pair (a real git test file being modernized) has different +# valid diffs at different contexts: at zero context, where +# trim_common_tail runs, "diff -U0" reports 9/6, while "diff -U3" +# reports 10/7. Such a trim-divergent pair is exactly what the writer +# must never record, since no single entry could serve both readers. +# A compact synthetic pair cannot show this count split: on small +# inputs xdiff produces minimal diffs, minimal diffs of one pair all +# add and delete the same number of lines, and trimming the common +# tail preserves minimality, so the counts agree by construction (a +# search over thousands of synthetic pairs up to 8 lines found no +# split). The split needs the cost-capping heuristics that only larger +# inputs trigger, so the pair is shipped as a fixture under t4220/. +test_expect_success 'a trim-divergent file is correct at each context' ' + cp "$TEST_DIRECTORY/t4220/trim-divergent-old" div.sh && + git add div.sh && + git commit -m divergent-old && + cp "$TEST_DIRECTORY/t4220/trim-divergent-new" div.sh && + git add div.sh && + git commit -m divergent-new && + no_store log -1 --format= --stat -- div.sh >expect_def && + no_store log -1 --format= -U0 --stat -- div.sh >expect_u0 && + warm && + git log -1 --format= --stat -- div.sh >got_def && + git log -1 --format= -U0 --stat -- div.sh >got_u0 && + test_cmp expect_def got_def && + test_cmp expect_u0 got_u0 && + # The fixture must actually diverge, or the test would pass without + # exercising the split; fail loudly if a diff change ever levels it. + ! test_cmp expect_def expect_u0 +' + +# A warming run displays the diffstat it computes. At zero context xdi_diff +# trims, so the displayed counts must be the trimmed ones (what a store-less +# run shows), not the untrimmed ones the writer compares against when it +# decides whether the pair is stable enough to record. +test_expect_success 'warming --stat at zero context matches a store-less run' ' + git init -q warm-u0 && + ( + cd warm-u0 && + cp "$TEST_DIRECTORY/t4220/trim-divergent-old" div.sh && + git add div.sh && git commit -q -m old && + cp "$TEST_DIRECTORY/t4220/trim-divergent-new" div.sh && + git add div.sh && git commit -q -m new && + git -c core.diffhunks=false log -1 --format= -U0 --stat -- div.sh >expect && + GIT_DIFF_HUNKS_WRITE=1 git log -1 --format= -U0 --stat -- div.sh >got && + test_cmp expect got + ) +' + +test_expect_success 'diff --stat matches with and without the store, both directions' ' + no_store diff --stat second fourth >expect_fwd && + no_store diff --stat fourth second >expect_rev && + warm && + git diff --stat second fourth >got_fwd && + git diff --stat fourth second >got_rev && + test_cmp expect_fwd got_fwd && + test_cmp expect_rev got_rev +' + +test_expect_success 'show and diff-tree --stat use the store' ' + test_when_finished "git diff-hunks clear" && + # diff_hunks_attach() runs for show and diff-tree: a write-enabled + # --stat records into the store (without the attach there is no + # writer, so nothing is written). + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git show --stat fourth >/dev/null && + test_path_is_file "$STORE" && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git diff-tree --stat fourth >/dev/null && + test_path_is_file "$STORE" && + # Reading never changes their output. + git diff-hunks clear && + no_store show --stat fourth >expect_show && + no_store diff-tree --stat fourth >expect_dt && + warm && + git show --stat fourth >got_show && + git diff-tree --stat fourth >got_dt && + test_cmp expect_show got_show && + test_cmp expect_dt got_dt +' + +test_expect_success 'log -R --stat matches (reversed pairs keyed apart)' ' + no_store log -R --stat >expect && + warm && + git log -R --stat >actual && + test_cmp expect actual +' + +# One warm serves both diffstat and blame: the blob pairs a blame +# walks are the same parent-child pairs the diffstat warm recorded. +test_expect_success 'a single warming run serves both blame and diffstat' ' + warm && + git blame --show-stats file.txt >out 2>&1 && + test_grep "num precomputed hits: [1-9][0-9]*" out +' + +# The diffstat read path produces identical output on a hit or a miss, so +# it emits a trace2 "read-hits" count to prove it consulted the store. +test_expect_success 'diffstat consults the store (trace shows read hits)' ' + warm && + GIT_TRACE2_EVENT="$PWD/trace_on.json" git log --stat >/dev/null && + test_grep read-hits trace_on.json && + test_env GIT_TRACE2_EVENT="$PWD/trace_off.json" no_store log --stat >/dev/null && + test_grep ! read-hits trace_off.json +' + +test_expect_success 'blame matches with and without the store' ' + no_store blame file.txt >expect && + warm && + git blame file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'blame --porcelain and --incremental match' ' + no_store blame --porcelain file.txt >expect_p && + no_store blame --incremental file.txt >expect_i && + warm && + git blame --porcelain file.txt >got_p && + git blame --incremental file.txt >got_i && + test_cmp expect_p got_p && + test_cmp expect_i got_i +' + +# Diff settings that change hunks but are not part of the store key must +# bypass it in both directions, so output stays byte-identical to a +# store-less run. +test_expect_success 'setup ignore fixture' ' + git init ignore-repo && + ( + cd ignore-repo && + test_write_lines code keep "# c" >f && + git add f && + git commit -m c1 && + test_write_lines codeCH keep "# cX" >f && + git add f && + git commit -m c2 && + warm + ) +' + +# Output parity alone cannot prove the guard: served counts can +# coincide with computed ones, so each bypass below also asserts the +# consultation itself (no read hit with the option, a hit without it) +# and that a warming run under the option records nothing. +test_expect_success '-I bypasses the store in both directions' ' + ( + cd ignore-repo && + no_store diff -I"^#" --numstat HEAD~ HEAD >expect && + git diff -I"^#" --numstat HEAD~ HEAD >actual && + test_cmp expect actual && + # -I does not change the key, so only the ignore_regex + # guard keeps the warmed entry from serving here. + GIT_TRACE2_EVENT="$PWD/trace_i.json" \ + git diff -I"^#" --numstat HEAD~ HEAD >/dev/null && + test_grep ! read-hits trace_i.json && + GIT_TRACE2_EVENT="$PWD/trace_i_ctl.json" \ + git diff --numstat HEAD~ HEAD >/dev/null && + test_grep read-hits trace_i_ctl.json && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git diff -I"^#" --numstat HEAD~ HEAD >/dev/null && + test_path_is_missing .git/objects/info/diff-hunks && + # Restore the warmed fixture for the tests below. + warm + ) +' + +test_expect_success '-B bypasses the store in both directions' ' + git init break-repo && + ( + cd break-repo && + test_write_lines a b c d e f g h >f && + git add f && + git commit -m orig && + test_write_lines 1 2 3 4 5 6 7 8 >f && + git add f && + git commit -m rewrite && + warm && + no_store diff -B --stat HEAD~ HEAD >expect && + git diff -B --stat HEAD~ HEAD >actual && + test_cmp expect actual && + GIT_TRACE2_EVENT="$PWD/trace_b.json" \ + git diff -B --stat HEAD~ HEAD >/dev/null && + test_grep ! read-hits trace_b.json && + GIT_TRACE2_EVENT="$PWD/trace_ctl.json" \ + git diff --stat HEAD~ HEAD >/dev/null && + test_grep read-hits trace_ctl.json && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git diff -B --stat HEAD~ HEAD >/dev/null && + test_path_is_missing .git/objects/info/diff-hunks + ) +' + +test_expect_success '--anchored bypasses the store in both directions' ' + ( + cd ignore-repo && + no_store diff --stat --anchored=keep HEAD~ HEAD >expect && + git diff --stat --anchored=keep HEAD~ HEAD >actual && + test_cmp expect actual && + # Anchors do not change the key, so only the anchors guard + # keeps the warmed entry from serving here. + GIT_TRACE2_EVENT="$PWD/trace_anchor.json" \ + git diff --stat --anchored=keep HEAD~ HEAD >/dev/null && + test_grep ! read-hits trace_anchor.json && + GIT_TRACE2_EVENT="$PWD/trace_plain.json" \ + git diff --stat HEAD~ HEAD >/dev/null && + test_grep read-hits trace_plain.json && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 \ + git diff --stat --anchored=keep HEAD~ HEAD >/dev/null && + test_path_is_missing .git/objects/info/diff-hunks + ) +' + +test_expect_success '--ignore-blank-lines bypasses the store in both directions' ' + git init ibl-repo && + ( + cd ibl-repo && + printf "a\n\nx\ny\nb\n" >f && + git add f && + git commit -m v1 && + printf "a\nx\ny\nB\n" >f && + git add f && + git commit -m v2 && + warm && + no_store diff --stat --ignore-blank-lines HEAD~ HEAD >expect && + git diff --stat --ignore-blank-lines HEAD~ HEAD >actual && + test_cmp expect actual && + # The flag is an xdl_opts bit and thus part of the key; the + # stat consumer excludes it before consulting at all. + GIT_TRACE2_EVENT="$PWD/trace_ibl.json" \ + git diff --stat --ignore-blank-lines HEAD~ HEAD >/dev/null && + test_grep ! read-hits trace_ibl.json && + GIT_TRACE2_EVENT="$PWD/trace_plain.json" \ + git diff --stat HEAD~ HEAD >/dev/null && + test_grep read-hits trace_plain.json && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 \ + git diff --stat --ignore-blank-lines HEAD~ HEAD >/dev/null && + test_path_is_missing .git/objects/info/diff-hunks + ) +' + +test_expect_success 'a whitespace-ignoring diff is not served default entries' ' + git init ws-repo && + ( + cd ws-repo && + test_write_lines alpha beta gamma >f && + git add f && + git commit -m c1 && + test_write_lines " alpha" beta gamma delta >f && + git add f && + git commit -m c2 && + warm && + no_store diff -w --numstat HEAD~ HEAD >expect && + git diff -w --numstat HEAD~ HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success 'blame -w stays correct and does not hit default entries' ' + ( + cd ws-repo && + no_store blame -w f >expect && + git blame -w --show-stats f >out 2>&1 && + test_grep "num precomputed hits: 0" out && + git blame -w f >actual && + test_cmp expect actual + ) +' + +test_expect_success 'blame with indentHeuristic off stays correct and misses' ' + warm && + git -c diff.indentHeuristic=false blame --show-stats file.txt >out 2>&1 && + test_grep "num precomputed hits: 0" out && + no_store -c diff.indentHeuristic=false blame file.txt >expect && + git -c diff.indentHeuristic=false blame file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'a driver algorithm override keeps output correct and keys apart' ' + git init driver-algo && + ( + cd driver-algo && + echo "file.foo diff=foo" >.gitattributes && + git add .gitattributes && + git commit -m attributes && + test_write_lines 1 2 3 4 5 >file.foo && + git add file.foo && + git commit -m one && + test_write_lines 1 2 X 4 5 6 >file.foo && + git add file.foo && + git commit -m two && + warm -c diff.foo.algorithm=histogram && + no_store -c diff.foo.algorithm=histogram log --stat >expect && + git -c diff.foo.algorithm=histogram log --stat >actual && + test_cmp expect actual && + # The driver algorithm is an xdl_opts key bit: entries + # warmed at the default settings must not serve a + # driver-forced histogram read, and output stays correct. + git diff-hunks clear && + warm && + no_store -c diff.foo.algorithm=histogram log --stat >expect2 && + git -c diff.foo.algorithm=histogram log --stat >actual2 && + test_cmp expect2 actual2 && + GIT_TRACE2_EVENT="$PWD/trace_algo.json" \ + git -c diff.foo.algorithm=histogram log --stat >/dev/null && + test_grep ! read-hits trace_algo.json && + GIT_TRACE2_EVENT="$PWD/trace_algo_ctl.json" \ + git log --stat >/dev/null && + test_grep read-hits trace_algo_ctl.json + ) +' + +test_expect_success 'blame --reverse never consults the store' ' + warm && + git blame --reverse HEAD~3..HEAD file.txt >actual 2>/dev/null && + no_store blame --reverse HEAD~3..HEAD file.txt >expect 2>/dev/null && + test_cmp expect actual && + # Reverse blame withholds the pair identity. Zero hits alone + # cannot prove that: reverse pairs are never warmed, so a + # consulted pair would miss, not hit. Zero misses is what shows + # the store was never consulted. + git blame --reverse --show-stats HEAD~3..HEAD file.txt \ + >stats 2>/dev/null && + test_grep "num precomputed hits: 0" stats && + test_grep "num precomputed misses: 0" stats +' + +test_expect_success 'blame with a textconv driver bypasses the store' ' + echo "tc.txt diff=tc" >>.gitattributes && + git add .gitattributes && + git commit -m tc-attr && + git config diff.tc.textconv "sed -e s/1/one/" && + test_commit tc1 tc.txt "line 1" && + test_commit tc2 tc.txt "line 1 +line 2" && + warm && + git blame --show-stats tc.txt >out 2>&1 && + test_grep "num precomputed hits: 0" out && + no_store blame tc.txt >expect && + git blame tc.txt >actual && + test_cmp expect actual +' + +test_expect_success 'a replaced blob makes the store step aside' ' + git init replace-repo && + ( + cd replace-repo && + test_commit r1 f.txt "a" && + test_commit r2 f.txt "a +b" && + warm && + # Control: without a replacement the pair is served. + GIT_TRACE2_EVENT="$PWD/trace_ctl.json" \ + git log -1 --format= --numstat -- f.txt >/dev/null && + test_grep read-hits trace_ctl.json && + # Replace r2 blob: the diff now reads different content + # (through OBJECT_INFO_LOOKUP_REPLACE) under the id the store + # keyed, so a served answer would be the pre-replacement diff. + # Identity is withheld, the store steps aside, and the builtin + # computes from the replaced content. + new_blob=$(git rev-parse HEAD:f.txt) && + repl=$(printf "a\nB\nC\nD\n" | git hash-object -w --stdin) && + git replace "$new_blob" "$repl" && + no_store log -1 --format= --numstat -- f.txt >expect && + git log -1 --format= --numstat -- f.txt >actual && + test_cmp expect actual && + GIT_TRACE2_EVENT="$PWD/trace_repl.json" \ + git log -1 --format= --numstat -- f.txt >/dev/null && + test_grep ! read-hits trace_repl.json + ) +' + +test_expect_success 'blame -M and -C stay correct with the store' ' + warm && + no_store blame -M file.txt >expect_m && + no_store blame -C file.txt >expect_c && + git blame -M file.txt >got_m && + git blame -C file.txt >got_c && + test_cmp expect_m got_m && + test_cmp expect_c got_c +' + +# Copy-detecting (and reverse) blame still diff blob pairs through +# pass_blame_to_parent, so they must use the real blame xdl_opts. A +# whitespace-only change is invisible under -w; if -w were dropped on +# these paths the -w and non-w results would coincide. +test_expect_success 'blame -C honors -w' ' + git init -q blame-cw && + ( + cd blame-cw && + printf "one\ntwo\nthree\n" >f && + git add f && git commit -q -m base && + printf "one\n two \nthree\n" >f && + git add f && git commit -q -m reindent && + git blame -C -w f >with_w && + git blame -C f >without_w && + ! test_cmp with_w without_w + ) +' + +# Cover the pair shapes an object walk encounters: binary and +# mode-only changes produce no text hunks to record. +test_expect_success 'binary and mode-only changes do not break the writer' ' + printf "\\000\\001\\002" >bin.dat && + git add bin.dat && + git commit -m binary-1 && + printf "\\000\\001\\003\\004" >bin.dat && + git add bin.dat && + git commit -m binary-2 && + echo "mode content" >mode.txt && + git add mode.txt && + git commit -m mode-1 && + test_chmod +x mode.txt && + git commit -m mode-2 && + no_store log --stat >expect && + warm && + git log --stat >actual && + test_cmp expect actual +' + +test_expect_success 'blame across a rename matches' ' + echo "original content" >rename-src.txt && + git add rename-src.txt && + git commit -m "add rename-src" && + echo "more" >>rename-src.txt && + git add rename-src.txt && + git commit -m "modify rename-src" && + git mv rename-src.txt rename-dst.txt && + git commit -m "rename" && + echo "post" >>rename-dst.txt && + git add rename-dst.txt && + git commit -m "modify after rename" && + no_store blame rename-dst.txt >expect && + warm && + git blame rename-dst.txt >actual && + test_cmp expect actual +' + +test_expect_success 'blame handles merge commits' ' + git checkout -b merge-side main~2 && + test_commit merge-change merge-file.txt "side content" && + git checkout main && + git merge --no-edit merge-side && + no_store blame merge-file.txt >expect && + warm && + git blame merge-file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'distinct --contents against one revision do not collide' ' + warm && + test_write_lines "line 1" "appended line" >c1 && + test_write_lines "rewritten line" >c2 && + # Ground truth without the store. + no_store blame -s --contents=c2 file.txt initial >expect && + # With the store, an intervening c1 run must not poison the c2 lookup. + git blame -s --contents=c1 file.txt initial >/dev/null && + git blame -s --contents=c2 file.txt initial >actual && + test_cmp expect actual && + # The --contents side is a working-tree pseudo-commit (a null commit + # id), so its pairs withhold identity and never consult the store. + # Output parity alone cannot show that: a consulted unwarmed pair + # would miss, not hit, so zero misses is what proves the pair was + # never looked up. + git blame -s --show-stats --contents=c2 file.txt initial >stats 2>&1 && + test_grep "num precomputed hits: 0" stats && + test_grep "num precomputed misses: 0" stats +' + +test_expect_success 'blame --ignore-rev bypasses the store for ignored pairs' ' + git init ignore-rev-repo && + ( + cd ignore-rev-repo && + test_commit ir1 f.txt "base" && + test_commit ir2 f.txt "base +more" && + warm && + # Control: the ordinary pass is served, nothing is computed. + git blame --show-stats f.txt >ctl 2>&1 && + test_grep "num precomputed hits: 1" ctl && + test_grep "num get patch: 0" ctl && + no_store blame --ignore-rev ir2 f.txt >expect && + git blame --ignore-rev ir2 f.txt >actual && + test_cmp expect actual && + # The ignored revision adds a pass that withholds identity: + # it computes its diff (get patch rises) instead of being + # served or even counted as a store consultation. + git blame --ignore-rev ir2 --show-stats f.txt >stats 2>&1 && + test_grep "num precomputed hits: 1" stats && + test_grep "num precomputed misses: 0" stats && + test_grep "num get patch: 1" stats + ) +' + +test_expect_success 'blame counts misses for pairs the store does not hold' ' + ( + cd ignore-rev-repo && + test_commit ir3 f.txt "base +more +third" && + git blame --show-stats f.txt >stats 2>&1 && + test_grep "num precomputed hits: 1" stats && + test_grep "num precomputed misses: 1" stats + ) +' + +test_expect_success 'log -L --stat neither reads nor records' ' + warm && + GIT_TRACE2_EVENT="$PWD/trace_linelog.json" \ + git log -L1,1:file.txt --stat >/dev/null && + test_grep ! read-hits trace_linelog.json && + git diff-hunks clear && + GIT_DIFF_HUNKS_WRITE=1 git log -L1,1:file.txt --stat >/dev/null && + test_path_is_missing $STORE +' + +# Integrity: a structurally broken header is read as absent (the reader +# falls back to xdiff and stays correct); a checksum mismatch is caught +# by verify, which is when integrity is checked. +test_expect_success 'a truncated store is read as absent' ' + warm && + test_copy_bytes 20 <$STORE >truncated && + mv truncated $STORE && + no_store blame file.txt >expect && + git blame file.txt >actual && + test_cmp expect actual +' + +test_expect_success 'a corrupt signature is read as absent' ' + warm && + printf "XXXX" >corrupt && + tail -c +5 <$STORE >>corrupt && + mv corrupt $STORE && + no_store blame file.txt >expect && + git blame file.txt >actual && + test_cmp expect actual +' + +# Byte 6 of the header is the chunk count; a value larger than the file +# can hold must be rejected before the chunk table is walked. +test_expect_success 'an over-claimed chunk count is read as absent' ' + warm && + printf "\377" | dd of=$STORE bs=1 seek=6 count=1 conv=notrunc 2>/dev/null && + no_store blame file.txt >expect && + git blame file.txt >actual && + test_cmp expect actual +' + +# A record with no hunks would replay as an equivalence claim, which +# the writer never records; the reader must treat such a record as a +# miss and recompute, and verify must flag it. +test_expect_success 'a zero-hunk record is read as a miss and fails verify' ' + git init zero-hunk && + ( + cd zero-hunk && + test_commit z1 f.txt "base" && + test_commit z2 f.txt "base +more" && + warm && + # The store holds one entry of one hunk: a 4-byte count and + # one 16-byte hunk record, just before the trailing + # checksum. Zero the count to craft the record the writer + # refuses to produce. + rawsz=$(test_oid rawsz) && + fsize=$(test_file_size $STORE) && + printf "\\0\\0\\0\\0" | dd of=$STORE bs=1 \ + seek=$((fsize - rawsz - 20)) count=4 conv=notrunc \ + 2>/dev/null && + no_store blame f.txt >expect && + git blame --show-stats f.txt >stats 2>&1 && + test_grep "num precomputed hits: 0" stats && + git blame f.txt >actual && + test_cmp expect actual && + test_must_fail git diff-hunks verify + ) +' + +test_expect_success 'verify succeeds on a valid store and on an absent one' ' + warm && + git diff-hunks verify && + git diff-hunks clear && + test_path_is_missing $STORE && + git diff-hunks verify +' + +test_expect_success 'verify detects a checksum mismatch' ' + test_when_finished "git diff-hunks clear" && + warm && + fsize=$(test_file_size $STORE) && + mid=$((fsize / 2)) && + printf "\\377" | dd of=$STORE bs=1 seek=$mid count=1 conv=notrunc 2>/dev/null && + test_must_fail git diff-hunks verify +' + +test_expect_success 'a warm discards a corrupt store rather than seeding from it' ' + test_when_finished "git diff-hunks clear" && + warm && + # Corrupt the checksum: the next warm must not carry the corrupt + # entries forward into a fresh checksum-valid file; it discards + # them (with a warning) and rewrites a store that verifies. + fsize=$(test_file_size $STORE) && + printf "\\377" | dd of=$STORE bs=1 seek=$((fsize / 2)) count=1 conv=notrunc 2>/dev/null && + warm 2>err && + test_grep "failed its checksum" err && + git diff-hunks verify && + no_store log --stat >expect && + git log --stat >actual && + test_cmp expect actual +' + +# A generated patch must carry the builtin diffstat, not one served from +# the sender's local store, so its counts do not depend on whether the +# sender warmed the store. Poison the store so a served answer diverges +# from the builtin, then confirm format-patch shows the builtin counts. +test_expect_success 'format-patch keeps its diffstat off the store' ' + git init fp-repo && + ( + cd fp-repo && + test_commit p1 f.txt "a" && + test_commit p2 f.txt "a +b" && + warm && + # Bump the new-side count of the single recorded hunk. The + # record stays structurally valid, and a read skips the + # trailing checksum, so the store serves this poisoned count. + rawsz=$(test_oid rawsz) && + fsize=$(test_file_size .git/objects/info/diff-hunks) && + printf "\\0\\0\\0\\7" | dd of=.git/objects/info/diff-hunks bs=1 \ + seek=$((fsize - rawsz - 4)) count=4 conv=notrunc 2>/dev/null && + # The store now serves a divergent count, proving the poison + # is live and observable through a store consumer. + printf "7\t0\tf.txt\n" >poisoned && + git log -1 --format= --numstat -- f.txt >served && + test_cmp poisoned served && + # format-patch does not consult the store, so its output is + # identical with the store poisoned and with it disabled. + no_store format-patch -1 --stdout --stat -- f.txt >expect && + git format-patch -1 --stdout --stat -- f.txt >actual && + test_cmp expect actual + ) +' + +test_expect_success 'diff-hunks clear removes the store file' ' + warm && + test_path_is_file $STORE && + git diff-hunks clear && + test_path_is_missing $STORE +' + +test_done diff --git a/t/t4220/README b/t/t4220/README new file mode 100644 index 00000000000000..b850fe7c61bea0 --- /dev/null +++ b/t/t4220/README @@ -0,0 +1,55 @@ +t4220 diff-hunks test fixtures +============================== + +trim-divergent-old, trim-divergent-new +-------------------------------------- + +Two revisions of a single real file, used by t4220-diff-hunks.sh to +exercise a "trim-divergent" blob pair: one whose diff hunk counts change +with the amount of context, so the trimmed and untrimmed results +disagree. + +They are two versions of git's own t/t6002-rev-list-bisect.sh, taken from +git.git history around: + + 090af9957c ("t6002: fix use of `expr` with `set -e`", + Patrick Steinhardt, 2026-04-21) + +which rewrites `$(expr ...)` arithmetic as `$((...))` and reformats a few +test_expect_success blocks. + + trim-divergent-old = 090af9957c^:t/t6002-rev-list-bisect.sh (blob daa009c9a1) + trim-divergent-new = 090af9957c :t/t6002-rev-list-bisect.sh (blob f2de40b5ed) + +To regenerate them from any git.git checkout: + + git show 090af9957c^:t/t6002-rev-list-bisect.sh >trim-divergent-old + git show 090af9957c:t/t6002-rev-list-bisect.sh >trim-divergent-new + +Why this pair +------------- + +The diff-hunks store records only "trim-stable" pairs: those whose hunks +are identical whether or not xdiff trims the common head and tail (which +it does at zero context, in trim_common_tail). This pair is deliberately +NOT trim-stable: + + diff -U0 reports 9 added / 6 deleted + diff -U3 reports 10 added / 7 deleted + +Because the counts diverge with context, the writer must refuse to record +this pair and every command must recompute it from the blobs. t4220 uses +it to prove that the displayed counts stay correct at each context, and +that a divergent pair is never served from the store. See +t4220-diff-hunks.sh ("a trim-divergent file is correct at each context" +and the store-poison test). + +Why not a synthesized fixture +----------------------------- + +The divergence needs real content that makes xdiff's common-tail trimming +shift a hunk boundary while the added/deleted balance stays equal. A +minimal hand-written file that reliably triggers the -U0 vs -U3 count +disagreement has not been found yet; until one is, this real pair is kept +verbatim. If you synthesize a smaller equivalent, replace these two files +and delete this note. diff --git a/t/t4220/trim-divergent-new b/t/t4220/trim-divergent-new new file mode 100644 index 00000000000000..f2de40b5ed8f14 --- /dev/null +++ b/t/t4220/trim-divergent-new @@ -0,0 +1,319 @@ +#!/bin/sh +# +# Copyright (c) 2005 Jon Seymour +# +test_description='Tests git rev-list --bisect functionality' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-t6000.sh # t6xxx specific functions + +# usage: test_bisection max-diff bisect-option head ^prune... +# +# e.g. test_bisection 1 --bisect l1 ^l0 +# +test_bisection_diff() +{ + _max_diff=$1 + _bisect_option=$2 + shift 2 + _bisection=$(git rev-list $_bisect_option "$@") + _list_size=$(git rev-list "$@" | wc -l) + _head=$1 + shift 1 + _bisection_size=$(git rev-list $_bisection "$@" | wc -l) + [ -n "$_list_size" -a -n "$_bisection_size" ] || + error "test_bisection_diff failed" + + # Test if bisection size is close to half of list size within + # tolerance. + # + _bisect_err=$(($_list_size - $_bisection_size * 2)) + if test "$_bisect_err" -lt 0 + then + _bisect_err=$((0 - $_bisect_err)) + fi + _bisect_err=$(($_bisect_err / 2)) ; # floor + + test_expect_success "bisection diff $_bisect_option $_head $* <= $_max_diff" ' + test $_bisect_err -le $_max_diff + ' +} + +date >path0 +git update-index --add path0 +save_tag tree git write-tree +on_committer_date "00:00" hide_error save_tag root unique_commit root tree +on_committer_date "00:01" save_tag l0 unique_commit l0 tree -p root +on_committer_date "00:02" save_tag l1 unique_commit l1 tree -p l0 +on_committer_date "00:03" save_tag l2 unique_commit l2 tree -p l1 +on_committer_date "00:04" save_tag a0 unique_commit a0 tree -p l2 +on_committer_date "00:05" save_tag a1 unique_commit a1 tree -p a0 +on_committer_date "00:06" save_tag b1 unique_commit b1 tree -p a0 +on_committer_date "00:07" save_tag c1 unique_commit c1 tree -p b1 +on_committer_date "00:08" save_tag b2 unique_commit b2 tree -p b1 +on_committer_date "00:09" save_tag b3 unique_commit b2 tree -p b2 +on_committer_date "00:10" save_tag c2 unique_commit c2 tree -p c1 -p b2 +on_committer_date "00:11" save_tag c3 unique_commit c3 tree -p c2 +on_committer_date "00:12" save_tag a2 unique_commit a2 tree -p a1 +on_committer_date "00:13" save_tag a3 unique_commit a3 tree -p a2 +on_committer_date "00:14" save_tag b4 unique_commit b4 tree -p b3 -p a3 +on_committer_date "00:15" save_tag a4 unique_commit a4 tree -p a3 -p b4 -p c3 +on_committer_date "00:16" save_tag l3 unique_commit l3 tree -p a4 +on_committer_date "00:17" save_tag l4 unique_commit l4 tree -p l3 +on_committer_date "00:18" save_tag l5 unique_commit l5 tree -p l4 +git update-ref HEAD $(tag l5) + + +# E +# / \ +# e1 | +# | | +# e2 | +# | | +# e3 | +# | | +# e4 | +# | | +# | f1 +# | | +# | f2 +# | | +# | f3 +# | | +# | f4 +# | | +# e5 | +# | | +# e6 | +# | | +# e7 | +# | | +# e8 | +# \ / +# F + + +on_committer_date "00:00" hide_error save_tag F unique_commit F tree +on_committer_date "00:01" save_tag e8 unique_commit e8 tree -p F +on_committer_date "00:02" save_tag e7 unique_commit e7 tree -p e8 +on_committer_date "00:03" save_tag e6 unique_commit e6 tree -p e7 +on_committer_date "00:04" save_tag e5 unique_commit e5 tree -p e6 +on_committer_date "00:05" save_tag f4 unique_commit f4 tree -p F +on_committer_date "00:06" save_tag f3 unique_commit f3 tree -p f4 +on_committer_date "00:07" save_tag f2 unique_commit f2 tree -p f3 +on_committer_date "00:08" save_tag f1 unique_commit f1 tree -p f2 +on_committer_date "00:09" save_tag e4 unique_commit e4 tree -p e5 +on_committer_date "00:10" save_tag e3 unique_commit e3 tree -p e4 +on_committer_date "00:11" save_tag e2 unique_commit e2 tree -p e3 +on_committer_date "00:12" save_tag e1 unique_commit e1 tree -p e2 +on_committer_date "00:13" save_tag E unique_commit E tree -p e1 -p f1 + +on_committer_date "00:00" hide_error save_tag U unique_commit U tree +on_committer_date "00:01" save_tag u0 unique_commit u0 tree -p U +on_committer_date "00:01" save_tag u1 unique_commit u1 tree -p u0 +on_committer_date "00:02" save_tag u2 unique_commit u2 tree -p u0 +on_committer_date "00:03" save_tag u3 unique_commit u3 tree -p u0 +on_committer_date "00:04" save_tag u4 unique_commit u4 tree -p u0 +on_committer_date "00:05" save_tag u5 unique_commit u5 tree -p u0 +on_committer_date "00:06" save_tag V unique_commit V tree -p u1 -p u2 -p u3 -p u4 -p u5 + +test_sequence() +{ + _bisect_option=$1 + + test_bisection_diff 0 $_bisect_option l0 ^root + test_bisection_diff 0 $_bisect_option l1 ^root + test_bisection_diff 0 $_bisect_option l2 ^root + test_bisection_diff 0 $_bisect_option a0 ^root + test_bisection_diff 0 $_bisect_option a1 ^root + test_bisection_diff 0 $_bisect_option a2 ^root + test_bisection_diff 0 $_bisect_option a3 ^root + test_bisection_diff 0 $_bisect_option b1 ^root + test_bisection_diff 0 $_bisect_option b2 ^root + test_bisection_diff 0 $_bisect_option b3 ^root + test_bisection_diff 0 $_bisect_option c1 ^root + test_bisection_diff 0 $_bisect_option c2 ^root + test_bisection_diff 0 $_bisect_option c3 ^root + test_bisection_diff 0 $_bisect_option E ^F + test_bisection_diff 0 $_bisect_option e1 ^F + test_bisection_diff 0 $_bisect_option e2 ^F + test_bisection_diff 0 $_bisect_option e3 ^F + test_bisection_diff 0 $_bisect_option e4 ^F + test_bisection_diff 0 $_bisect_option e5 ^F + test_bisection_diff 0 $_bisect_option e6 ^F + test_bisection_diff 0 $_bisect_option e7 ^F + test_bisection_diff 0 $_bisect_option f1 ^F + test_bisection_diff 0 $_bisect_option f2 ^F + test_bisection_diff 0 $_bisect_option f3 ^F + test_bisection_diff 0 $_bisect_option f4 ^F + test_bisection_diff 0 $_bisect_option E ^F + + test_bisection_diff 1 $_bisect_option V ^U + test_bisection_diff 0 $_bisect_option V ^U ^u1 ^u2 ^u3 + test_bisection_diff 0 $_bisect_option u1 ^U + test_bisection_diff 0 $_bisect_option u2 ^U + test_bisection_diff 0 $_bisect_option u3 ^U + test_bisection_diff 0 $_bisect_option u4 ^U + test_bisection_diff 0 $_bisect_option u5 ^U + +# +# the following illustrates Linus' binary bug blatt idea. +# +# assume the bug is actually at l3, but you don't know that - all you know is that l3 is broken +# and it wasn't broken before +# +# keep bisecting the list, advancing the "bad" head and accumulating "good" heads until +# the bisection point is the head - this is the bad point. +# + +test_output_expect_success "$_bisect_option l5 ^root" 'git rev-list $_bisect_option l5 ^root' <expect && + git rev-list --bisect >actual && + test_cmp expect actual +' + +test_expect_success 'rev-parse --bisect can default to good/bad refs' ' + git rev-parse c3 ^b1 ^c1 >expect && + git rev-parse --bisect >actual && + + # output order depends on the refnames, which in turn depends on + # the exact sha1s. We just want to make sure we have the same set + # of lines in any order. + sort expect.sorted && + sort actual.sorted && + test_cmp expect.sorted actual.sorted +' + +test_output_expect_success '--bisect --first-parent' 'git rev-list --bisect --first-parent E ^F' <expect.unsorted <<-EOF && + $(git rev-parse E) (tag: E, dist=0) + $(git rev-parse e1) (tag: e1, dist=1) + $(git rev-parse e2) (tag: e2, dist=2) + $(git rev-parse e3) (tag: e3, dist=3) + $(git rev-parse e4) (tag: e4, dist=4) + $(git rev-parse e5) (tag: e5, dist=4) + $(git rev-parse e6) (tag: e6, dist=3) + $(git rev-parse e7) (tag: e7, dist=2) + $(git rev-parse e8) (tag: e8, dist=1) + EOF + + # expect results to be ordered by distance (descending), + # commit hash (ascending) + sort -k4,4r -k1,1 expect.unsorted >expect && + git rev-list --bisect-all --first-parent E ^F >actual && + test_cmp expect actual +' + +test_expect_success '--bisect without any revisions' ' + git rev-list --bisect HEAD..HEAD >out && + test_must_be_empty out +' + +test_done diff --git a/t/t4220/trim-divergent-old b/t/t4220/trim-divergent-old new file mode 100644 index 00000000000000..daa009c9a1b4b6 --- /dev/null +++ b/t/t4220/trim-divergent-old @@ -0,0 +1,316 @@ +#!/bin/sh +# +# Copyright (c) 2005 Jon Seymour +# +test_description='Tests git rev-list --bisect functionality' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-t6000.sh # t6xxx specific functions + +# usage: test_bisection max-diff bisect-option head ^prune... +# +# e.g. test_bisection 1 --bisect l1 ^l0 +# +test_bisection_diff() +{ + _max_diff=$1 + _bisect_option=$2 + shift 2 + _bisection=$(git rev-list $_bisect_option "$@") + _list_size=$(git rev-list "$@" | wc -l) + _head=$1 + shift 1 + _bisection_size=$(git rev-list $_bisection "$@" | wc -l) + [ -n "$_list_size" -a -n "$_bisection_size" ] || + error "test_bisection_diff failed" + + # Test if bisection size is close to half of list size within + # tolerance. + # + _bisect_err=$(expr $_list_size - $_bisection_size \* 2) + test "$_bisect_err" -lt 0 && _bisect_err=$(expr 0 - $_bisect_err) + _bisect_err=$(expr $_bisect_err / 2) ; # floor + + test_expect_success \ + "bisection diff $_bisect_option $_head $* <= $_max_diff" \ + 'test $_bisect_err -le $_max_diff' +} + +date >path0 +git update-index --add path0 +save_tag tree git write-tree +on_committer_date "00:00" hide_error save_tag root unique_commit root tree +on_committer_date "00:01" save_tag l0 unique_commit l0 tree -p root +on_committer_date "00:02" save_tag l1 unique_commit l1 tree -p l0 +on_committer_date "00:03" save_tag l2 unique_commit l2 tree -p l1 +on_committer_date "00:04" save_tag a0 unique_commit a0 tree -p l2 +on_committer_date "00:05" save_tag a1 unique_commit a1 tree -p a0 +on_committer_date "00:06" save_tag b1 unique_commit b1 tree -p a0 +on_committer_date "00:07" save_tag c1 unique_commit c1 tree -p b1 +on_committer_date "00:08" save_tag b2 unique_commit b2 tree -p b1 +on_committer_date "00:09" save_tag b3 unique_commit b2 tree -p b2 +on_committer_date "00:10" save_tag c2 unique_commit c2 tree -p c1 -p b2 +on_committer_date "00:11" save_tag c3 unique_commit c3 tree -p c2 +on_committer_date "00:12" save_tag a2 unique_commit a2 tree -p a1 +on_committer_date "00:13" save_tag a3 unique_commit a3 tree -p a2 +on_committer_date "00:14" save_tag b4 unique_commit b4 tree -p b3 -p a3 +on_committer_date "00:15" save_tag a4 unique_commit a4 tree -p a3 -p b4 -p c3 +on_committer_date "00:16" save_tag l3 unique_commit l3 tree -p a4 +on_committer_date "00:17" save_tag l4 unique_commit l4 tree -p l3 +on_committer_date "00:18" save_tag l5 unique_commit l5 tree -p l4 +git update-ref HEAD $(tag l5) + + +# E +# / \ +# e1 | +# | | +# e2 | +# | | +# e3 | +# | | +# e4 | +# | | +# | f1 +# | | +# | f2 +# | | +# | f3 +# | | +# | f4 +# | | +# e5 | +# | | +# e6 | +# | | +# e7 | +# | | +# e8 | +# \ / +# F + + +on_committer_date "00:00" hide_error save_tag F unique_commit F tree +on_committer_date "00:01" save_tag e8 unique_commit e8 tree -p F +on_committer_date "00:02" save_tag e7 unique_commit e7 tree -p e8 +on_committer_date "00:03" save_tag e6 unique_commit e6 tree -p e7 +on_committer_date "00:04" save_tag e5 unique_commit e5 tree -p e6 +on_committer_date "00:05" save_tag f4 unique_commit f4 tree -p F +on_committer_date "00:06" save_tag f3 unique_commit f3 tree -p f4 +on_committer_date "00:07" save_tag f2 unique_commit f2 tree -p f3 +on_committer_date "00:08" save_tag f1 unique_commit f1 tree -p f2 +on_committer_date "00:09" save_tag e4 unique_commit e4 tree -p e5 +on_committer_date "00:10" save_tag e3 unique_commit e3 tree -p e4 +on_committer_date "00:11" save_tag e2 unique_commit e2 tree -p e3 +on_committer_date "00:12" save_tag e1 unique_commit e1 tree -p e2 +on_committer_date "00:13" save_tag E unique_commit E tree -p e1 -p f1 + +on_committer_date "00:00" hide_error save_tag U unique_commit U tree +on_committer_date "00:01" save_tag u0 unique_commit u0 tree -p U +on_committer_date "00:01" save_tag u1 unique_commit u1 tree -p u0 +on_committer_date "00:02" save_tag u2 unique_commit u2 tree -p u0 +on_committer_date "00:03" save_tag u3 unique_commit u3 tree -p u0 +on_committer_date "00:04" save_tag u4 unique_commit u4 tree -p u0 +on_committer_date "00:05" save_tag u5 unique_commit u5 tree -p u0 +on_committer_date "00:06" save_tag V unique_commit V tree -p u1 -p u2 -p u3 -p u4 -p u5 + +test_sequence() +{ + _bisect_option=$1 + + test_bisection_diff 0 $_bisect_option l0 ^root + test_bisection_diff 0 $_bisect_option l1 ^root + test_bisection_diff 0 $_bisect_option l2 ^root + test_bisection_diff 0 $_bisect_option a0 ^root + test_bisection_diff 0 $_bisect_option a1 ^root + test_bisection_diff 0 $_bisect_option a2 ^root + test_bisection_diff 0 $_bisect_option a3 ^root + test_bisection_diff 0 $_bisect_option b1 ^root + test_bisection_diff 0 $_bisect_option b2 ^root + test_bisection_diff 0 $_bisect_option b3 ^root + test_bisection_diff 0 $_bisect_option c1 ^root + test_bisection_diff 0 $_bisect_option c2 ^root + test_bisection_diff 0 $_bisect_option c3 ^root + test_bisection_diff 0 $_bisect_option E ^F + test_bisection_diff 0 $_bisect_option e1 ^F + test_bisection_diff 0 $_bisect_option e2 ^F + test_bisection_diff 0 $_bisect_option e3 ^F + test_bisection_diff 0 $_bisect_option e4 ^F + test_bisection_diff 0 $_bisect_option e5 ^F + test_bisection_diff 0 $_bisect_option e6 ^F + test_bisection_diff 0 $_bisect_option e7 ^F + test_bisection_diff 0 $_bisect_option f1 ^F + test_bisection_diff 0 $_bisect_option f2 ^F + test_bisection_diff 0 $_bisect_option f3 ^F + test_bisection_diff 0 $_bisect_option f4 ^F + test_bisection_diff 0 $_bisect_option E ^F + + test_bisection_diff 1 $_bisect_option V ^U + test_bisection_diff 0 $_bisect_option V ^U ^u1 ^u2 ^u3 + test_bisection_diff 0 $_bisect_option u1 ^U + test_bisection_diff 0 $_bisect_option u2 ^U + test_bisection_diff 0 $_bisect_option u3 ^U + test_bisection_diff 0 $_bisect_option u4 ^U + test_bisection_diff 0 $_bisect_option u5 ^U + +# +# the following illustrates Linus' binary bug blatt idea. +# +# assume the bug is actually at l3, but you don't know that - all you know is that l3 is broken +# and it wasn't broken before +# +# keep bisecting the list, advancing the "bad" head and accumulating "good" heads until +# the bisection point is the head - this is the bad point. +# + +test_output_expect_success "$_bisect_option l5 ^root" 'git rev-list $_bisect_option l5 ^root' <expect && + git rev-list --bisect >actual && + test_cmp expect actual +' + +test_expect_success 'rev-parse --bisect can default to good/bad refs' ' + git rev-parse c3 ^b1 ^c1 >expect && + git rev-parse --bisect >actual && + + # output order depends on the refnames, which in turn depends on + # the exact sha1s. We just want to make sure we have the same set + # of lines in any order. + sort expect.sorted && + sort actual.sorted && + test_cmp expect.sorted actual.sorted +' + +test_output_expect_success '--bisect --first-parent' 'git rev-list --bisect --first-parent E ^F' <expect.unsorted <<-EOF && + $(git rev-parse E) (tag: E, dist=0) + $(git rev-parse e1) (tag: e1, dist=1) + $(git rev-parse e2) (tag: e2, dist=2) + $(git rev-parse e3) (tag: e3, dist=3) + $(git rev-parse e4) (tag: e4, dist=4) + $(git rev-parse e5) (tag: e5, dist=4) + $(git rev-parse e6) (tag: e6, dist=3) + $(git rev-parse e7) (tag: e7, dist=2) + $(git rev-parse e8) (tag: e8, dist=1) + EOF + + # expect results to be ordered by distance (descending), + # commit hash (ascending) + sort -k4,4r -k1,1 expect.unsorted >expect && + git rev-list --bisect-all --first-parent E ^F >actual && + test_cmp expect actual +' + +test_expect_success '--bisect without any revisions' ' + git rev-list --bisect HEAD..HEAD >out && + test_must_be_empty out +' + +test_done diff --git a/t/t5308-pack-detect-duplicates.sh b/t/t5308-pack-detect-duplicates.sh index 0f841378677165..c6273a1aeb2973 100755 --- a/t/t5308-pack-detect-duplicates.sh +++ b/t/t5308-pack-detect-duplicates.sh @@ -27,6 +27,11 @@ HI_SHA1=$EMPTY_BLOB # duplicate runs). MISSING_SHA1=$(test_oid missing_oid) +# Three distinct objects for tests where physical pack order matters. +A=$(test_oid packlib_7_0) +B=$LO_SHA1 +C=$HI_SHA1 + # git will never intentionally create packfiles with # duplicate objects, so we have to construct them by hand. # @@ -54,6 +59,13 @@ test_expect_success 'index-pack will allow duplicate objects by default' ' git index-pack --stdin err && + test_grep "fatal: pack contains duplicate object" err +' + test_expect_success 'create batch-check test vectors' ' cat >input <<-EOF && $LO_SHA1 @@ -72,6 +84,54 @@ test_expect_success 'lookup in duplicated pack' ' test_cmp expect actual ' +test_expect_success 'verify MIDX containing duplicated pack objects' ' + git multi-pack-index write && + test-tool read-midx --show-objects .git/objects >midx-objects && + midx_offset=$( + awk -v oid="$LO_SHA1" "\$1 == oid { print \$2 }" physical-order.pack && + pack_trailer physical-order.pack && + + test_must_fail git index-pack --rev-index --stdin --strict \ + err && + test_grep "appears twice in the pack" err && + + git index-pack --rev-index --stdin offsets.raw && + + sort -n offsets.raw | grep -A1 "$B" | cut -d" " -f1 >adjacent && + echo $(($(tail -n1 adjacent) - $(head -n1 adjacent))) >expect && + echo "$B" >in && + + GIT_TEST_REV_INDEX_DIE_IN_MEMORY=1 \ + git cat-file --batch-check="%(objectsize:disk)" \ + actual.disk && + GIT_TEST_REV_INDEX_DIE_ON_DISK=1 \ + git -c pack.readReverseIndex=false \ + cat-file --batch-check="%(objectsize:disk)" \ + actual.mem && + + test_cmp expect actual.disk && + test_cmp expect actual.mem +' + test_expect_success 'index-pack can reject packs with duplicates' ' clear_packs && create_pack dups.pack 2 && diff --git a/t/t5309-pack-delta-cycles.sh b/t/t5309-pack-delta-cycles.sh index 6b03675d91b5e1..f613950e38f262 100755 --- a/t/t5309-pack-delta-cycles.sh +++ b/t/t5309-pack-delta-cycles.sh @@ -9,6 +9,83 @@ test_description='test index-pack handling of delta cycles in packfiles' A=$(test_oid packlib_7_0) B=$(test_oid packlib_7_76) +# Copy the entries from a complete pack without its header or trailer. +pack_entries () { + entry_size=$(wc -c <"$1") && + dd if="$1" bs=1 skip=12 \ + count=$((entry_size - 12 - $(test_oid rawsz))) 2>/dev/null +} + +# B as an OFS_DELTA against A at the given one-byte distance. +pack_obj_b_ofs_a () { + pack_obj "$B" "$A" >b-ref.tmp && + printf "\145" && + printf "\\$(printf "%03o" "$1")" && + dd if=b-ref.tmp bs=1 skip=$((1 + $(test_oid rawsz))) 2>/dev/null +} + +# Return the base of the first one-byte-header REF_DELTA for the given OID. +first_ref_base () { + idx=$(echo .git/objects/pack/*.idx) && + offset=$(git show-index <"$idx" | + awk -v oid="$1" '$2 == oid { print $1; exit }') && + dd if="${idx%.idx}.pack" bs=1 skip=$((offset + 1)) \ + count=$(test_oid rawsz) 2>/dev/null | + test-tool hexdump | + tr -d " \n" +} + +# The order of equal-OID entries in the .idx is unspecified. Retain a pack +# which selects $1 as a delta against $2. Unless $3 is "-", also require the +# first copy searched during recovery to be a REF_DELTA against $3. +install_cycle () { + cycle_oid=$1 && + cycle_base=$2 && + first_base=$3 && + shift 3 && + for pack + do + clear_packs && + git index-pack --fix-thin --stdin <"$pack" && + selected_base=$(echo "$cycle_oid" | + git cat-file --batch-check="%(deltabase)") || + return 1 + if test "$selected_base" = "$cycle_base" && + { test "$first_base" = "-" || + test "$(first_ref_base "$cycle_oid")" = "$first_base"; } + then + return 0 + fi + done + return 1 +} + +make_cycle_pack () { + cycle_pack=$1 && + shift && + test-tool -C alt-source pack-deltas --num-objects=6 >refs.tmp <<-EOF && + REF_DELTA $T $X + REF_DELTA $X $1 + REF_DELTA $Y $Z + REF_DELTA $Z $X + REF_DELTA $X $2 + REF_DELTA $X $3 + EOF + { + pack_header 8 && + pack_entries refs.tmp && + cat a-full && + pack_obj_b_ofs_a "$a_full_size" + } >"$cycle_pack" && + pack_trailer "$cycle_pack" +} + +check_blob () { + test "$(git cat-file -t "$1")" = blob && + git cat-file blob "$1" >actual && + test_cmp_bin "$2" actual +} + # double-check our hand-constucted packs test_expect_success 'index-pack works with a single delta (A->B)' ' clear_packs && @@ -67,18 +144,77 @@ test_expect_success 'failover to an object in another pack' ' ' test_expect_success 'failover to a duplicate object in the same pack' ' - clear_packs && + { + pack_header 3 && + pack_obj $A && + pack_obj $B $A && + pack_obj $A $B + } >recoverable-1.pack && + pack_trailer recoverable-1.pack && { pack_header 3 && pack_obj $A $B && pack_obj $B $A && pack_obj $A - } >recoverable.pack && - pack_trailer recoverable.pack && + } >recoverable-2.pack && + pack_trailer recoverable-2.pack && + + # The selected copy of A is part of the cycle, but the full copy + # lets both type and content lookups resolve it. + install_cycle "$A" "$B" - recoverable-1.pack recoverable-2.pack && + printf "\7\0" >expect && + check_blob "$A" expect +' + +test_expect_success 'failover from a mixed REF/OFS cycle' ' + pack_obj "$A" "$B" >a-ref && + pack_obj "$B" >b-full && + a_ref_size=$(wc -c mixed-1.pack && + pack_trailer mixed-1.pack && + { + pack_header 3 && + cat a-ref && + pack_obj_b_ofs_a "$a_ref_size" && + cat b-full + } >mixed-2.pack && + pack_trailer mixed-2.pack && + + # The REF_DELTA for A selects the OFS_DELTA copy of B; the + # full B is its escape. + install_cycle "$B" "$A" - mixed-1.pack mixed-2.pack && + printf "\7\0" >expect && + check_blob "$A" expect +' - # This cycle does not fail since the existence of a full copy - # of A in the pack allows us to resolve the cycle. - git index-pack --fix-thin --stdin /dev/null && + X=$(printf x | git -C alt-source hash-object -w --stdin) && + Y=$(printf y | git -C alt-source hash-object -w --stdin) && + Z=$(printf z | git -C alt-source hash-object -w --stdin) && + printf "tail T\n" >tail && + T=$(git -C alt-source hash-object -w --stdin a-full && + a_full_size=$(wc -c X->Y->Z->X. Recovery must exhaust that + # branch, then use X->B->A, whose final edge is an OFS_DELTA. + install_cycle "$X" "$Y" "$Y" \ + alternate-1.pack alternate-2.pack alternate-3.pack && + check_blob "$T" tail ' test_expect_success 'index-pack works with thin pack A->B->C with B on disk' ' diff --git a/t/t5324-split-commit-graph.sh b/t/t5324-split-commit-graph.sh index bf7ba0e5580d84..29db815c773a98 100755 --- a/t/t5324-split-commit-graph.sh +++ b/t/t5324-split-commit-graph.sh @@ -349,7 +349,7 @@ test_expect_success 'verify after commit-graph-chain corruption (base)' ' test_must_fail git commit-graph verify 2>test_err && grep -v "^+" test_err >err && test_grep "invalid commit-graph chain" err && - corrupt_file "$graphdir/commit-graph-chain" 30 "A" && + corrupt_file "$graphdir/commit-graph-chain" 30 "a" && test_must_fail git commit-graph verify 2>test_err && grep -v "^+" test_err >err && test_grep "unable to find all commit-graph files" err @@ -364,7 +364,7 @@ test_expect_success 'verify after commit-graph-chain corruption (tip)' ' test_must_fail git commit-graph verify 2>test_err && grep -v "^+" test_err >err && test_grep "invalid commit-graph chain" err && - corrupt_file "$graphdir/commit-graph-chain" 70 "A" && + corrupt_file "$graphdir/commit-graph-chain" 70 "b" && test_must_fail git commit-graph verify 2>test_err && grep -v "^+" test_err >err && test_grep "unable to find all commit-graph files" err diff --git a/t/t5332-multi-pack-reuse.sh b/t/t5332-multi-pack-reuse.sh index 881ce668e1d14e..126ab9df991ce4 100755 --- a/t/t5332-multi-pack-reuse.sh +++ b/t/t5332-multi-pack-reuse.sh @@ -4,6 +4,7 @@ test_description='pack-objects multi-pack reuse' . ./test-lib.sh . "$TEST_DIRECTORY"/lib-bitmap.sh +. "$TEST_DIRECTORY"/lib-pack.sh GIT_TEST_MULTI_PACK_INDEX=0 GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0 @@ -32,6 +33,14 @@ pack_position () { grep "$1" objects | cut -d" " -f1 } +# B as an OFS_DELTA against A at the given one-byte distance. +pack_obj_b_ofs_a () { + pack_obj "$B" "$A" >b-ref.tmp && + printf "\145" && + printf "\\$(printf "%03o" "$1")" && + dd if=b-ref.tmp bs=1 skip=$((1 + $(test_oid rawsz))) 2>/dev/null +} + # test_pack_objects_reused_all test_pack_objects_reused_all () { : >trace2.txt && @@ -288,4 +297,84 @@ test_expect_success 'duplicate objects with verbatim reuse' ' ) ' +test_expect_success 'reuse with intra-pack duplicate objects' ' + git init intra-pack-duplicate-objects && + ( + cd intra-pack-duplicate-objects && + + # Make enough objects to exercise whole-word reuse. + test_commit_bulk 20 && + test_commit --printf A a "\7\0" && + test_commit --printf B b "\7\76" && + + objects_nr=$(git rev-list --count --objects --all) && + git rev-list --objects --all | + cut -d" " -f1 >objects && + A=$(test_oid packlib_7_0) && + B=$(test_oid packlib_7_76) && + grep -v -e "^$A$" -e "^$B$" objects >rest && + pack_obj "$A" >a-full && + pack_obj "$B" >b-full && + while read oid + do + pack_obj "$oid" || exit 1 + done rest.entries && + { + # Arrange the pack as A, B, A, C..., so that physical + # positions diverge from MIDX pseudo-pack order. + pack_header $((objects_nr + 1)) && + cat a-full b-full a-full rest.entries + } >duplicate.pack && + pack_trailer duplicate.pack && + clear_packs && + git index-pack --stdin b-delta && + { + pack_header "$((objects_nr + 1))" && + cat a-full a-full b-delta rest.entries + } >candidate.pack && + pack_trailer candidate.pack && + clear_packs && + git index-pack --stdin "$midx_chain" && + test-tool read-midx --show-objects "$objdir" "$layer" >midx.objects && + test_grep "^$(git rev-parse 2.2) " midx.objects && + cp "$midx_chain.bak" "$midx_chain" ' test_expect_success 'write MIDX layer with --base= and --no-write-chain-file' ' @@ -136,11 +141,20 @@ test_expect_success 'write MIDX layer with --base= and --no-write-chain-fi git repack -d && cp "$midx_chain" "$midx_chain.bak" && + base="$(nth_line 1 "$midx_chain")" && layer="$(git multi-pack-index write --bitmap --incremental \ - --no-write-chain-file --base="$(nth_line 1 "$midx_chain")")" && + --no-write-chain-file --base="$base")" && test_cmp "$midx_chain.bak" "$midx_chain" && - test_path_is_file "$midxdir/multi-pack-index-$layer.midx" + test_path_is_file "$midxdir/multi-pack-index-$layer.midx" && + + { + echo "$base" && + echo "$layer" + } >"$midx_chain" && + test-tool read-midx --show-objects "$objdir" "$layer" >midx.objects && + test_grep "^$(git rev-parse 2.2) " midx.objects && + cp "$midx_chain.bak" "$midx_chain" ' for reuse in false single multi diff --git a/t/t5335-compact-multi-pack-index.sh b/t/t5335-compact-multi-pack-index.sh index 0f75ef2ccc96b3..adcf94b2fbe726 100755 --- a/t/t5335-compact-multi-pack-index.sh +++ b/t/t5335-compact-multi-pack-index.sh @@ -3,6 +3,7 @@ test_description='multi-pack-index compaction' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-midx.sh GIT_TEST_MULTI_PACK_INDEX=0 GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0 @@ -13,12 +14,6 @@ packdir=$objdir/pack midxdir=$packdir/multi-pack-index.d midx_chain=$midxdir/multi-pack-index-chain -nth_line() { - local n="$1" - shift - awk "NR==$n" "$@" -} - write_packs () { for c in "$@" do diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index f3b3efc47f8a48..b982b209bfd51e 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -1903,20 +1903,20 @@ test_expect_success 'push with config push.useBitmaps' ' test_unconfig push.useBitmaps && GIT_TRACE2_EVENT="$PWD/default" \ git push --quiet testrepo main:test && - test_subcommand git pack-objects --all-progress-implied --revs --stdout \ - --thin --delta-base-offset -q +create_err_env () { + local envdir="$1" && + mkdir "$envdir" && + + git init --bare "$envdir/sub_bare" && + git clone "$envdir/sub_bare" "$envdir/sub_work" && + test_commit -C "$envdir/sub_work" "${envdir}_base" && + git -C "$envdir/sub_work" push && + + git init --bare "$envdir/super_bare" && + git clone "$envdir/super_bare" "$envdir/super_work" && + git -C "$envdir/super_work" submodule add \ + "$pwd/$envdir/sub_bare" sub && + git -C "$envdir/super_work" commit -m "add submodule" && + git -C "$envdir/super_work" push && + + git clone "$envdir/super_bare" "$envdir/clone" && + git -C "$envdir/clone" submodule update --init +} + +# Push a commit to /super_bare that records a submodule SHA that is +# present locally in super_work/sub but NOT pushed to sub_bare, making the +# submodule commit unreachable from clone's sub remote. +push_unreachable_commit () { + local envdir="$1" && + git -C "$envdir/super_work/sub" commit --allow-empty -m "unreachable" && + git -C "$envdir/super_work" add sub && + git -C "$envdir/super_work" commit -m "point sub to unreachable commit" && + git -C "$envdir/super_work" push +} + +test_expect_success 'setup for submodule fetch error tests' ' + git config --global protocol.file.allow always +' + +test_expect_success 'fetch --recurse-submodules fails when submodule commit is unreachable (default)' ' + test_when_finished "rm -fr env_default" && + create_err_env env_default && + push_unreachable_commit env_default && + test_must_fail git -C env_default/clone fetch --recurse-submodules 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success 'fetch.submoduleErrors=warn: unreachable submodule commit is non-fatal' ' + test_when_finished "rm -fr env_warn_cfg" && + create_err_env env_warn_cfg && + push_unreachable_commit env_warn_cfg && + git -C env_warn_cfg/clone -c fetch.submoduleErrors=warn \ + fetch --recurse-submodules 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success '--submodule-errors=warn: unreachable submodule commit is non-fatal' ' + test_when_finished "rm -fr env_warn_cli" && + create_err_env env_warn_cli && + push_unreachable_commit env_warn_cli && + git -C env_warn_cli/clone fetch --recurse-submodules \ + --submodule-errors=warn 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success '--submodule-errors=fail: unreachable submodule commit is fatal' ' + test_when_finished "rm -fr env_fail_cli" && + create_err_env env_fail_cli && + push_unreachable_commit env_fail_cli && + test_must_fail git -C env_fail_cli/clone fetch --recurse-submodules \ + --submodule-errors=fail 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success 'fetch.submoduleErrors=warn does not suppress successful fetch' ' + # A new reachable submodule commit (pushed to sub_bare) should be + # fetched without any error summary. + test_when_finished "rm -fr env_ok" && + create_err_env env_ok && + test_commit -C env_ok/sub_work reachable_ok && + git -C env_ok/sub_work push && + git -C env_ok/super_work submodule update --remote && + git -C env_ok/super_work add sub && + git -C env_ok/super_work commit -m "point sub to reachable commit" && + git -C env_ok/super_work push && + git -C env_ok/clone -c fetch.submoduleErrors=warn \ + fetch --recurse-submodules 2>err && + test_grep ! "Errors during submodule fetch" err +' + +test_expect_success 'failed submodule fetch is fatal even when its commits are present locally' ' + # Create the same commit (unreferenced, via commit-tree with fixed + # dates) in both super_work/sub and clone/sub, point the gitlink at + # it, and break clone/sub'\''s remote. The commit exists in clone/sub + # but is unreachable, so the submodule stays in the changed list; the + # fetch failure must still be reported even though there is nothing + # left to fetch by commit hash. + test_when_finished "rm -fr env_phase1" && + create_err_env env_phase1 && + commit=$(GIT_AUTHOR_DATE="1234567890 +0000" \ + GIT_COMMITTER_DATE="1234567890 +0000" \ + git -C env_phase1/super_work/sub commit-tree \ + "HEAD^{tree}" -p HEAD -m present) && + present=$(GIT_AUTHOR_DATE="1234567890 +0000" \ + GIT_COMMITTER_DATE="1234567890 +0000" \ + git -C env_phase1/clone/sub commit-tree \ + "HEAD^{tree}" -p HEAD -m present) && + test "$commit" = "$present" && + git -C env_phase1/super_work/sub checkout "$commit" && + git -C env_phase1/super_work add sub && + git -C env_phase1/super_work commit -m "gitlink to locally-present commit" && + git -C env_phase1/super_work push && + git -C env_phase1/clone/sub remote set-url origin "$pwd/env_phase1/missing" && + test_must_fail git -C env_phase1/clone fetch --recurse-submodules 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success '--submodule-errors=warn is honored by fetch --all' ' + # A second remote forces fetch_multiple(), which hands the submodule + # recursion off to per-remote child processes; the option must be + # forwarded to them. + test_when_finished "rm -fr env_all" && + create_err_env env_all && + push_unreachable_commit env_all && + git -C env_all/clone remote add second "$pwd/env_all/super_bare" && + git -C env_all/clone fetch --all --recurse-submodules \ + --submodule-errors=warn 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success '--submodule-errors=fail overrides warn config for fetch --all' ' + # The per-remote child processes re-read the repository config, so + # the command-line override must be forwarded to them explicitly. + test_when_finished "rm -fr env_override" && + create_err_env env_override && + push_unreachable_commit env_override && + git -C env_override/clone remote add second "$pwd/env_override/super_bare" && + git -C env_override/clone config fetch.submoduleErrors warn && + test_must_fail git -C env_override/clone fetch --all --recurse-submodules \ + --submodule-errors=fail 2>err && + test_grep "Errors during submodule fetch" err +' + +test_expect_success 'fetch.submoduleErrors=warn: inaccessible submodule is non-fatal' ' + test_when_finished "rm -fr env_access" && + create_err_env env_access && + rm env_access/clone/sub/.git && + rm -r env_access/clone/.git/modules/sub && + git -C env_access/clone -c fetch.submoduleErrors=warn \ + fetch --recurse-submodules 2>err && + test_grep "Could not access submodule" err && + test_must_fail git -C env_access/clone fetch --recurse-submodules 2>err && + test_grep "Could not access submodule" err +' + test_done diff --git a/t/t5547-push-quarantine.sh b/t/t5547-push-quarantine.sh index 0798ddab02bc8b..400a59760600c6 100755 --- a/t/t5547-push-quarantine.sh +++ b/t/t5547-push-quarantine.sh @@ -70,4 +70,18 @@ test_expect_success 'updating a ref from quarantine is forbidden' ' git -C update.git fsck ' +test_expect_success '.keep file is removed after push' ' + test_when_finished rm -rf keep.git && + git init --bare keep.git && + + git -C keep.git config set receive.unpackLimit 0 && + test_commit foo && + git push keep.git HEAD && + pack="$(ls keep.git/objects/pack/pack-*.pack)" && + keep="${pack%.pack}.keep" && + + test_path_is_file "$pack" && + test_path_is_missing "$keep" +' + test_done diff --git a/t/t5551-http-fetch-smart.sh b/t/t5551-http-fetch-smart.sh index 805bec025c589f..c11e96c1ac81e5 100755 --- a/t/t5551-http-fetch-smart.sh +++ b/t/t5551-http-fetch-smart.sh @@ -680,6 +680,35 @@ test_expect_success 'passing hostname resolution information works' ' git -c "http.curloptResolve=$BOGUS_HOST:$LIB_HTTPD_PORT:127.0.0.1" ls-remote "$BOGUS_HTTPD_URL/smart/repo.git" >/dev/null ' +test_lazy_prereq SSL_VERIFYSTATUS ' + test "$HTTPD_PROTO" = "https" && + test_might_fail git -c http.sslVerifyStatus=true \ + ls-remote "$HTTPD_URL/smart/repo.git" 2>err && + ! grep "cannot verify certificate status" err +' + +test_expect_success SSL_VERIFYSTATUS 'http.sslVerifyStatus=true fails without a staple' ' + test_must_fail git -c http.sslVerifyStatus=true \ + ls-remote "$HTTPD_URL/smart/repo.git" +' + +test_expect_success SSL_VERIFYSTATUS 'http.sslVerifyStatus=false is a no-op' ' + git -c http.sslVerifyStatus=false \ + ls-remote "$HTTPD_URL/smart/repo.git" >actual && + test_line_count -gt 0 actual +' + +test_expect_success SSL_VERIFYSTATUS 'per-URL sslVerifyStatus applies to a matching URL' ' + test_must_fail git -c "http.$HTTPD_URL/.sslVerifyStatus=true" \ + ls-remote "$HTTPD_URL/smart/repo.git" +' + +test_expect_success SSL_VERIFYSTATUS 'per-URL sslVerifyStatus is not applied to other URLs' ' + git -c "http.https://example.com/.sslVerifyStatus=true" \ + ls-remote "$HTTPD_URL/smart/repo.git" >actual && + test_line_count -gt 0 actual +' + # here user%40host is the URL-encoded version of user@host, # which is our intentionally-odd username to catch parsing errors url_user=$HTTPD_URL_USER/auth/smart/repo.git diff --git a/t/t5567-one-time-script.sh b/t/t5567-one-time-script.sh new file mode 100755 index 00000000000000..cd8e6560056160 --- /dev/null +++ b/t/t5567-one-time-script.sh @@ -0,0 +1,96 @@ +#!/bin/sh + +test_description='apply-one-time-script CGI helper is safe under concurrent requests' + +. ./test-lib.sh + +HELPER="$TEST_DIRECTORY/lib-httpd/apply-one-time-script.sh" + +test_expect_success PIPE 'concurrent requests: one rewritten, one passed through, neither empty' ' + mkdir workdir fakebin && + ENTERED="$PWD/entered" && + GATE="$PWD/gate" && + export ENTERED GATE && + mkfifo "$ENTERED" "$GATE" && + + # Stand in for git-http-backend. The modify role returns a response + # containing "packfile", which the one-time script rewrites. The + # passthrough role returns a response that is left untouched, but first + # announces that it has entered the helper and then blocks, so that it + # is still in flight when the modify role claims and removes the marker. + write_script fakebin/git-http-backend <<-\EOF && + printf "Status: 200 OK\r\n" + printf "Content-Type: application/x-git-result\r\n" + printf "\r\n" + if test "$ROLE" = modify + then + printf "packfile\n" + else + echo entered >"$ENTERED" + read -r released <"$GATE" + printf "refs\n" + fi + EOF + + # The transform that replace_packfile would install as one-time-script: + # rewrite responses that contain "packfile", leave the rest alone. + write_script workdir/one-time-script <<-\EOF && + if grep packfile "$1" >/dev/null + then + sed "/packfile/q" "$1" && + printf "REPLACED\n" + else + cat "$1" + fi + EOF + + GIT_EXEC_PATH="$PWD/fakebin" && + export GIT_EXEC_PATH && + + # Hold GATE open read-write on fd 9 for the duration, so releasing the + # passthrough request below cannot block even if that request has + # already exited (it keeps a reader on the FIFO). + exec 9<>"$GATE" && + + # Launch the passthrough request in the background. It enters the + # helper, signals us through ENTERED, then blocks on GATE inside the + # fake backend. The braces keep the && chain intact while backgrounding + # only the subshell, so "wait" can reap it by pid; kill it on any exit + # so a stray blocked child cannot hold the test output open and stall a + # reader such as prove. + { ( + cd workdir && + ROLE=passthrough sh "$HELPER" >../passthrough.out 2>../passthrough.err + ) & } && + passthrough_pid=$! && + test_when_finished "kill $passthrough_pid 2>/dev/null || :" && + + # Wait until the passthrough request is past the marker check. + read -r entered <"$ENTERED" && + + # Run the modifying request to completion while the passthrough request + # is still blocked. + ( + cd workdir && + ROLE=modify sh "$HELPER" >../modify.out 2>../modify.err + ) && + + # Release the passthrough request and let it finish. Ignore the helper + # exit status here so a broken helper is diagnosed by the assertions + # below rather than aborting the test. + echo released >&9 && + { wait "$passthrough_pid" || :; } && + + # Neither request may error out or produce an empty (HTTP 500) body, + # and each must have played its role: the modify request rewrote its + # response and the passthrough request came through untouched. + test_must_be_empty passthrough.err && + test_must_be_empty modify.err && + test_grep "Status: 200 OK" passthrough.out && + test_grep "Status: 200 OK" modify.out && + test_grep REPLACED modify.out && + test_grep ! REPLACED passthrough.out && + test_grep refs passthrough.out +' + +test_done diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh index 9a575aa098afd3..30f6c41af00397 100755 --- a/t/t5701-git-serve.sh +++ b/t/t5701-git-serve.sh @@ -71,6 +71,17 @@ test_expect_success 'request invalid capability' ' test_grep "unknown capability" err ' +test_expect_success 'promisor-remote capability requires an argument' ' + test-tool pkt-line pack >in <<-EOF && + command=ls-refs + object-format=$(test_oid algo) + promisor-remote + 0000 + EOF + test_must_fail test-tool serve-v2 --stateless-rpc 2>err in <<-EOF && agent=git/test @@ -344,20 +355,53 @@ test_expect_success 'unexpected lines are not allowed in fetch request' ' test_expect_success 'basics of object-info' ' test_config transfer.advertiseObjectInfo true && + two_oid=$(git rev-parse two:two.t) && + two_size=$(test_file_size two.t) && + test-tool pkt-line pack >in <<-EOF && command=object-info object-format=$(test_oid algo) 0001 size - oid $(git rev-parse two:two.t) - oid $(git rev-parse two:two.t) + oid $two_oid + oid $two_oid + 0000 + EOF + + cat >expect <<-EOF && + size + $two_oid $two_size + $two_oid $two_size + 0000 + EOF + + test-tool serve-v2 --stateless-rpc out && + test-tool pkt-line unpack actual && + test_cmp expect actual +' + +test_expect_success 'object-info supports type' ' + test_config transfer.advertiseObjectInfo true && + + two_oid=$(git rev-parse two:two.t) && + two_size=$(test_file_size two.t) && + + test-tool pkt-line pack >in <<-EOF && + command=object-info + object-format=$(test_oid algo) + 0001 + size + type + oid $two_oid + oid $two_oid 0000 EOF cat >expect <<-EOF && size - $(git rev-parse two:two.t) $(wc -c err && + test_grep "lazy fetching disabled" err && + + # Check that the largest object is still missing on the server + check_missing_objects server 1 "$oid" +' + +test_expect_success "uploadpack.lazyFetchTrusted is ignored in repo config" ' + # No promisors are advertised + git -C server config promisor.advertise false && + test_when_finished "rm -rf client" && + + # The served repo is trusted for lazy fetching, but this is + # done in the repo config, not in protected config, so this is + # ignored. + test_config -C server uploadpack.lazyFetchTrusted "$(pwd)/server" && + + # Clone without GIT_NO_LAZY_FETCH=0 fails + test_must_fail git clone --no-local --filter="blob:limit=5k" server client 2>err && + test_grep "lazy fetching disabled" err && + + # Check that the largest object is still missing on the server + check_missing_objects server 1 "$oid" +' + +test_expect_success "explicit GIT_NO_LAZY_FETCH overrides uploadpack.lazyFetchTrusted" ' + # No promisors are advertised + git -C server config promisor.advertise false && + test_when_finished "rm -rf client" && + + # The served repo is trusted for lazy fetching + test_config_global uploadpack.lazyFetchTrusted "$(pwd)/server" && + + # But GIT_NO_LAZY_FETCH=1 disables lazy fetching, so clone fails + test_must_fail env GIT_NO_LAZY_FETCH=1 git clone --no-local \ + --filter="blob:limit=5k" server client 2>err && + test_grep "lazy fetching disabled" err && + + # Check that the largest object is still missing on the server + check_missing_objects server 1 "$oid" +' + test_expect_success "init + fetch with promisor.advertise set to 'true'" ' git -C server config promisor.advertise true && test_when_finished "rm -rf client" && diff --git a/t/t6030-bisect-porcelain.sh b/t/t6030-bisect-porcelain.sh index 338df4f886a424..a7588222a8f084 100755 --- a/t/t6030-bisect-porcelain.sh +++ b/t/t6030-bisect-porcelain.sh @@ -43,6 +43,42 @@ test_bisect_usage () { test_cmp expect actual } +test_bisect_state_file () { + local file && + file=$(git rev-parse --git-path "$1") && + test_path_is_file "$file" +} + +test_bisect_state_missing () { + local file && + file=$(git rev-parse --git-path "$1") && + test_path_is_missing "$file" +} + +bisect_start_and_finish () { + git bisect start "$1" $HASH4 $HASH2 && + git bisect bad +} + +bisect_run_reset_when_found () { + write_script test_script.sh <<-\EOF && + ! grep Another hello >/dev/null + EOF + git bisect start $HASH4 $HASH2 && + git bisect run "$1" ./test_script.sh >my_bisect_log.txt && + test_grep "$HASH3 is the first .bad. commit" my_bisect_log.txt && + test_bisect_state_missing BISECT_RUN +} + +test_reset_when_found_fails () { + local pattern="$1" && + local state_file="$2" && + shift 2 && + test_must_fail "$@" 2>err && + test_grep -- "$pattern" err && + test_bisect_state_missing "$state_file" +} + test_expect_success 'bisect usage' " test_bisect_usage 1 git bisect reset extra1 extra2 <<-\EOF && error: 'git bisect reset' requires either no argument or a commit @@ -453,6 +489,91 @@ test_expect_success '"git bisect run" simple case' ' git bisect reset ' +test_expect_success '"git bisect start --reset-when-found" defaults to original' ' + test_when_finished "git bisect reset && git checkout main" && + git checkout main && + bisect_start_and_finish --reset-when-found && + actual=$(git rev-parse HEAD) && + test "$HASH4" = "$actual" && + actual=$(git branch --show-current) && + test main = "$actual" && + test_bisect_state_missing BISECT_START && + + bisect_start_and_finish --reset-when-found=original && + actual=$(git rev-parse HEAD) && + test "$HASH4" = "$actual" && + actual=$(git branch --show-current) && + test main = "$actual" && + test_bisect_state_missing BISECT_START +' + +test_expect_success '"git bisect start --reset-when-found=found" leaves first bad checked out' ' + test_when_finished "git bisect reset && git checkout main" && + bisect_start_and_finish --reset-when-found=found && + actual=$(git rev-parse HEAD) && + test "$HASH3" = "$actual" && + test_bisect_state_missing BISECT_START +' + +test_expect_success '"git bisect run --reset-when-found" defaults to original' ' + test_when_finished "git bisect reset && git checkout main" && + bisect_run_reset_when_found --reset-when-found && + actual=$(git rev-parse HEAD) && + test "$HASH4" = "$actual" && + actual=$(git branch --show-current) && + test main = "$actual" && + test_bisect_state_missing BISECT_START +' + +test_expect_success '"git bisect run --reset-when-found=found" leaves first bad checked out' ' + test_when_finished "git bisect reset && git checkout main" && + bisect_run_reset_when_found --reset-when-found=found && + actual=$(git rev-parse HEAD) && + test "$HASH3" = "$actual" && + test_bisect_state_missing BISECT_START +' + +test_expect_success '--reset-when-found rejects an unknown reset target' ' + test_when_finished "git bisect reset && git checkout main" && + test_reset_when_found_fails \ + "invalid value for.*--reset-when-found.*unknown" BISECT_START \ + git bisect start --reset-when-found=unknown $HASH4 $HASH2 && + + git bisect start $HASH4 $HASH2 && + test_reset_when_found_fails \ + "invalid value for.*--reset-when-found.*unknown" \ + BISECT_RESET_WHEN_FOUND \ + git bisect run --reset-when-found=unknown true +' + +test_expect_success '--reset-when-found cannot be used with --no-checkout' ' + test_when_finished "git bisect reset" && + test_reset_when_found_fails \ + "options .*--reset-when-found.* and .*--no-checkout.* cannot be used together" BISECT_START \ + git bisect start --reset-when-found=original --no-checkout $HASH4 $HASH2 && + + git bisect start --no-checkout $HASH4 $HASH2 && + test_reset_when_found_fails \ + "options .*--reset-when-found.* and .*--no-checkout.* cannot be used together" BISECT_RESET_WHEN_FOUND \ + git bisect run --reset-when-found=found true +' + +test_expect_success 'without --reset-when-found the bisection state is kept' ' + test_when_finished "git bisect reset" && + git bisect start $HASH4 $HASH2 && + git bisect bad && + test_bisect_state_file BISECT_START +' + +test_expect_success '--reset-when-found does not leak into a later bisection' ' + test_when_finished "git bisect reset && git checkout main" && + bisect_start_and_finish --reset-when-found && + + git bisect start $HASH4 $HASH2 && + git bisect bad && + test_bisect_state_file BISECT_START +' + # We want to automatically find the commit that # added "Ciao" into hello. test_expect_success '"git bisect run" with more complex "git bisect start"' ' diff --git a/t/t6099-merge-base-side-exhaustion.sh b/t/t6099-merge-base-side-exhaustion.sh new file mode 100755 index 00000000000000..0e12b5a0b06366 --- /dev/null +++ b/t/t6099-merge-base-side-exhaustion.sh @@ -0,0 +1,82 @@ +#!/bin/sh + +test_description='merge-base with ancestor among merge-base candidates + +Test that merge-base --all correctly handles cases where +multiple merge-base candidates exist and one is an ancestor +of another. The side-exhaustion optimization in +paint_down_to_common may exit before STALE propagation +removes the ancestor, but remove_redundant catches it. + +Graph shape (parents are below children): + + A ----- X + |\ /| + | B---/ | + | \ | + e2 \ f2 + | | | + e1 d1 f1 + \ | / + \ | / + \|/ + C + +A and X are the two tips. +B and C are both reachable from A and X. +B reaches C through d1. +Only B should appear in merge-base --all output. +' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh + +test_expect_success 'setup ancestor merge-base candidate' ' + test_commit C && + + git checkout -b d-chain HEAD && + test_commit d1 && + test_commit B && + + git checkout -b e-path C && + test_commit e1 && + test_commit e2 && + + git checkout -b f-path C && + test_commit f1 && + test_commit f2 && + + git checkout -b branch-A e-path && + test_merge A B && + + git checkout -b branch-X f-path && + test_merge X B && + + git commit-graph write --reachable +' + +test_expect_success 'merge-base --all excludes ancestor candidate' ' + git rev-parse B >expected && + git merge-base --all A X >actual && + test_cmp expected actual +' + +test_expect_success 'merge-base (single) finds shallowest' ' + git rev-parse B >expected && + git merge-base A X >actual && + test_cmp expected actual +' + +# Without commit-graph: generation numbers are INFINITY, +# side-exhaustion optimization does not fire. +test_expect_success 'merge-base --all without commit-graph' ' + rm -f .git/objects/info/commit-graph && + git rev-parse B >expected && + git merge-base --all A X >actual && + test_cmp expected actual +' + +test_done diff --git a/t/t6120-describe.sh b/t/t6120-describe.sh index 7a7c46658a3a81..2621edb5937ff0 100755 --- a/t/t6120-describe.sh +++ b/t/t6120-describe.sh @@ -1017,4 +1017,48 @@ do ' done input <<-\EOF && + third + second + first + EOF + git -C repo-format log --stdin --no-walk \ + "$opts" --format="$format" >expect actual expect && + test_must_fail git -C repo-format format-rev "$opts" \ + --stdin-mode=revs --format="$format" 2>actual && + test_cmp expect actual +} + +test_expect_success 'format-rev --color' ' + format_rev_cmp_log --color=always && + format_rev_cmp_log --color && + format_rev_cmp_log --no-color && + format_rev_err_cmp_log --color=not-valid +' + +test_expect_success 'format-rev --abbrev' ' + format_rev_cmp_log --abbrev=31 && + format_rev_cmp_log --no-abbrev +' + +test_expect_success 'format-rev --date' ' + format_rev_cmp_log --date=relative && + format_rev_cmp_log --date=iso-strict && + format_rev_err_cmp_log --date=not-valid +' + test_done diff --git a/t/t6600-test-reach.sh b/t/t6600-test-reach.sh index 019fed9f5ec0ea..13c84e0a423138 100755 --- a/t/t6600-test-reach.sh +++ b/t/t6600-test-reach.sh @@ -85,6 +85,103 @@ test_expect_success 'setup' ' git branch -f skew-P2 "$skew_P2" && git tag skew-M2 "$skew_M2" && + # Build a small side topology to exercise the (PARENT1|PARENT2) -> + # (PARENT1|PARENT2|STALE) transition in paint_down_to_common(); the + # 10x10 grid above does not exercise it because no merge-base candidate + # there is a descendant of another, so STALE never reaches a + # still-pending candidate. + # + # ps-X + # /|\ + # / | \ + # ps-Z ps-B ps-W + # | / \ | + # | / \ | + # |/ \| + # ps-T1 ps-T2 + # + # where ps-T1=merge(ps-Z,ps-B), ps-T2=merge(ps-W,ps-B), so + # merge-base(ps-T1,ps-T2) = ps-B. During the walk, ps-X transitions + # to (PARENT1|PARENT2) via ps-Z and ps-W before ps-B is dequeued; + # then the STALE-walk from ps-B transitions ps-X to + # (PARENT1|PARENT2|STALE). + git checkout --orphan ps-orphan && + test_commit ps-X && + git checkout -b ps-B-br ps-X && test_commit ps-B && + git checkout -b ps-Z-br ps-X && test_commit ps-Z && + git checkout -b ps-W-br ps-X && test_commit ps-W && + git checkout -b ps-T1 ps-Z && + git merge --no-ff -m ps-T1 ps-B && + git checkout -b ps-T2 ps-W && + git merge --no-ff -m ps-T2 ps-B && + + # Build a side topology that lives entirely outside the half + # commit-graph and has non-monotonic commit dates, to exercise the + # INFINITY-gate in paint_down_to_common. With both tips outside + # the graph, generation is INFINITY and the queue falls back to + # commit-date order, which here is non-monotonic. + # + # pi-X (date 500, PARENT1 tip) --> pi-P, pi-D + # pi-D (date 480) --> pi-C + # pi-C (date 200) --> pi-B + # pi-B (date 100, PARENT2 tip) --> pi-P + # pi-P (date 450, root) + # + # merge-base(pi-X, pi-B) = pi-B (it is an ancestor of pi-X and is + # itself one of the queried tips). + git checkout --orphan pi-orphan && + test_commit --date "@450 +0000" pi-P && + test_commit --date "@100 +0000" pi-B && + test_commit --date "@200 +0000" pi-C && + test_commit --date "@480 +0000" pi-D && + GIT_AUTHOR_DATE="@500 +0000" GIT_COMMITTER_DATE="@500 +0000" \ + git commit-tree -p pi-D -p pi-P -m pi-X pi-D^{tree} >pi-X-oid && + pi_x="$(cat pi-X-oid)" && + git branch -f pi-X-br "$pi_x" && + git tag pi-X "$pi_x" && + + # Clock-skew topology for side-exhaustion testing. + # D is the correct merge base but has a higher committer date + # than C (its child). With date ordering, D would be dequeued + # before C, causing side-exhaustion to fire too early. + # Generation ordering prevents this by visiting children + # before parents regardless of dates. + # + # se-A (date 7000) --> se-C (date 3000) --> se-D (date 5000) --> se-root (date 4000) + # se-B (date 6000) --> se-D + # + se_root=$(skew_commit 4000 se-root) && + se_D=$(skew_commit 5000 se-D -p "$se_root") && + se_C=$(skew_commit 3000 se-C -p "$se_D") && + se_A=$(skew_commit 7000 se-A -p "$se_C") && + se_B=$(skew_commit 6000 se-B -p "$se_D") && + git branch -f se-A "$se_A" && + git branch -f se-B "$se_B" && + git tag se-D "$se_D" && + + # Clock-skew topology with redundant ancestor for + # side-exhaustion testing. MB1 is the correct merge base; + # MB2 is its parent. A reaches MB2 via E (high date) and + # MB1 via C (low date). B reaches MB1 via D. With date + # ordering, side-exhaustion would fire before C is dequeued, + # missing MB1. Generation ordering ensures both are found. + # + # se2-A (date 8000) --> se2-C (date 2000) --> se2-MB1 (date 5000) --> se2-MB2 (date 4000) --> se2-root (date 1000) + # se2-A --> se2-E (date 6500) --> se2-MB2 + # se2-B (date 7000) --> se2-D (date 6000) --> se2-MB1 + # + se2_root=$(skew_commit 1000 se2-root) && + se2_MB2=$(skew_commit 4000 se2-MB2 -p "$se2_root") && + se2_MB1=$(skew_commit 5000 se2-MB1 -p "$se2_MB2") && + se2_C=$(skew_commit 2000 se2-C -p "$se2_MB1") && + se2_D=$(skew_commit 6000 se2-D -p "$se2_MB1") && + se2_E=$(skew_commit 6500 se2-E -p "$se2_MB2") && + se2_A=$(skew_commit 8000 se2-A -p "$se2_C" -p "$se2_E") && + se2_B=$(skew_commit 7000 se2-B -p "$se2_D") && + git branch -f se2-A "$se2_A" && + git branch -f se2-B "$se2_B" && + git tag se2-MB1 "$se2_MB1" && + git commit-graph write --reachable && mv .git/objects/info/commit-graph commit-graph-full && chmod u+w commit-graph-full && @@ -98,24 +195,34 @@ test_expect_success 'setup' ' ' run_all_modes () { - test_when_finished rm -rf .git/objects/info/commit-graph && - "$@" actual && - test_cmp expect actual && - cp commit-graph-full .git/objects/info/commit-graph && - "$@" actual && - test_cmp expect actual && - cp commit-graph-half .git/objects/info/commit-graph && - "$@" actual && - test_cmp expect actual && - cp commit-graph-no-gdat .git/objects/info/commit-graph && - "$@" actual && - test_cmp expect actual + graph=.git/objects/info/commit-graph && + test_when_finished rm -rf "$graph" "${graph}s" && + rm -f trace-mode-*.txt && + + for mode in none full half no-gdat + do + rm -rf "$graph" "${graph}s" && + cp "commit-graph-${mode}" "$graph" 2>/dev/null || + true && + GIT_TRACE2_EVENT="$(pwd)/trace-mode-${mode}.txt" \ + "$@" actual && + test_cmp expect actual || return 1 + done } test_all_modes () { run_all_modes test-tool reach "$@" } +test_paint_down_steps () { + for mode in none full half no-gdat + do + test_trace2_data_singular paint_down_to_common steps "$1" \ + "mode=$mode" <"trace-mode-${mode}.txt" || return 1 + shift + done +} + test_expect_success 'ref_newer:miss' ' cat >input <<-\EOF && A:commit-5-7 @@ -182,6 +289,17 @@ test_expect_success 'in_merge_bases_many:miss-heuristic' ' test_all_modes in_merge_bases_many ' +test_expect_success 'in_merge_bases_many:self' ' + cat >input <<-\EOF && + A:commit-6-8 + X:commit-5-9 + X:commit-6-8 + EOF + echo "in_merge_bases_many(A,X):1" >expect && + test_all_modes in_merge_bases_many && + test_paint_down_steps 45 1 25 1 +' + test_expect_success 'is_descendant_of:hit' ' cat >input <<-\EOF && A:commit-5-7 @@ -219,6 +337,105 @@ test_expect_success 'get_merge_bases_many' ' test_all_modes get_merge_bases_many ' +test_expect_success 'get_merge_bases_many:duplicate-twos' ' + cat >input <<-\EOF && + A:commit-5-7 + X:commit-4-8 + X:commit-4-8 + X:commit-6-6 + X:commit-6-6 + X:commit-8-3 + EOF + { + echo "get_merge_bases_many(A,X):" && + git rev-parse commit-5-6 \ + commit-4-7 | sort + } >expect && + test_all_modes get_merge_bases_many +' + +test_expect_success 'get_merge_bases_many:pending-stale' ' + # Exercises the (PARENT1|PARENT2) -> (...|STALE) transition path in + # paint_down_to_common(). See the topology comment in the setup test. + cat >input <<-\EOF && + A:ps-T1 + X:ps-T2 + EOF + { + echo "get_merge_bases_many(A,X):" && + git rev-parse ps-B + } >expect && + test_all_modes get_merge_bases_many && + test_paint_down_steps 5 5 5 5 +' + +test_expect_success 'get_merge_bases_many:infinity-both-sides' ' + # Exercises the push-time INFINITY-gate in paint_down_to_common(). See + # the pi-* topology comment in the setup test. + cat >input <<-\EOF && + A:pi-X + X:pi-B + EOF + { + echo "get_merge_bases_many(A,X):" && + git rev-parse pi-B + } >expect && + test_all_modes get_merge_bases_many && + test_paint_down_steps 5 4 5 4 +' + +test_expect_success 'setup mixed finite/INFINITY topology' ' + # Create a commit outside all saved commit-graph files so it always + # has INFINITY generation, while its parent (ps-X) is in the graph + # with a finite generation. Use the ps-* orphan topology so we do + # not pollute the grid-based rev-list tests. + git checkout ps-X && + test_env GIT_TEST_COMMIT_GRAPH= test_commit pm-INF +' + +test_expect_success 'get_merge_bases_many:mixed-finite-infinity' ' + # One tip (pm-INF) is outside the commit-graph with INFINITY + # generation; the other (ps-B) is in the graph with finite + # generation. The walk starts in the INFINITY region and crosses + # into the finite region where side-exhaustion can fire. + cat >input <<-\EOF && + A:pm-INF + X:ps-B + EOF + { + echo "get_merge_bases_many(A,X):" && + git rev-parse ps-X + } >expect && + test_all_modes get_merge_bases_many && + test_paint_down_steps 3 3 3 3 +' + +test_expect_success 'merge-base --all commit-walk steps' ' + >input && + git rev-parse commit-9-1 >expect && + run_all_modes git merge-base --all commit-9-9 commit-9-1 && + test_paint_down_steps 81 9 57 37 +' + +test_expect_success 'merge-base --all with clock skew (side-exhaustion)' ' + # Verify that the merge base is computed correctly even + # when commits have non-monotonic commit dates. + >input && + git rev-parse se-D >expect && + run_all_modes git merge-base --all se-A se-B && + test_paint_down_steps 6 4 6 4 +' + +test_expect_success 'merge-base --all with clock skew and redundant ancestor (side-exhaustion)' ' + # Verify that the correct merge base is found even when + # non-monotonic commit dates could cause a redundant + # ancestor to be visited first. + >input && + git rev-parse se2-MB1 >expect && + run_all_modes git merge-base --all se2-A se2-B && + test_paint_down_steps 8 6 8 6 +' + test_expect_success 'reduce_heads' ' cat >input <<-\EOF && X:commit-1-10 diff --git a/t/t7201-co.sh b/t/t7201-co.sh index 0ddd1ad7aab06a..f9696dab36618b 100755 --- a/t/t7201-co.sh +++ b/t/t7201-co.sh @@ -240,6 +240,14 @@ test_expect_success 'checkout -m creates a recoverable stash on conflict' ' test_grep "git stash drop" actual && test_grep "git stash pop" actual && test_grep "The following paths have local changes" actual && + sed -n "/apply the local changes later/,/Switched to branch/p" \ + actual >separator.actual && + cat >separator.expect <<-EOF && + apply the local changes later by running "git stash pop". + + Switched to branch ${SQ}side${SQ} + EOF + test_cmp separator.expect separator.actual && git log -p -1 --format="%gs%n%B" -g --diff-merges=1 refs/stash >actual && sed /^index/d actual >actual.trimmed && cat >expect <<-EOF && @@ -262,11 +270,18 @@ test_expect_success 'checkout -m creates a recoverable stash on conflict' ' git reset --hard ' -test_expect_success 'checkout -m which would overwrite untracked file' ' +test_expect_success 'checkout -m only retries untracked-file failure with local changes' ' git checkout -f --detach main && test_commit another-file && git checkout HEAD^ && >another-file.t && + test_must_fail env GIT_TRACE2_EVENT="$(pwd)/trace" \ + git checkout -m @{-1} 2>err && + test_grep "untracked working tree files" err && + grep "\"region_enter\".*\"category\":\"index\",\"label\":\"refresh\"" \ + trace >refresh.events && + test_line_count = 1 refresh.events && + fill 1 2 3 4 5 >one && test_must_fail git checkout -m @{-1} 2>err && q_to_tab >expect <<-\EOF && diff --git a/t/t7500-commit-template-squash-signoff.sh b/t/t7500-commit-template-squash-signoff.sh index 5e782ad8d798a0..de3221ea52ff5a 100755 --- a/t/t7500-commit-template-squash-signoff.sh +++ b/t/t7500-commit-template-squash-signoff.sh @@ -384,18 +384,24 @@ test_expect_success '--fixup=reword: ignores staged changes' ' test_cmp foo actual ' -test_expect_success '--fixup=reword: error out with -m option' ' +test_expect_success 'commit --fixup=reword: works with -m' ' commit_for_rebase_autosquash_setup && - echo "fatal: options '\''-m'\'' and '\''--fixup:reword'\'' cannot be used together" >expect && - test_must_fail git commit --fixup=reword:HEAD~ -m "reword commit message" 2>actual && - test_cmp expect actual + git commit --fixup=reword:HEAD~ -m "reword commit message" && + test_commit_message HEAD <<-EOF + amend! $(git log -1 --format=%s HEAD~2) + + reword commit message + EOF ' -test_expect_success '--fixup=amend: error out with -m option' ' +test_expect_success 'commit --fixup=amend: works with -m' ' commit_for_rebase_autosquash_setup && - echo "fatal: options '\''-m'\'' and '\''--fixup:amend'\'' cannot be used together" >expect && - test_must_fail git commit --fixup=amend:HEAD~ -m "amend commit message" 2>actual && - test_cmp expect actual + git commit --fixup=amend:HEAD~ -m "amend commit message" && + test_commit_message HEAD <<-EOF + amend! $(git log -1 --format=%s HEAD~2) + + amend commit message + EOF ' test_expect_success 'consecutive amend! commits remove amend! line from commit msg body' ' @@ -432,6 +438,13 @@ test_expect_success 'deny to create amend! commit if its commit msg body is empt test_cmp expected actual ' +test_expect_success 'deny to create amend! commit if -m is empty' ' + commit_for_rebase_autosquash_setup && + echo "Aborting commit due to empty commit message body." >expect && + test_must_fail git commit --fixup=amend:HEAD~ -m "" 2>actual && + test_cmp expect actual +' + test_expect_success 'amend! commit allows empty commit msg body with --allow-empty-message' ' commit_for_rebase_autosquash_setup && cat >expected <<-EOF && @@ -468,10 +481,82 @@ test_expect_success '--fixup=reword: give error with pathsec' ' test_cmp expect actual ' -test_expect_success '--fixup=reword: -F give error message' ' - echo "fatal: options '\''-F'\'' and '\''--fixup'\'' cannot be used together" >expect && - test_must_fail git commit --fixup=reword:HEAD~ -F msg 2>actual && - test_cmp expect actual +test_expect_success 'commit --fixup works with -F' ' + commit_for_rebase_autosquash_setup && + echo "message" >msgfile && + git commit --fixup HEAD~ -F msgfile && + test_commit_message HEAD <<-EOF + fixup! $(git log -1 --format=%s HEAD~2) + + message + EOF +' + +test_expect_success 'commit --fixup works with -C' ' + commit_for_rebase_autosquash_setup && + git commit --fixup HEAD~ -C HEAD && + test_commit_message HEAD <<-EOF + fixup! $(git log -1 --format=%s HEAD~2) + + $(get_commit_msg HEAD~) + EOF +' + +test_expect_success 'commit --fixup=amend: works with -c' ' + commit_for_rebase_autosquash_setup && + test_set_editor : && + git commit --fixup=amend:HEAD -c HEAD~ && + test_commit_message HEAD <<-EOF + amend! intermediate commit + + target message subject line + + target message body line 1 + target message body line 2 + EOF +' + +test_expect_success 'commit --fixup=amend:HEAD with -C HEAD and without have the same message' ' + commit_for_rebase_autosquash_setup && + start=$(git rev-parse HEAD) && + + git commit --fixup=amend:HEAD -C HEAD && + git commit --fixup=amend:HEAD -C HEAD && + git log -1 --pretty=%B >with-c && + + git reset --hard "$start" && + test_set_editor : && + git commit --fixup=amend:HEAD && + git commit --fixup=amend:HEAD && + git log -1 --pretty=%B >without-c && + + test_cmp with-c without-c +' + +test_expect_success 'commit --fixup=amend: with -C copies full subject + body of squash commit' ' + commit_for_rebase_autosquash_setup && + git commit --squash HEAD~ -m "inner body" && + echo "extra" >>foo && + git add foo && + git commit --fixup=amend:HEAD -C HEAD && + test_commit_message HEAD <<-EOF + amend! squash! $(git log -1 --format=%s HEAD~3) + + squash! $(git log -1 --format=%s HEAD~3) + + inner body + EOF +' + +test_expect_success 'commit --fixup=reword: works with -F' ' + commit_for_rebase_autosquash_setup && + echo "message from file" >msgfile && + git commit --fixup=reword:HEAD~ -F msgfile && + test_commit_message HEAD <<-EOF + amend! $(git log -1 --format=%s HEAD~2) + + $(cat msgfile) + EOF ' test_expect_success 'commit --squash works with -F' ' @@ -524,10 +609,7 @@ test_expect_success 'invalid message options when using --fixup' ' echo changes >>foo && echo "message" >log && git add foo && - test_must_fail git commit --fixup HEAD~1 --squash HEAD~2 && - test_must_fail git commit --fixup HEAD~1 -C HEAD~2 && - test_must_fail git commit --fixup HEAD~1 -c HEAD~2 && - test_must_fail git commit --fixup HEAD~1 -F log + test_must_fail git commit --fixup HEAD~1 --squash HEAD~2 ' cat >expected-template <expect <<-\EOF && + https: //www.a-trailer.org + https: //www.another-trailer.org + Signed-off-by: somebody + EOF + git interpret-trailers --only-trailers >actual <<-\EOF && + subject + + body + + https://www.not-a-trailer.org + https ://www.a-trailer.org + https: //www.another-trailer.org + Signed-off-by: somebody + EOF + test_cmp expect actual +' + test_done diff --git a/t/t7706-repack-drop-filtered.sh b/t/t7706-repack-drop-filtered.sh new file mode 100755 index 00000000000000..cb361158349c13 --- /dev/null +++ b/t/t7706-repack-drop-filtered.sh @@ -0,0 +1,185 @@ +#!/bin/sh + +test_description='git repack --drop-filtered enumerates filtered promisor blobs' + +. ./test-lib.sh + +# Delete a loose or packed object from "repo". +delete_object () { + local repo="$1" && + local obj="$2" && + local path="$repo/.git/objects/$(test_oid_to_path "$obj")" && + rm "$path" +} + +# Pack the objects into a promisor pack inside "repo". It is a pack +# accompanied by an empty ".promisor" marker file. Objects +# in such a pack are treated as recoverable from the promisor remote. +pack_as_from_promisor () { + HASH=$(git -C repo pack-objects .git/objects/pack/pack) && + >repo/.git/objects/pack/pack-$HASH.promisor && + echo $HASH +} + +# Write a blob of $1 bytes into "repo", record it as coming from the +# promisor remote, and remove the loose copy so the object is only +# present in the promisor pack. +promisor_blob () { + test-tool genrandom "$1" "$2" >blob_content && + OID=$(git -C repo hash-object -w --stdin /dev/null && + delete_object repo "$OID" && + echo "$OID" +} + +# Check option validation before any promisor walk +test_expect_success 'setup plain repo for validation' ' + git init plain && + test_commit -C plain initial && + git clone --bare plain plain.git && + git -C plain.git repack -a -d +' + +test_expect_success '--drop-filtered requires --filter' ' + test_must_fail git -C plain.git repack --drop-filtered --dry-run -a 2>err && + test_grep "drop-filtered requires --filter" err +' + +test_expect_success '--drop-filtered cannot be used with --filter-to' ' + test_must_fail git -C plain.git repack --drop-filtered \ + --filter=blob:limit=1k --filter-to=./filter-out 2>err && + test_grep "options .--drop-filtered. and .--filter-to. cannot be used together" err +' + +test_expect_success '--dry-run only takes effect with --drop-filtered' ' + test_must_fail git -C plain.git repack --dry-run 2>err && + test_grep "dry-run only takes effect with --drop-filtered" err +' + +test_expect_success '--drop-filtered requires -a' ' + test_must_fail git -C plain.git repack --drop-filtered \ + --filter=blob:limit=1k --dry-run 2>err && + test_grep "drop-filtered requires -a" err +' + +test_expect_success '--drop-filtered fails with --write-bitmap-index' ' + test_must_fail git -C plain.git repack --drop-filtered \ + --filter=blob:limit=1k --dry-run -a -b 2>err && + test_grep "options .--drop-filtered. and .--write-bitmap-index. cannot be used together" err +' + +test_expect_success '--drop-filtered rejects explicit -b even when repack.writeBitmaps=true' ' + test_must_fail git -C plain.git -c repack.writeBitmaps=true \ + repack --drop-filtered --filter=blob:limit=1k --dry-run -a -b 2>err && + test_grep "options .--drop-filtered. and .--write-bitmap-index. cannot be used together" err +' + +test_expect_success '--drop-filtered fails without a promisor remote' ' + test_must_fail git -C plain.git repack --drop-filtered \ + --filter=blob:limit=1k --dry-run -a 2>err && + test_grep "drop-filtered requires a promisor remote" err +' + +# Enumeration tests using promisor pack +test_expect_success 'setup repo with a promisor remote' ' + rm -rf repo && + test_create_repo repo && + test_commit -C repo base && + + # Mark the repo as a partial clone with a promisor remote so the + # promisor walk and the safety guard are satisfied. + git -C repo config core.repositoryformatversion 1 && + git -C repo config extensions.partialclone origin && + git -C repo config remote.origin.promisor true && + git -C repo config remote.origin.url "." && + + BIG=$(promisor_blob big 3072) && + SMALL=$(promisor_blob small 512) && + echo "$BIG" >big_oid && + echo "$SMALL" >small_oid +' + +test_expect_success 'promisor blob over the threshold is listed' ' + BIG=$(cat big_oid) && + SMALL=$(cat small_oid) && + + git -C repo -c repack.writeBitmaps=false \ + repack --drop-filtered --filter=blob:limit=1k --dry-run -a >out && + + test_grep "$BIG" out && + test_grep ! "$SMALL" out +' + +test_expect_success 'locally created blob is never listed' ' + BIG=$(cat big_oid) && + + # Large blob that exists only locally must never be a drop candidate. + # Dropping it would be unrecoverable. + test-tool genrandom local 4096 >local_content && + LOCAL=$(git -C repo hash-object -w --stdin out && + + test_grep "$BIG" out && + test_grep ! "$LOCAL" out +' + +test_expect_success '--dry-run does not remove the filtered objects' ' + BIG=$(cat big_oid) && + + git -C repo -c repack.writeBitmaps=false \ + repack --drop-filtered --filter=blob:limit=1k --dry-run -a >out && + + # Candidate blob must still be present after a dry run. + git -C repo cat-file -e "$BIG" +' + +test_expect_success '--drop-filtered removes the promisor blob locally' ' + BIG=$(cat big_oid) && + SMALL=$(cat small_oid) && + + git -C repo -c repack.writeBitmaps=false \ + repack --drop-filtered --filter=blob:limit=1k -a && + + git -C repo cat-file --batch-all-objects --batch-check="%(objectname)" >present && + test_grep ! "$BIG" present && + test_grep "$SMALL" present +' + +test_expect_success '--drop-filtered refuses when a merge is in progress' ' + test_when_finished "git -C repo merge --abort || :" && + + # Create a conflicting merge so wt_status reports it. + git -C repo checkout -B mergebase base && + echo one >repo/conflict.txt && + git -C repo add conflict.txt && + git -C repo commit -m one && + + git -C repo checkout -B mergeother base && + echo two >repo/conflict.txt && + git -C repo add conflict.txt && + git -C repo commit -m two && + + test_must_fail git -C repo merge mergebase && + + test_must_fail git -C repo -c repack.writeBitmaps=false \ + repack --drop-filtered --filter=blob:limit=1k --dry-run -a 2>err && + test_grep "in progress" err +' + +test_expect_success '--drop-filtered refuses to drop an index-referenced blob' ' + # Create a large blob, add it to the index and make it a promisor object + # so the index references it and enumeration picks it up. + test-tool genrandom idx 4096 >repo/tracked-big.bin && + git -C repo add tracked-big.bin && + OID=$(git -C repo rev-parse :tracked-big.bin) && + printf "%s\n" "$OID" | pack_as_from_promisor >/dev/null && + delete_object repo "$OID" && + + test_must_fail git -C repo -c repack.writeBitmaps=false \ + repack --drop-filtered --filter=blob:limit=1k --dry-run -a 2>err && + test_grep "referenced by the current index" err +' + +test_done diff --git a/t/t7810-grep.sh b/t/t7810-grep.sh index ac7ea9cf8d0268..ffa5f350250d0f 100755 --- a/t/t7810-grep.sh +++ b/t/t7810-grep.sh @@ -89,6 +89,10 @@ test_expect_success setup ' function dummy() {} EOF printf "\200\nASCII\n" >invalid-utf8 && + printf "before\346world\n" >invalid-utf8-embedded && + printf "a\346b\347c\n" >invalid-utf8-multi && + printf "\346world\n" >invalid-utf8-leading && + printf "before\346\n" >invalid-utf8-trailing && if test_have_prereq FUNNYNAMES then echo unusual >"\"unusual\" pathname" && @@ -595,6 +599,39 @@ test_expect_success MB_REGEX 'grep two chars in single-char multibyte file' ' LC_ALL=en_US.UTF-8 test_expect_code 1 git grep ".." reverse-question-mark ' +test_expect_success MACOS,MB_REGEX 'grep matches valid text on both sides of invalid UTF-8' ' + LC_ALL=en_US.UTF-8 git grep -h "befo[r]e" invalid-utf8-embedded >actual && + test_cmp invalid-utf8-embedded actual && + LC_ALL=en_US.UTF-8 git grep -h "worl[d]" invalid-utf8-embedded >actual && + test_cmp invalid-utf8-embedded actual && + LC_ALL=en_US.UTF-8 git grep -h -o "worl[d]" invalid-utf8-embedded >actual && + echo world >expected && + test_cmp expected actual +' + +test_expect_success MACOS,MB_REGEX 'grep matches a run between two invalid sequences' ' + LC_ALL=en_US.UTF-8 git grep -h "[b]" invalid-utf8-multi >actual && + test_cmp invalid-utf8-multi actual +' + +test_expect_success MB_REGEX 'grep does not anchor ^ or $ inside an invalid-byte line' ' + test_expect_code 1 env LC_ALL=en_US.UTF-8 \ + git grep -h "^world" invalid-utf8-embedded && + test_expect_code 1 env LC_ALL=en_US.UTF-8 \ + git grep -h "before\$" invalid-utf8-embedded +' + +test_expect_success MACOS,MB_REGEX 'grep anchors ^ and $ at true line ends past invalid UTF-8' ' + LC_ALL=en_US.UTF-8 git grep -h "^before" invalid-utf8-embedded >actual && + test_cmp invalid-utf8-embedded actual && + LC_ALL=en_US.UTF-8 git grep -h "world\$" invalid-utf8-embedded >actual && + test_cmp invalid-utf8-embedded actual && + LC_ALL=en_US.UTF-8 git grep -h "^" invalid-utf8-leading >actual && + test_cmp invalid-utf8-leading actual && + LC_ALL=en_US.UTF-8 git grep -h "\$" invalid-utf8-trailing >actual && + test_cmp invalid-utf8-trailing actual +' + cat >expected <expect && + git -c core.commitGraph=true last-modified -t HEAD -- d/a \ + >actual && + + test_cmp expect actual + ) +' + test_expect_success 'cannot run last-modified on two commits' ' test_must_fail git last-modified HEAD HEAD~1 2>err && test_grep "last-modified can only operate on one commit at a time" err diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh index e9d814a34aa86c..d1393ef1978761 100755 --- a/t/t9001-send-email.sh +++ b/t/t9001-send-email.sh @@ -1422,6 +1422,21 @@ test_expect_success $PREREQ 'detects ambiguous reference/file conflict' ' test_grep disambiguate errors ' +test_expect_success $PREREQ 'missing subject omits Perl location' ' + cat >no-subject.patch <<-\EOF && + This is the body. + EOF + test_must_fail git send-email \ + --dry-run \ + --from="Example " \ + --to=nobody@example.com \ + no-subject.patch 2>actual && + cat >expect <<-\EOF && + No '\''Subject:'\'' line in '\''no-subject.patch'\'' + EOF + test_cmp expect actual +' + test_expect_success $PREREQ 'feed two files' ' rm -fr outdir && git format-patch -2 -o outdir && diff --git a/t/t9300-fast-import.sh b/t/t9300-fast-import.sh index fe6c2617acb2fe..d9de2ef0d88bc8 100755 --- a/t/t9300-fast-import.sh +++ b/t/t9300-fast-import.sh @@ -2827,6 +2827,13 @@ test_expect_success 'R: unknown commandline options are rejected' '\ test_must_fail git fast-import --non-existing-option < /dev/null ' +test_expect_success 'R: feature-only names are rejected on the command line' ' + for opt in --alias --get-mark --ls --notes + do + test_must_fail git fast-import "$opt" file1 && echo more >file2 && git add file1 file2 && + echo untracked >file3 && + echo untracked >ufile && git commit -m one && git branch mybranch && git tag mytag @@ -2712,6 +2714,118 @@ test_expect_success 'git -C checkout uses the right repo' ' EOF ' +test_expect_success 'git checkout completes tracked paths when no refs match' ' + # file1 and file2 are tracked but file3 is not + # there is no ref that begins with f + test_completion "git checkout f" <<-\EOF && + file1 + file2 + EOF + test_completion "git checkout -- f" <<-\EOF + file1 + file2 + EOF +' + +test_expect_success 'git checkout completes untracked paths, too' ' + # ufile is not tracked and there is no ref that begins with u + test_completion "git checkout u" <<-\EOF && + ufile + EOF + test_completion "git checkout -- u" <<-\EOF + ufile + EOF +' + +test_expect_success 'git -C checkout completes paths in specified repo' ' + # otherfile is tracked, oops is not + # lostfile is tracked but lost, ufile is untracked. + test_when_finished "rm -rf repo-for-checkout" && + git init repo-for-checkout && + echo content >repo-for-checkout/otherfile && + echo content >repo-for-checkout/lostfile && + git -C repo-for-checkout add otherfile && + git -C repo-for-checkout add lostfile && + git -C repo-for-checkout commit -m otherfile && + echo untracked >repo-for-checkout/oops && + echo untracked >repo-for-checkout/ufile && + rm -f repo-for-checkout/lostfile && + test_completion "git -C repo-for-checkout checkout o" <<-\EOF && + otherfile + EOF + test_completion "git -C repo-for-checkout checkout -- o" <<-\EOF && + otherfile + EOF + test_completion "git -C repo-for-checkout checkout l" <<-\EOF && + lostfile + EOF + test_completion "git -C repo-for-checkout checkout -- l" <<-\EOF && + lostfile + EOF + test_completion "git -C repo-for-checkout checkout u" <<-\EOF && + ufile + EOF + test_completion "git -C repo-for-checkout checkout -- u" <<-\EOF + ufile + EOF +' + +test_expect_success 'git diff completes tracked paths when no refs match' ' + # file1 and file2 are tracked but file3 is not + # there is no ref that begins with f + test_completion "git diff f" <<-\EOF && + file1 + file2 + EOF + test_completion "git diff -- f" <<-\EOF + file1 + file2 + EOF +' + +test_expect_success 'git diff [--] completes untracked paths, too' ' + # ufile is not tracked and there is no ref that begins with u + test_completion "git diff u" <<-\EOF && + ufile + EOF + test_completion "git diff -- u" <<-\EOF + ufile + EOF +' + +test_expect_success 'git -C diff completes paths in specified repo' ' + test_when_finished "rm -rf repo-for-diff" && + git init repo-for-diff && + echo content >repo-for-diff/otherfile && + echo content >repo-for-diff/lostfile && + git -C repo-for-diff add otherfile && + git -C repo-for-diff add lostfile && + git -C repo-for-diff commit -m otherfile && + echo untracked >repo-for-diff/oops && + echo untracked >repo-for-diff/ufile && + rm -f repo-for-diff/lostfile && + + test_completion "git -C repo-for-diff diff o" <<-\EOF && + otherfile + EOF + test_completion "git -C repo-for-diff diff l" <<-\EOF && + lostfile + EOF + test_completion "git -C repo-for-diff diff u" <<-\EOF && + ufile + EOF + + test_completion "git -C repo-for-diff diff -- o" <<-\EOF && + otherfile + EOF + test_completion "git -C repo-for-diff diff -- l" <<-\EOF && + lostfile + EOF + test_completion "git -C repo-for-diff diff -- u" <<-\EOF + ufile + EOF +' + test_expect_success 'show completes all refs' ' test_completion "git show m" <<-\EOF main Z @@ -3107,6 +3221,57 @@ test_expect_success 'git clone --config= - value' ' EOF ' +test_expect_success 'git history subcommands' ' + test_completion "git history " <<-\EOF && + drop Z + fixup Z + reword Z + split Z + squash Z + EOF + test_completion "git history --" "" +' + +test_expect_success 'git history subcommand options' ' + test_completion "git history split main --" <<-\EOF && + --update-refs=Z + --dry-run Z + --no-dry-run Z + EOF + test_completion "git history fixup --upd" "--update-refs=" && + test_completion "git history fixup --ree" "--reedit-message " && + test_completion "git history split --upd" "--update-refs=" && + test_completion "git history split main --dry" "--dry-run " && + test_completion "git history reword main -- --d" "" && + test_completion "git history fixup --empty=ke" "keep " && + test_completion "git history fixup --empty=drop" "drop " && + test_completion "git history drop --empty=ab" "abort " && + test_completion "git history reword --empty=ke" "" && + test_completion "git history fixup --update-refs=branch" "branches " && + test_completion "git history split --update-refs=he" "head " && + test_completion "git history reword main -- --update-refs=he" "" +' + +test_expect_success 'git history revisions' ' + test_completion "git history split ma" "main " && + test_completion "git history split --update-refs=head ma" "main " && + test_completion "git history fixup --empty=drop ma" "main " && + test_completion "git history reword main m" "" +' + +test_expect_success 'git history split pathspecs' ' + test_completion "git history split main -- --update-refs=h" "" && + test_completion "git history split main -- --update-refs h" "" && + test_completion "git history split --dry-run main file" <<-\EOF && + file1Z + file2Z + EOF + test_completion "git history split main -- file" <<-\EOF + file1Z + file2Z + EOF +' + test_expect_success 'git reflog show' ' test_when_finished "git checkout - && git branch -d shown" && git checkout -b shown && diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 03bf31d8efea30..ed9779b0f73310 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -2004,6 +2004,41 @@ test_trace2_data () { grep -e '"category":"'"$1"'","key":"'"$2"'","value":"'"$3"'"' } +# Check that the given trace2 data event has the expected value and +# appears exactly once. Produces a diagnostic on failure. +# +# test_trace2_data_singular [