From 7566334e873384babff43200d386974720bdf8d6 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Thu, 6 Aug 2026 13:48:38 -0400 Subject: [PATCH 01/11] Fix intermittent connection resets on scan comparison by polling the diff-scans endpoints (#284) * Poll diff-scans endpoints for scan comparison instead of streaming The scan comparison (fullscans.stream_diff) held a single HTTP connection open, fully idle, while the API computed the diff. Network middleboxes with TCP idle timeouts - notably Azure NAT gateways, which default to 4 minutes - kill that connection with a RST, surfacing as intermittent "Connection reset by peer" / blank "API Error:" failures on the final comparison step of long scans (CE-354). The comparison now creates a diff-scan resource (POST /orgs/{org}/diff-scans/from-ids) and polls GET /orgs/{org}/diff-scans/{id}?cached=true with short bounded requests: 202 while the diff is computing, 200 with the result once ready. No request is ever idle long enough to be reaped, and the poll interval backs off 5s -> 30s to stay quota-friendly (each poll costs 1 quota unit). Transient poll failures retry; a 30-minute backstop guards against a diff scan that never completes. Any failure of the new flow (e.g. org tokens missing the diff-scans:create / diff-scans:list / full-scans:list scopes) logs a warning and falls back to the legacy streaming comparison, so the change is transparent to existing users. Requires socketdev>=3.4.0 for diffscans.get query-param/202 support. Co-Authored-By: Claude Fable 5 * Drop ignored omit_license_details param from cached diff-scan polls The API ignores omit_license_details when cached=true - cached diff-scan results always embed license details - so sending the param suggested a lean-response guarantee the polling path doesn't have. Document the caveat instead: if the heavier payload ever gets truncated on a huge dependency tree, JSON parsing fails and the caller already falls back to the legacy streaming comparison, which still requests the lean payload. include_license_details now only governs that fallback call. Flagged by Cursor Bugbot on #284. Co-Authored-By: Claude Fable 5 * Keep duplicate diff scans on cached polling * Require bundled socketdev 3.4.2 release * Stage CLI 2.6.1 * Require socketdev 3.5.0 * Drop ticket references from code comments, workflows, and changelog Co-Authored-By: Claude Fable 5 Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> * Align changelog with pinned SDK dependency --------- Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- CHANGELOG.md | 22 +++ pyproject.toml | 2 +- socketsecurity/__init__.py | 2 +- socketsecurity/core/__init__.py | 213 +++++++++++++++++++++++---- tests/core/conftest.py | 21 +++ tests/core/test_diff_scan_polling.py | 122 +++++++++++++++ tests/core/test_sdk_methods.py | 38 +++-- uv.lock | 2 +- 8 files changed, 377 insertions(+), 45 deletions(-) create mode 100644 tests/core/test_diff_scan_polling.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 202c6ed2..22976e66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 2.6.1 + +### Changed: scan comparison now polls the diff-scans endpoints + +- Diff mode no longer holds a single idle HTTP connection open while the API + computes the scan comparison. The CLI now creates a diff-scan resource + (`POST /orgs/{org}/diff-scans/from-ids`) and polls + `GET /orgs/{org}/diff-scans/{id}?cached=true` with short, bounded requests + until the comparison is ready (HTTP 200 instead of 202). This fixes + intermittent `Connection reset by peer` failures on the final comparison + step when scans take several minutes to compare and network middleboxes + (e.g. Azure NAT gateways, which default to a 4-minute TCP idle timeout) + reap the idle connection. +- Duplicate scan pairs are resolved after an HTTP 409 and then polled through + the same cached endpoint. This avoids automatically following the API's 302 + duplicate redirect with an uncached, potentially long-lived GET request. +- The change is transparent: no flags or workflow changes are needed. If the + org API token is missing the `diff-scans:create`, `diff-scans:list` or + `full-scans:list` scopes — or the new flow fails for any other reason — the + CLI logs a warning and falls back to the legacy streaming comparison. +- Requires the pinned `socketdev==3.5.0` SDK. + ## 2.6.0 ### Changed: pin all Python dependencies diff --git a/pyproject.toml b/pyproject.toml index 1b49c2ca..c70b629b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.6.0" +version = "2.6.1" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 2a2ecb9c..312c5053 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,3 +1,3 @@ __author__ = 'socket.dev' -__version__ = '2.6.0' +__version__ = '2.6.1' USER_AGENT = f'SocketPythonCLI/{__version__}' diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index a372de4d..7bd4a33e 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -15,7 +15,7 @@ from socketsecurity.config import CliConfig from socketdev import socketdev from socketdev.exceptions import APIFailure -from socketdev.fullscans import FullScanParams, SocketArtifact +from socketdev.fullscans import DiffArtifacts, FullScanParams, SocketArtifact from socketdev.org import Organization from socketdev.repos import RepositoryInfo import copy @@ -92,6 +92,25 @@ FULL_SCAN_UPLOAD_MAX_ATTEMPTS = len(FULL_SCAN_UPLOAD_BACKOFF_SCHEDULE_SECONDS) FULL_SCAN_UPLOAD_BACKOFF_JITTER_SECONDS = 2.0 +# Diff-scan polling policy. The legacy scan comparison (fullscans.stream_diff) holds a +# single HTTP connection open, fully idle, while the backend computes the diff; network +# middleboxes with TCP idle timeouts (notably Azure NAT gateways, which default to +# 4 minutes) kill that connection with a RST, surfacing as an intermittent +# ConnectionResetError on large scans. The diff-scans flow instead creates a +# diff-scan resource and polls its cached endpoint with short bounded requests: the API +# answers 202 while the comparison is still computing and 200 with the result once it is +# ready, so no connection is ever idle long enough to be reaped. +# +# Each poll consumes 1 unit of API quota, so the interval backs off toward +# DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS to stay quota-friendly on comparisons that take +# minutes to compute. The timeout is a backstop against a diff scan that never +# completes; on expiry (or any other failure of this flow) the caller falls back to the +# legacy streaming comparison rather than failing the scan outright. +DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS = 5.0 +DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS = 30.0 +DIFF_SCAN_POLL_BACKOFF_MULTIPLIER = 1.5 +DIFF_SCAN_POLL_TIMEOUT_SECONDS = 30 * 60.0 + def _humanize_alert_type(alert_type: str) -> str: """Convert a camelCase/PascalCase alert type into a Title-Cased label. @@ -1303,6 +1322,120 @@ def get_license_text_via_purl(self, packages: dict[str, Package], batch_size: in return packages + def get_diff_scan_artifacts( + self, + head_full_scan_id: str, + new_full_scan_id: str + ) -> DiffArtifacts: + """Compare two full scans via the diff-scans endpoints, polling for the result. + + Creates a diff-scan resource from the two full scan IDs, then polls + ``GET /orgs/{org}/diff-scans/{id}?cached=true`` until the API returns the + computed comparison (200) instead of a processing status (202). Unlike the + legacy ``fullscans.stream_diff`` call, no request is ever left idle while + the backend computes, so the comparison survives network idle timeouts. + See the DIFF_SCAN_POLL_* constants for the polling policy. + + Requires an org token with the ``diff-scans:create``, ``diff-scans:list`` + and ``full-scans:list`` scopes; callers are expected to catch failures and + fall back to the legacy streaming comparison. + + Note that cached diff-scan responses always embed per-package license + details (the API ignores ``omit_license_details`` when ``cached=true``), + so unlike the legacy streaming comparison there is no lean-response + option here; see the comment on ``poll_params`` below. + + Args: + head_full_scan_id: The before/base full scan ID + new_full_scan_id: The after/head full scan ID + + Returns: + DiffArtifacts with the added/removed/unchanged/replaced/updated lists + """ + create_params = { + "before": head_full_scan_id, + "after": new_full_scan_id, + "description": f"Socket Security CLI v{__version__} scan comparison", + } + try: + result = self.sdk.diffscans.create_from_ids(self.config.org_slug, create_params) + diff_scan = result.get("diff_scan") or {} + response_summary = result + except APIFailure as error: + if error.status_code != 409: + raise + + # Do not use on_duplicate=redirect here. The SDK follows that 302 + # automatically with a GET that lacks cached=true, which can leave + # the connection idle while an existing diff scan is still computing. + # Resolve the duplicate resource explicitly so every result fetch + # continues through the bounded cached polling path below. + existing = self.sdk.diffscans.list( + self.config.org_slug, + params={ + "before_full_scan_id": head_full_scan_id, + "after_full_scan_id": new_full_scan_id, + "per_page": 1, + }, + ) + matches = existing.get("results") or [] + diff_scan = matches[0] if matches else {} + response_summary = existing + + diff_scan_id = diff_scan.get("id") + if not diff_scan_id: + raise Exception( + "Error creating or resolving diff scan: " + f"unexpected response: {str(response_summary)[:500]}" + ) + artifacts_dict = diff_scan.get("artifacts") + + # cached=true is the polling contract (202 while computing, 200 when + # ready). The API ignores omit_license_details when cached=true - cached + # results always embed license details - so there is no lean-response + # option on this path (unlike stream_diff with + # include_license_details=false, the lean-payload mitigation). If that extra + # payload ever gets a response truncated on a huge dependency tree, + # response.json() fails and the caller falls back to the legacy + # streaming comparison, which still requests the lean payload. + poll_params = {"cached": "true"} + deadline = time.monotonic() + DIFF_SCAN_POLL_TIMEOUT_SECONDS + interval = DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS + while artifacts_dict is None: + try: + response = self.sdk.diffscans.get(self.config.org_slug, diff_scan_id, params=poll_params) + except APIFailure as error: + if not error.is_transient_error(): + raise + # A dropped/timed-out poll is retryable: the diff scan keeps + # computing server-side regardless of what happens to any one poll. + log.warning( + f"Transient error polling diff scan {diff_scan_id} " + f"({type(error).__name__}), retrying in {interval:.0f}s" + ) + response = {"status": "processing"} + if response.get("status") != "processing": + scan = response.get("diff_scan") or {} + if scan.get("artifacts") is None: + raise Exception( + f"Error fetching diff scan {diff_scan_id}: unexpected response: {str(response)[:500]}" + ) + artifacts_dict = scan["artifacts"] + break + if time.monotonic() >= deadline: + raise Exception( + f"Timed out waiting for diff scan {diff_scan_id} after " + f"{DIFF_SCAN_POLL_TIMEOUT_SECONDS:.0f} seconds" + ) + log.debug(f"Diff scan {diff_scan_id} still processing, polling again in {interval:.0f}s") + time.sleep(interval) + interval = min(interval * DIFF_SCAN_POLL_BACKOFF_MULTIPLIER, DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS) + + return DiffArtifacts.from_dict({ + key: artifacts_dict.get(key) or [] + for key in ("added", "removed", "unchanged", "replaced", "updated") + }) + def get_added_and_removed_packages( self, head_full_scan_id: str, @@ -1315,8 +1448,12 @@ def get_added_and_removed_packages( Args: head_full_scan_id: Previous scan (maybe None if first scan) new_full_scan_id: New scan just created - include_license_details: Whether to ask the diff endpoint to embed - per-package license attribution/details in the response. + include_license_details: Whether to ask the *legacy streaming* diff + endpoint to embed per-package license attribution/details in the + response. Only consulted on the fallback path: the primary + diff-scans path always receives embedded license details, since + the API ignores ``omit_license_details`` for cached reads (see + get_diff_scan_artifacts). Defaults to ``False`` on purpose. The diff endpoint exists to compare alerts between two scans; the license fields it can embed @@ -1343,39 +1480,55 @@ def get_added_and_removed_packages( log.info(f"Comparing scans - Head scan ID: {head_full_scan_id}, New scan ID: {new_full_scan_id}") diff_start = time.time() + diff_artifacts = None try: - diff_report = ( - self.sdk.fullscans.stream_diff( - self.config.org_slug, - head_full_scan_id, - new_full_scan_id, - use_types=True, - include_license_details=str(include_license_details).lower() - ).data + diff_artifacts = self.get_diff_scan_artifacts( + head_full_scan_id, + new_full_scan_id ) - except APIFailure as e: - log.error(f"API Error: {e}") - if self.cli_config and self.cli_config.disable_blocking: - sys.exit(0) - sys.exit(1) - except Exception as e: - import traceback - log.error(f"Error getting diff report: {str(e)}") - log.error(f"Stack trace:\n{traceback.format_exc()}") - raise + except Exception as error: + # SDK error messages can span many lines (path + response headers); the + # first line carries the status, which is all the warning needs. + error_summary = str(error).strip().splitlines()[0] if str(error).strip() else "" + log.warning( + f"Diff scan comparison failed with {type(error).__name__}({error_summary}), " + "falling back to the streaming scan comparison" + ) + + if diff_artifacts is None: + try: + diff_artifacts = ( + self.sdk.fullscans.stream_diff( + self.config.org_slug, + head_full_scan_id, + new_full_scan_id, + use_types=True, + include_license_details=str(include_license_details).lower() + ).data.artifacts + ) + except APIFailure as e: + log.error(f"API Error: {e}") + if self.cli_config and self.cli_config.disable_blocking: + sys.exit(0) + sys.exit(1) + except Exception as e: + import traceback + log.error(f"Error getting diff report: {str(e)}") + log.error(f"Stack trace:\n{traceback.format_exc()}") + raise diff_end = time.time() log.info(f"Diff Report Gathered in {diff_end - diff_start:.2f} seconds") log.info("Diff report artifact counts:") - log.info(f"Added: {len(diff_report.artifacts.added)}") - log.info(f"Removed: {len(diff_report.artifacts.removed)}") - log.info(f"Unchanged: {len(diff_report.artifacts.unchanged)}") - log.info(f"Replaced: {len(diff_report.artifacts.replaced)}") - log.info(f"Updated: {len(diff_report.artifacts.updated)}") - - added_artifacts = diff_report.artifacts.added + diff_report.artifacts.updated - removed_artifacts = diff_report.artifacts.removed + diff_report.artifacts.replaced - unchanged_artifacts = diff_report.artifacts.unchanged + log.info(f"Added: {len(diff_artifacts.added)}") + log.info(f"Removed: {len(diff_artifacts.removed)}") + log.info(f"Unchanged: {len(diff_artifacts.unchanged)}") + log.info(f"Replaced: {len(diff_artifacts.replaced)}") + log.info(f"Updated: {len(diff_artifacts.updated)}") + + added_artifacts = diff_artifacts.added + diff_artifacts.updated + removed_artifacts = diff_artifacts.removed + diff_artifacts.replaced + unchanged_artifacts = diff_artifacts.unchanged added_packages: Dict[str, Package] = {} removed_packages: Dict[str, Package] = {} diff --git a/tests/core/conftest.py b/tests/core/conftest.py index 381c2c3f..ae6b10c0 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -87,6 +87,22 @@ def stream_diff_response(data_dir, load_json): }) +@pytest.fixture +def diff_scan_get_response(data_dir, load_json): + """GET /orgs/{org}/diff-scans/{id} response built from the stream_diff fixture. + + The diff-scans endpoint returns the same artifact shape as the legacy + streaming diff, wrapped in a diff_scan object. + """ + json_data = load_json(data_dir / "fullscans" / "diff" / "stream_diff.json") + return { + "diff_scan": { + "id": "diff-scan-123", + "artifacts": json_data["data"]["artifacts"], + } + } + + @@ -138,6 +154,7 @@ def mock_sdk_with_responses( new_scan_metadata, new_scan_stream, stream_diff_response, + diff_scan_get_response, create_full_scan_response, ): sdk = mock_socket_sdk.return_value @@ -173,4 +190,8 @@ def mock_sdk_with_responses( lambda org_slug, head_id, new_id, **kwargs: stream_diff_response ) + # Diff-scans endpoints (primary scan-comparison path) + sdk.diffscans.create_from_ids.return_value = {"diff_scan": {"id": "diff-scan-123"}} + sdk.diffscans.get.return_value = diff_scan_get_response + return sdk diff --git a/tests/core/test_diff_scan_polling.py b/tests/core/test_diff_scan_polling.py new file mode 100644 index 00000000..be369429 --- /dev/null +++ b/tests/core/test_diff_scan_polling.py @@ -0,0 +1,122 @@ +"""Tests for the diff-scans polling scan comparison. + +The comparison must never hold an idle connection open: it creates a diff-scan +resource and polls the cached endpoint (202 while processing, 200 when ready), +falling back to the legacy streaming diff if the new flow is unavailable. +""" +import pytest +from socketdev.exceptions import APIConnectionError, APIFailure + +import socketsecurity.core as core_module +from socketsecurity.core import Core +from socketsecurity.core.socket_config import SocketConfig + + +@pytest.fixture +def core(mock_sdk_with_responses): + config = SocketConfig(api_key="test_key") + return Core(config=config, sdk=mock_sdk_with_responses) + + +@pytest.fixture +def no_sleep(mocker): + return mocker.patch("socketsecurity.core.time.sleep") + + +def test_polls_until_diff_scan_ready(core, diff_scan_get_response, no_sleep): + """202 processing responses are polled through until the 200 result arrives.""" + processing = {"status": "processing", "id": "diff-scan-123"} + core.sdk.diffscans.get.side_effect = [processing, processing, diff_scan_get_response] + + artifacts = core.get_diff_scan_artifacts("head", "new") + + assert core.sdk.diffscans.get.call_count == 3 + assert no_sleep.call_count == 2 # slept between polls, never during them + assert len(artifacts.added) > 0 + + +def test_poll_interval_backs_off(core, diff_scan_get_response, no_sleep, monkeypatch): + """The poll interval grows toward the max so long comparisons stay quota-friendly.""" + monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS", 4.0) + monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS", 10.0) + processing = {"status": "processing", "id": "diff-scan-123"} + core.sdk.diffscans.get.side_effect = [processing] * 4 + [diff_scan_get_response] + + core.get_diff_scan_artifacts("head", "new") + + waits = [call.args[0] for call in no_sleep.call_args_list] + assert waits == [4.0, 6.0, 9.0, 10.0] # 1.5x backoff, capped at the max + + +def test_transient_poll_error_is_retried(core, diff_scan_get_response, no_sleep): + """A dropped poll doesn't abandon the flow - the diff keeps computing server-side.""" + core.sdk.diffscans.get.side_effect = [APIConnectionError("reset"), diff_scan_get_response] + + artifacts = core.get_diff_scan_artifacts("head", "new") + + assert core.sdk.diffscans.get.call_count == 2 + assert len(artifacts.added) > 0 + + +def test_non_transient_poll_error_raises(core, no_sleep): + """Deterministic API errors (e.g. 403 missing scopes) propagate to the caller.""" + core.sdk.diffscans.get.side_effect = APIFailure("forbidden", status_code=403) + + with pytest.raises(APIFailure): + core.get_diff_scan_artifacts("head", "new") + + +def test_poll_timeout_raises(core, no_sleep, monkeypatch): + """A diff scan that never completes hits the polling backstop.""" + monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_TIMEOUT_SECONDS", 0.0) + core.sdk.diffscans.get.return_value = {"status": "processing", "id": "diff-scan-123"} + + with pytest.raises(Exception, match="Timed out waiting for diff scan"): + core.get_diff_scan_artifacts("head", "new") + + +def test_duplicate_conflict_uses_cached_polling(core, diff_scan_get_response): + """A duplicate is resolved explicitly so the SDK cannot follow an uncached redirect.""" + core.sdk.diffscans.create_from_ids.side_effect = APIFailure( + "duplicate", status_code=409 + ) + core.sdk.diffscans.list.return_value = { + "results": [{"id": "existing-diff-scan"}], + } + + artifacts = core.get_diff_scan_artifacts("head", "new") + + create_params = core.sdk.diffscans.create_from_ids.call_args.args[1] + assert "on_duplicate" not in create_params + core.sdk.diffscans.list.assert_called_once_with( + core.config.org_slug, + params={ + "before_full_scan_id": "head", + "after_full_scan_id": "new", + "per_page": 1, + }, + ) + core.sdk.diffscans.get.assert_called_once_with( + core.config.org_slug, + "existing-diff-scan", + params={"cached": "true"}, + ) + assert len(artifacts.added) > 0 + + +def test_fallback_to_streaming_diff_on_failure(core): + """If the diff-scans flow fails (e.g. token missing the diff-scans scopes), + the comparison falls back to the legacy streaming diff transparently.""" + core.sdk.diffscans.create_from_ids.side_effect = APIFailure("forbidden", status_code=403) + + added, removed, all_packages = core.get_added_and_removed_packages("head", "new") + + core.sdk.fullscans.stream_diff.assert_called_once_with( + core.config.org_slug, + "head", + "new", + use_types=True, + include_license_details="false", + ) + assert "dp3" in added + assert "dp2" in removed diff --git a/tests/core/test_sdk_methods.py b/tests/core/test_sdk_methods.py index 9b1ce449..02967315 100644 --- a/tests/core/test_sdk_methods.py +++ b/tests/core/test_sdk_methods.py @@ -225,19 +225,27 @@ def test_get_added_and_removed_packages(core): """Test getting added and removed packages between two scans""" # Get two different scans to compare added, removed, all_packages = core.get_added_and_removed_packages("head", "new") - - # Verify SDK was called correctly. - # include_license_details defaults to "false": the diff path never consumes - # embedded license data (license artifacts come from the PURL endpoint), so - # requesting it only bloats the response and risks the truncation - # crash on large repos. - core.sdk.fullscans.stream_diff.assert_called_once_with( + + # Verify SDK was called correctly: the comparison goes through the diff-scans + # endpoints (create + poll) rather than the legacy streaming diff, so no + # connection is left idle while the backend computes. + create_args = core.sdk.diffscans.create_from_ids.call_args + assert create_args[0][0] == core.config.org_slug + create_params = create_args[0][1] + assert create_params["before"] == "head" + assert create_params["after"] == "new" + assert "on_duplicate" not in create_params + + # cached=true is the polling contract (202 while computing, 200 when ready). + # No omit_license_details param: the API ignores it for cached reads (cached + # results always embed license details), so sending it would only suggest a + # leanness guarantee this path doesn't have. + core.sdk.diffscans.get.assert_called_once_with( core.config.org_slug, - "head", - "new", - use_types=True, - include_license_details="false", + "diff-scan-123", + params={"cached": "true"}, ) + core.sdk.fullscans.stream_diff.assert_not_called() # Verify the results # Added packages @@ -252,7 +260,13 @@ def test_get_added_and_removed_packages(core): assert "pypi/direct_package_1@1.6.0" in all_packages # Unchanged package is in full package map def test_get_added_and_removed_packages_license_override(core): - """The include_license_details override seam still works when explicitly requested.""" + """include_license_details only governs the legacy fallback path now: the + diff-scans path always receives embedded license details (the API ignores + omit_license_details for cached reads), so the seam must survive through to + the stream_diff call when the primary path is unavailable.""" + from socketdev.exceptions import APIFailure + + core.sdk.diffscans.create_from_ids.side_effect = APIFailure("forbidden", status_code=403) core.get_added_and_removed_packages("head", "new", include_license_details=True) core.sdk.fullscans.stream_diff.assert_called_once_with( diff --git a/uv.lock b/uv.lock index fb5a540f..172c2af4 100644 --- a/uv.lock +++ b/uv.lock @@ -1282,7 +1282,7 @@ wheels = [ [[package]] name = "socketsecurity" -version = "2.6.0" +version = "2.6.1" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, From 29bbc568a49bc5ae02c64630a0cffa9c1d5e1781 Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:09:30 -0400 Subject: [PATCH 02/11] Raise failure on SBOM fetch errors (#288) * fix(core): raise on SBOM fetch failure instead of writing empty reports (CE-362) get_sbom_data returned {} when the full-scan stream fetch failed, so report generation continued and produced empty GitLab dependency scanning, license, and SARIF output with exit code 0. Raise APIFailure instead so the failure goes through the CLI's existing API-error handling (exit code 3 by default, still exit 0 with --disable-blocking). Bump the socketdev floor to 3.4.2, the bundled release that adds the missing purl types (e.g. "generic") and per-artifact parse resilience that caused this failure mode. Merge after socketdev 3.4.2 is on PyPI. Co-Authored-By: Claude Fable 5 * chore: lock socketdev 3.4.2 Co-Authored-By: Claude Fable 5 * chore: bump version to 2.5.11 Co-Authored-By: Claude Fable 5 * ci(e2e): retry reachability on empty results, upload diagnostics on failure The e2e-reachability job intermittently fails with 'no components with alerts in .socket.facts.json': the tier-1 reachability backend can return empty results while the CLI reports success (ENG-5093), and the same flake has hit unrelated PRs. - Add a retry-probe hook to the e2e matrix: entries that define it get up to 3 scan attempts, retrying only when the probe says the output looks incomplete. Persistent failures still fail via the validate step. Each retry emits a warning annotation and a step-summary line so flake frequency stays visible. - Add tests/e2e/reach-facts-probe.sh: exits 0 when the facts file has alerted components, non-zero (retry) when empty or missing. - Upload /tmp/e2e-output.log, SARIF/GitLab outputs, and facts files as artifacts when any e2e job fails, so flakes are diagnosable without a re-run. Also bump version to 2.6.2 (2.6.0 and 2.6.1 are being released ahead of this PR). Co-Authored-By: Claude Fable 5 * chore: require socketdev 3.5.0 Co-Authored-By: Claude Fable 5 * Drop ticket references from e2e comments and note the retry hardening in the changelog Co-Authored-By: Claude Fable 5 Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> * Move e2e retry changelog entry out and drop remaining ticket reference The e2e retry hardening ships with the dependency pinning PR instead, so its changelog entry moves there. Co-Authored-By: Claude Fable 5 Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> * docs: changelog phrasing tweak Co-Authored-By: Claude Fable 5 --------- Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- CHANGELOG.md | 13 +++++++++++++ pyproject.toml | 2 +- socketsecurity/__init__.py | 2 +- socketsecurity/core/__init__.py | 10 +++++++--- tests/core/test_sdk_methods.py | 20 +++++++++++++++++++- uv.lock | 2 +- 6 files changed, 42 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22976e66..53adb48d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 2.6.2 + +### Fixed: SBOM fetch failures no longer produce empty reports + +- `Core.get_sbom_data` now raises `APIFailure` when the full-scan stream fetch + fails, so the run exits through the CLI's API-error handling (exit code 3 by + default; `--disable-blocking` still exits 0) instead of writing empty + GitLab dependency-scanning, license, and SARIF reports. +- The underlying stream-parse failure was fixed in `socketdev` 3.4.2 (already + pinned to `3.5.0`): unrecognized purl types such as `generic` now resolve + instead of raising, and individual unparseable artifacts are skipped rather + than failing the whole response. + ## 2.6.1 ### Changed: scan comparison now polls the diff-scans endpoints diff --git a/pyproject.toml b/pyproject.toml index c70b629b..09ef2181 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.6.1" +version = "2.6.2" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 312c5053..c4c8bad0 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,3 +1,3 @@ __author__ = 'socket.dev' -__version__ = '2.6.1' +__version__ = '2.6.2' USER_AGENT = f'SocketPythonCLI/{__version__}' diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 7bd4a33e..7be24858 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -173,9 +173,13 @@ def get_sbom_data(self, full_scan_id: str) -> Dict[str, SocketArtifact]: """Returns SBOM artifacts for a full scan keyed by artifact ID.""" response = self.sdk.fullscans.stream(self.config.org_slug, full_scan_id, use_types=True) if not response.success: - log.debug(f"Failed to get SBOM data for full-scan {full_scan_id}") - log.debug(response.message) - return {} + # Raise instead of returning {} so a failed fetch surfaces as an + # API error (exit code 3 by default) rather than empty reports. + log.error(f"Failed to get SBOM data for full-scan {full_scan_id}") + log.error(response.message) + raise APIFailure( + f"Failed to get SBOM data for full-scan {full_scan_id}: {response.message}" + ) if not hasattr(response, "artifacts") or not response.artifacts: return {} return response.artifacts diff --git a/tests/core/test_sdk_methods.py b/tests/core/test_sdk_methods.py index 02967315..da0efc62 100644 --- a/tests/core/test_sdk_methods.py +++ b/tests/core/test_sdk_methods.py @@ -1,5 +1,6 @@ import pytest -from socketdev.fullscans import FullScanParams +from socketdev.exceptions import APIFailure +from socketdev.fullscans import FullScanParams, FullScanStreamResponse from socketsecurity.config import CliConfig from socketsecurity.core import Core @@ -277,6 +278,23 @@ def test_get_added_and_removed_packages_license_override(core): include_license_details="true", ) +def test_get_sbom_data_failure_raises(core): + """A failed SBOM stream fetch raises instead of returning {}. + + Returning {} let report generation continue and emit empty results with + exit code 0; raising routes the failure through the CLI's API-error + handling instead. + """ + core.sdk.fullscans.stream.side_effect = None + core.sdk.fullscans.stream.return_value = FullScanStreamResponse.from_dict({ + "success": False, + "status": 200, + "message": "Error parsing stream response", + }) + + with pytest.raises(APIFailure, match="Failed to get SBOM data"): + core.get_sbom_data("head") + def test_empty_alerts_preserved(core): """Test that empty alerts arrays stay as empty arrays and don't become None""" # Get the scan that contains dp2 (which has empty alerts array) diff --git a/uv.lock b/uv.lock index 172c2af4..4a7c16ad 100644 --- a/uv.lock +++ b/uv.lock @@ -1282,7 +1282,7 @@ wheels = [ [[package]] name = "socketsecurity" -version = "2.6.1" +version = "2.6.2" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, From a993c9d4090a4990d7c1dd8880a147ba473f7bb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:19:41 -0400 Subject: [PATCH 03/11] ci(deps): bump actions/setup-python from 6.2.0 to 7.0.0 (#293) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6.2.0 to 7.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/a309ff8b426b58ec0e2a45f0f869d46889d02405...5fda3b95a4ea91299a34e894583c3862153e4b97) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: lelia <2418071+lelia@users.noreply.github.com> --- .github/workflows/e2e-test.yml | 2 +- .github/workflows/package-check.yml | 2 +- .github/workflows/pr-preview.yml | 2 +- .github/workflows/python-tests.yml | 4 ++-- .github/workflows/release.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index a1d2f290..777777be 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -80,7 +80,7 @@ jobs: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.12' diff --git a/.github/workflows/package-check.yml b/.github/workflows/package-check.yml index b2e8409a..d71083a4 100644 --- a/.github/workflows/package-check.yml +++ b/.github/workflows/package-check.yml @@ -23,7 +23,7 @@ jobs: fetch-depth: 1 persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" diff --git a/.github/workflows/pr-preview.yml b/.github/workflows/pr-preview.yml index 473a68a0..3a2ab1a9 100644 --- a/.github/workflows/pr-preview.yml +++ b/.github/workflows/pr-preview.yml @@ -107,7 +107,7 @@ jobs: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 77133062..34717226 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -40,7 +40,7 @@ jobs: fetch-depth: 1 persist-credentials: false - name: 🐍 setup python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: 🛠️ install deps @@ -76,7 +76,7 @@ jobs: fetch-depth: 1 persist-credentials: false - name: 🐍 setup python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.10" - name: 🚫 verify install is rejected on unsupported python diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5be9e266..1c5faa9a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.13' From dd0835e369e96d5c0fcd7ec9afaaa486ddc23e5d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:33:31 -0400 Subject: [PATCH 04/11] chore(deps): bump twine from 6.2.0 to 7.0.0 in the python-major group (#296) * chore(deps): bump twine from 6.2.0 to 7.0.0 in the python-major group Bumps the python-major group with 1 update: [twine](https://github.com/pypa/twine). Updates `twine` from 6.2.0 to 7.0.0 - [Release notes](https://github.com/pypa/twine/releases) - [Changelog](https://github.com/pypa/twine/blob/main/docs/changelog.rst) - [Commits](https://github.com/pypa/twine/compare/6.2.0...7.0.0) --- updated-dependencies: - dependency-name: twine dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: python-major ... Signed-off-by: dependabot[bot] * Pin dev and test extras to exact versions Runtime dependencies were pinned exactly in 2.6.0; this applies the same policy to the dev and test extras (matching the currently locked versions, including the twine 7.0.0 bump from this PR) so future dependency updates surface in pyproject.toml rather than only in uv.lock. Co-Authored-By: Claude Fable 5 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: lelia <2418071+lelia@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- pyproject.toml | 20 ++++++++++---------- uv.lock | 32 ++++++++++++++++---------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 09ef2181..d66e4bb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,18 +40,18 @@ classifiers = [ [project.optional-dependencies] test = [ - "pytest>=7.4.0", - "pytest-cov>=4.1.0", - "pytest-mock>=3.12.0", - "pytest-asyncio>=0.23.0", - "pytest-watch >=4.2.0" + "pytest==9.1.1", + "pytest-cov==7.1.0", + "pytest-mock==3.15.1", + "pytest-asyncio==1.4.0", + "pytest-watch==4.2.0" ] dev = [ - "ruff>=0.3.0", - "twine", # for building - "uv>=0.1.0", # for dependency management - "pre-commit", - "hatch" + "ruff==0.16.0", + "twine==7.0.0", # for building + "uv==0.12.0", # for dependency management + "pre-commit==4.6.1", + "hatch==1.17.1" ] [project.scripts] diff --git a/uv.lock b/uv.lock index 4a7c16ad..b9f904ad 100644 --- a/uv.lock +++ b/uv.lock @@ -1200,15 +1200,15 @@ wheels = [ [[package]] name = "rich" -version = "14.2.0" +version = "15.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown-it-py" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fb/d2/8920e102050a0de7bfabeb4c4614a49248cf8d5d7a8d01885fbb24dc767a/rich-14.2.0.tar.gz", hash = "sha256:73ff50c7c0c1c77c8243079283f4edb376f0f6442433aecb8ce7e6d0b92d1fe4", size = 219990, upload-time = "2025-10-09T14:16:53.064Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl", hash = "sha256:76bc51fe2e57d2b1be1f96c524b890b816e334ab4c1e45888799bfaab0021edd", size = 243393, upload-time = "2025-10-09T14:16:51.245Z" }, + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, ] [[package]] @@ -1325,23 +1325,23 @@ requires-dist = [ { name = "brotli", marker = "platform_python_implementation == 'CPython'", specifier = "==1.2.0" }, { name = "brotlicffi", marker = "platform_python_implementation != 'CPython'", specifier = "==1.2.0.1" }, { name = "gitpython", specifier = "==3.1.57" }, - { name = "hatch", marker = "extra == 'dev'" }, + { name = "hatch", marker = "extra == 'dev'", specifier = "==1.17.1" }, { name = "markdown", specifier = "==3.10.2" }, { name = "mdutils", specifier = "==1.8.1" }, { name = "packaging", specifier = "==26.2" }, - { name = "pre-commit", marker = "extra == 'dev'" }, + { name = "pre-commit", marker = "extra == 'dev'", specifier = "==4.6.1" }, { name = "prettytable", specifier = "==3.18.0" }, - { name = "pytest", marker = "extra == 'test'", specifier = ">=7.4.0" }, - { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.23.0" }, - { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=4.1.0" }, - { name = "pytest-mock", marker = "extra == 'test'", specifier = ">=3.12.0" }, - { name = "pytest-watch", marker = "extra == 'test'", specifier = ">=4.2.0" }, + { name = "pytest", marker = "extra == 'test'", specifier = "==9.1.1" }, + { name = "pytest-asyncio", marker = "extra == 'test'", specifier = "==1.4.0" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = "==7.1.0" }, + { name = "pytest-mock", marker = "extra == 'test'", specifier = "==3.15.1" }, + { name = "pytest-watch", marker = "extra == 'test'", specifier = "==4.2.0" }, { name = "python-dotenv", specifier = "==1.2.2" }, { name = "requests", specifier = "==2.34.2" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.3.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.16.0" }, { name = "socketdev", specifier = "==3.5.0" }, - { name = "twine", marker = "extra == 'dev'" }, - { name = "uv", marker = "extra == 'dev'", specifier = ">=0.1.0" }, + { name = "twine", marker = "extra == 'dev'", specifier = "==7.0.0" }, + { name = "uv", marker = "extra == 'dev'", specifier = "==0.12.0" }, ] provides-extras = ["test", "dev"] @@ -1444,7 +1444,7 @@ wheels = [ [[package]] name = "twine" -version = "6.2.0" +version = "7.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "id" }, @@ -1457,9 +1457,9 @@ dependencies = [ { name = "rich" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e0/a8/949edebe3a82774c1ec34f637f5dd82d1cf22c25e963b7d63771083bbee5/twine-6.2.0.tar.gz", hash = "sha256:e5ed0d2fd70c9959770dce51c8f39c8945c574e18173a7b81802dab51b4b75cf", size = 172262, upload-time = "2025-09-04T15:43:17.255Z" } +sdist = { url = "https://files.pythonhosted.org/packages/92/3c/58f808a359700f39a967dffede33efeac809262c03303fa3eec6afff8f49/twine-7.0.0.tar.gz", hash = "sha256:85cdb29c518efef867360ae4acd4b0dfd61c8654a22fca08e6f8539f05022177", size = 215032, upload-time = "2026-07-27T15:59:00.825Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/7a/882d99539b19b1490cac5d77c67338d126e4122c8276bf640e411650c830/twine-6.2.0-py3-none-any.whl", hash = "sha256:418ebf08ccda9a8caaebe414433b0ba5e25eb5e4a927667122fbe8f829f985d8", size = 42727, upload-time = "2025-09-04T15:43:15.994Z" }, + { url = "https://files.pythonhosted.org/packages/96/08/ddcdc06225eaad6de0e48e1002b06d919dbde20582d0662c7af51308e5d6/twine-7.0.0-py3-none-any.whl", hash = "sha256:b854164df26db268af05f49aa5c0344b10e27a494343ff05b1e0bad3b135f5a7", size = 43204, upload-time = "2026-07-27T15:58:59.26Z" }, ] [[package]] From 70bbde99db5daacf8db4a08b928dd3f65057c8ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:35:46 -0400 Subject: [PATCH 05/11] ci(deps): bump the github-actions-minor-patch group across 2 directories with 4 updates (#292) Bumps the github-actions-minor-patch group with 1 update in the / directory: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Bumps the github-actions-minor-patch group with 3 updates in the /.github/actions/setup-docker directory: [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action), [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) and [docker/login-action](https://github.com/docker/login-action). Updates `pypa/gh-action-pypi-publish` from 1.14.1 to 1.14.2 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.14.1...dc37677b2e1c63e2034f94d8a5b11f265b73ba33) Updates `docker/setup-qemu-action` from 4.1.0 to 4.2.0 - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/06116385d9baf250c9f4dcb4858b16962ea869c3...96fe6ef7f33517b61c61be40b68a1882f3264fb8) Updates `docker/setup-buildx-action` from 4.1.0 to 4.2.0 - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5...bb05f3f5519dd87d3ba754cc423b652a5edd6d2c) Updates `docker/login-action` from 4.2.0 to 4.6.0 - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/650006c6eb7dba73a995cc03b0b2d7f5ca915bee...dbcb813823bdd20940b903addbd779551569679f) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-version: 1.14.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions-minor-patch - dependency-name: docker/setup-qemu-action dependency-version: 4.2.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions-minor-patch - dependency-name: docker/setup-buildx-action dependency-version: 4.2.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions-minor-patch - dependency-name: docker/login-action dependency-version: 4.6.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions-minor-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: lelia <2418071+lelia@users.noreply.github.com> --- .github/actions/setup-docker/action.yml | 6 +++--- .github/workflows/release.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup-docker/action.yml b/.github/actions/setup-docker/action.yml index 2eea2fb8..58239ff6 100644 --- a/.github/actions/setup-docker/action.yml +++ b/.github/actions/setup-docker/action.yml @@ -19,10 +19,10 @@ inputs: runs: using: "composite" steps: - - uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0 + - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 if: inputs.enable-qemu == 'true' - - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ inputs.dockerhub-username }} password: ${{ inputs.dockerhub-token }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1c5faa9a..62b2f2b7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,7 +66,7 @@ jobs: - name: Publish to PyPI if: steps.version_check.outputs.pypi_exists != 'true' - uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 - name: Set up Docker publishing uses: ./.github/actions/setup-docker From 1cf2246b29dc64018bbd39793a052b0499a73566 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:42:57 -0400 Subject: [PATCH 06/11] ci(deps): bump actions/setup-python in /.github/actions/setup-sfw (#295) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6.2.0 to 7.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/a309ff8b426b58ec0e2a45f0f869d46889d02405...5fda3b95a4ea91299a34e894583c3862153e4b97) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: lelia <2418071+lelia@users.noreply.github.com> --- .github/actions/setup-sfw/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-sfw/action.yml b/.github/actions/setup-sfw/action.yml index 456f90b9..66b54ffc 100644 --- a/.github/actions/setup-sfw/action.yml +++ b/.github/actions/setup-sfw/action.yml @@ -27,7 +27,7 @@ runs: using: "composite" steps: - if: ${{ inputs.python == 'true' || inputs.uv == 'true' }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" From 56181bcfde32b22c4217200e25e1dcef399da0a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:52:35 -0400 Subject: [PATCH 07/11] chore(deps): bump the python-minor-patch group with 3 updates (#294) Bumps the python-minor-patch group with 3 updates: [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/bs4/), [markdown](https://github.com/Python-Markdown/markdown) and [ruff](https://github.com/astral-sh/ruff). Updates `beautifulsoup4` from 4.14.3 to 4.15.0 Updates `markdown` from 3.10.2 to 3.10.3 - [Release notes](https://github.com/Python-Markdown/markdown/releases) - [Changelog](https://github.com/Python-Markdown/markdown/blob/master/docs/changelog.md) - [Commits](https://github.com/Python-Markdown/markdown/compare/3.10.2...3.10.3) Updates `ruff` from 0.16.0 to 0.16.1 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.16.0...0.16.1) --- updated-dependencies: - dependency-name: beautifulsoup4 dependency-version: 4.15.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: python-minor-patch - dependency-name: markdown dependency-version: 3.10.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-minor-patch - dependency-name: ruff dependency-version: 0.16.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-minor-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: lelia <2418071+lelia@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- pyproject.toml | 6 ++--- uv.lock | 60 +++++++++++++++++++++++++------------------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d66e4bb7..8275fdb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,8 +17,8 @@ dependencies = [ "packaging==26.2", "python-dotenv==1.2.2", "socketdev==3.5.0", - "beautifulsoup4==4.14.3", - "markdown==3.10.2", + "beautifulsoup4==4.15.0", + "markdown==3.10.3", "brotli==1.2.0; platform_python_implementation == 'CPython'", "brotlicffi==1.2.0.1; platform_python_implementation != 'CPython'", ] @@ -47,7 +47,7 @@ test = [ "pytest-watch==4.2.0" ] dev = [ - "ruff==0.16.0", + "ruff==0.16.1", "twine==7.0.0", # for building "uv==0.12.0", # for dependency management "pre-commit==4.6.1", diff --git a/uv.lock b/uv.lock index b9f904ad..58516703 100644 --- a/uv.lock +++ b/uv.lock @@ -112,15 +112,15 @@ wheels = [ [[package]] name = "beautifulsoup4" -version = "4.14.3" +version = "4.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "soupsieve" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c3/b0/1c6a16426d389813b48d95e26898aff79abbde42ad353958ad95cc8c9b21/beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86", size = 627737, upload-time = "2025-11-30T15:08:26.084Z" } +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, ] [[package]] @@ -795,11 +795,11 @@ wheels = [ [[package]] name = "markdown" -version = "3.10.2" +version = "3.10.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2b/f4/69fa6ed85ae003c2378ffa8f6d2e3234662abd02c10d216c0ba96081a238/markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950", size = 368805, upload-time = "2026-02-09T14:57:26.942Z" } +sdist = { url = "https://files.pythonhosted.org/packages/29/6f/da4c6aea59b3001f2e8c0ec7497475aadaf3b021c10cab5b2858f0f32b26/markdown-3.10.3.tar.gz", hash = "sha256:3589362618f743188b4d955b874402bc814f4f83f544dc207719f4baa7d9c45f", size = 372596, upload-time = "2026-07-30T19:05:29.005Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36", size = 108180, upload-time = "2026-02-09T14:57:25.787Z" }, + { url = "https://files.pythonhosted.org/packages/64/69/4a5af2bc115a9a33fefe51709749de8262be3f9ba063d1753a837cdbc49c/markdown-3.10.3-py3-none-any.whl", hash = "sha256:fa6c92a00a4a3c98b22728c64a935ae1928250ae65058a6ded814d2cc29a4cea", size = 110757, upload-time = "2026-07-30T19:05:27.883Z" }, ] [[package]] @@ -1213,27 +1213,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4d/94/1e5e4967626faf12fa56999cd6222dff6992ceb086ad7945756baf70c7a7/ruff-0.16.0.tar.gz", hash = "sha256:e460aafd5495ec89efaa6ced2e4a9a581116451e1c88b9d37ef497e0f8e93982", size = 4790557, upload-time = "2026-07-23T19:11:30.981Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/81/1c8818fee7ce1a04cd7d1b3172e0a8f8e4f1dc4feb7fc390e16daa8af323/ruff-0.16.0-py3-none-linux_armv6l.whl", hash = "sha256:e5115729eb08c585e5121978ba5d5b60caeae394ce21b9fb5e6cd33a1c6c9b1e", size = 10754633, upload-time = "2026-07-23T19:10:46.415Z" }, - { url = "https://files.pythonhosted.org/packages/23/df/beaf59c09d68db84304d555f188b276a77132a5d5b0b67a5c762aa143628/ruff-0.16.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3c954b1d580bfa035b41654f7858cc7e71d5fc3ac5b723dd62bd9133830ed522", size = 10969164, upload-time = "2026-07-23T19:10:50.271Z" }, - { url = "https://files.pythonhosted.org/packages/42/ce/741cd197496a1abbf51352710fd15ed995d2a2be87189c1da26a450d6e83/ruff-0.16.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e01c21d10eb1b29f47b7454e1f4056db9a3f0260c646aa88457c610291db9f81", size = 10488846, upload-time = "2026-07-23T19:10:52.639Z" }, - { url = "https://files.pythonhosted.org/packages/52/2a/a2db8e88cade358f5cdcb05674a917751074109315d014eb6352d9a893f7/ruff-0.16.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e364e5ed22ed8dc05082fd78e35308618260907ac2d3c1d637b2e682415b6c9", size = 10889729, upload-time = "2026-07-23T19:10:54.89Z" }, - { url = "https://files.pythonhosted.org/packages/42/65/62a771694ebd63029dc953e27dbad40e1588bd4860ff9fe881018fddaa49/ruff-0.16.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d327b8fc113a1d4421a04f3839d3752057c8dd1ee320223a6f3f52d04ada462a", size = 10568275, upload-time = "2026-07-23T19:10:56.993Z" }, - { url = "https://files.pythonhosted.org/packages/3f/e2/ced249fe8af5f086c5c58cc21cc3356d50f32f7401c5df87050c999620a7/ruff-0.16.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9b50c55e263103586b3dcf5f73d479eb8cb5fdb6098fec59a62891dab653717", size = 11385112, upload-time = "2026-07-23T19:10:59.615Z" }, - { url = "https://files.pythonhosted.org/packages/87/0b/05154977a8fd69eeb6c103271f55403bfd8711f5c0f8ed07489d95a504e7/ruff-0.16.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ff4a79ce3ec0172f3241943835de1c4cb4e2dcd07f0f8c2d02603dbbbee4b17", size = 12207008, upload-time = "2026-07-23T19:11:02.154Z" }, - { url = "https://files.pythonhosted.org/packages/fb/29/98225831a3a1eab0e02f4acc6ca6559a98611dcc68b6965ff4b7234627c1/ruff-0.16.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e95c448fca1fb2a18372a9440926c5a6ee789639bb975c72e7ae6d0b04218ab4", size = 11650842, upload-time = "2026-07-23T19:11:04.557Z" }, - { url = "https://files.pythonhosted.org/packages/91/66/6bd3cf90500653d55dc0ffc8507aa8300bd49d0214b2e8cb4d3fef2943ba/ruff-0.16.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f11a8d11010301d0a398a2fdef67691feca7294da6aef55e2150e8fa2cd520b", size = 11400718, upload-time = "2026-07-23T19:11:09.233Z" }, - { url = "https://files.pythonhosted.org/packages/8e/a2/a54eb4eae05d66364050a5d3b8a9c5ef88196531b3cbe7109d873f87f819/ruff-0.16.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:48044c678e9cb8698246c99b14aaccfa6601dea7379eb48a6f8f73f7a6d86cd0", size = 11426177, upload-time = "2026-07-23T19:11:11.994Z" }, - { url = "https://files.pythonhosted.org/packages/1a/be/16e3eea4b2a478a496919f5e36f17c4559e54620bd3bbac5d6affa068006/ruff-0.16.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:7aa0959bad8eb8bef50340154fc9b58678dae31fa4293afa38b44b6e552c0213", size = 10856126, upload-time = "2026-07-23T19:11:14.221Z" }, - { url = "https://files.pythonhosted.org/packages/a2/84/252eb8b868a16eec7257c14f504f77537e734b2d69c762e639e588e304a3/ruff-0.16.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:28ea2b7df8ebf7f9da6b7d47b230ab48f387c0a29be3b474c4d0740e197bb9af", size = 10571208, upload-time = "2026-07-23T19:11:16.378Z" }, - { url = "https://files.pythonhosted.org/packages/21/09/817a482f542f7570cbb4554b26e896610c7114f539b1d9e2d2145bf6bef6/ruff-0.16.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:33a3dfac8c35f81498dea9181bccc2f4c4bc8f1521a1dd9406e77643e0f0fb09", size = 11063329, upload-time = "2026-07-23T19:11:19.173Z" }, - { url = "https://files.pythonhosted.org/packages/2e/23/9403c180ca1cb9b1f7335f5c3e5305c09d49ea5b345196682a36028bde4a/ruff-0.16.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a5237a0bda500d30d81b8e07a6973a5cbc772864cbf746ae2f4e8a2e01c9f4ed", size = 11489751, upload-time = "2026-07-23T19:11:21.74Z" }, - { url = "https://files.pythonhosted.org/packages/b2/1d/1b2ef7bcde851c78d7f17f1cca13fd6dc695fc4b3d6197941e72cae5b132/ruff-0.16.0-py3-none-win32.whl", hash = "sha256:7fab76fa065c873f41ff744347c6e77bcc3dfec4bcc754dc26b63d23c0f7f5fb", size = 10785885, upload-time = "2026-07-23T19:11:23.947Z" }, - { url = "https://files.pythonhosted.org/packages/b2/a3/d5e4ef7a56be3f928ffb90b94c25ba7d3cb9c7fe0736aeaaedf361770712/ruff-0.16.0-py3-none-win_amd64.whl", hash = "sha256:429c117f022bf481fabd9d551e7a3952b24c65e6ef44337ea09d90bebef14472", size = 11923141, upload-time = "2026-07-23T19:11:26.409Z" }, - { url = "https://files.pythonhosted.org/packages/cb/9a/8415f2657cbe200f41a4531ccededf135505a92d4a012229121f885b26f9/ruff-0.16.0-py3-none-win_arm64.whl", hash = "sha256:14296fedcd2705c77ab8235439278bbb38f285cf7da5528b00b3e330c3d4872d", size = 11273407, upload-time = "2026-07-23T19:11:28.705Z" }, +version = "0.16.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/70/25/7113f6d5498888c5fb7db34081cba7d5971c4cb1bfb26819966eee68f003/ruff-0.16.1.tar.gz", hash = "sha256:fedad7c801dabd3fb9741d76aca39246e6ddd9ca446a015875207bf19f1e6bc7", size = 4877500, upload-time = "2026-07-30T19:37:01.379Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/bd/694da69368e0973de65df2ddc73ab18d43c469d5963d9b150911de6bc513/ruff-0.16.1-py3-none-linux_armv6l.whl", hash = "sha256:58edb313b88f0c5460a26adf5f39a37a3be789494a15e3e411e35fa78b89f9a0", size = 10839126, upload-time = "2026-07-30T19:36:13.697Z" }, + { url = "https://files.pythonhosted.org/packages/3f/f0/b626e5d5bd0dd9576263658ef12885e2288afd1029a48e26ffed65ec1ac1/ruff-0.16.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fde5a99e2f97479af66edd6622c6d5a2a7592c77cf4153d9e4428f5eeb55b60c", size = 11070253, upload-time = "2026-07-30T19:36:17.14Z" }, + { url = "https://files.pythonhosted.org/packages/83/63/f40acfb6b35b88623e71684942b552c3edd96035f5d98f313815f7b277de/ruff-0.16.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d4c20532fca4f7fa609369161d968dd28f65d83dabbd61d8e9c7edbf7001f6", size = 10561425, upload-time = "2026-07-30T19:36:20.04Z" }, + { url = "https://files.pythonhosted.org/packages/aa/dd/14ec0e9c2b4d315547dd38765004b4863e354e1b52cb308272215d9f6f6d/ruff-0.16.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30affbcedf59ad5703d9c91f82266e02b47739f797e1a7b6e158e5526a6dae38", size = 10948879, upload-time = "2026-07-30T19:36:22.476Z" }, + { url = "https://files.pythonhosted.org/packages/33/e9/9d870cbae575030fdef595f04b4b97573c525b5497cce4f4498cf2f85446/ruff-0.16.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24e9c631573cbca9d20f1283f8f479b2afa4a8503504822bd71a293889f16743", size = 10643691, upload-time = "2026-07-30T19:36:24.914Z" }, + { url = "https://files.pythonhosted.org/packages/c4/09/12743d544e2173f53ecd27217c65f90d2bc0f8424a66a60339e56bbc0457/ruff-0.16.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b41bdd48fb420987a9b5212e4957c26ad4abce401fa9ea9d4d85843727945f4f", size = 11435354, upload-time = "2026-07-30T19:36:28.447Z" }, + { url = "https://files.pythonhosted.org/packages/7f/89/a1652b2daee52083c9554a6333b678a8b01d0400f976827bb87857f9449a/ruff-0.16.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0d1e1393b7648079e13669de1c1f4fde06d4583e84d8fd5c1551e0a77a2aa75", size = 12259033, upload-time = "2026-07-30T19:36:31.326Z" }, + { url = "https://files.pythonhosted.org/packages/16/96/ecdcb8c54ee7b123b487f807eb014e6e019155a0b81dfb669acd52f28ce3/ruff-0.16.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07bf434b1c95f4e093be4532068ef4fcf00924eb2ade8796075980902d6fd54a", size = 11667981, upload-time = "2026-07-30T19:36:34.394Z" }, + { url = "https://files.pythonhosted.org/packages/cd/90/c52e12e0d862e9572f2a33aa227409143520abe53111e9a6babbac7b4af8/ruff-0.16.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39897739f112253ee4fdd2e8aa9a4f9ded99fb2be367d5f31dfa4ded6025584c", size = 11468183, upload-time = "2026-07-30T19:36:37.339Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6b/4ffb7ad1d83eb16cf8cbb3c8815d3f11c88460fd162d4b372a2059be1c2a/ruff-0.16.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82ae3c0c0d74daf17b968a10b7b3bb3ef297ab7de0c1f749646b25e690ccb150", size = 11470071, upload-time = "2026-07-30T19:36:39.91Z" }, + { url = "https://files.pythonhosted.org/packages/9c/72/32ae7db4c0b5e32ab611787caa19d1546800676d79f7483b7100a3561bf4/ruff-0.16.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4d5f2ed10f8242d83fc08d521301089364e3375375705356f20c0e31606ef3ef", size = 10919503, upload-time = "2026-07-30T19:36:42.65Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ca/3d901ba6ad6fc38da39c3448fc6c59ac945679293a17c3ceb6d6c1cba13e/ruff-0.16.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a4665b309891f83f3e3c25447935f1213e9abbd4b5640af7a1f2def9f8d413c1", size = 10649861, upload-time = "2026-07-30T19:36:45.18Z" }, + { url = "https://files.pythonhosted.org/packages/92/79/894ef1ced26552d5f8c9cf6d85b0687840e1128c55aeab7b9c2d54a0d880/ruff-0.16.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26e9ca5c9bc3971f20d3cf18a957f52ffd6a5f6564ff15c4912a144dcac22494", size = 11148137, upload-time = "2026-07-30T19:36:47.936Z" }, + { url = "https://files.pythonhosted.org/packages/2d/69/3609a09fa1cb46cc28b762363e440a354204e5dff01bd0c8d7437874d6b9/ruff-0.16.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:67e1e1e3fa4f0c82f0e36d4cd61e661f6e7a6196cb1aa92fe0828fa7b8f257cd", size = 11559211, upload-time = "2026-07-30T19:36:50.448Z" }, + { url = "https://files.pythonhosted.org/packages/fc/8a/fb22af2fd78a736e241fabf67e30ce1799a64244026377a49e133af90762/ruff-0.16.1-py3-none-win32.whl", hash = "sha256:d31765e131295b8445caf301e3e8a85b34d1b9b211b4109b7ba457888b051806", size = 10838258, upload-time = "2026-07-30T19:36:53.298Z" }, + { url = "https://files.pythonhosted.org/packages/d4/35/e57fd9fb5d423961df087a00b12d42c0a830288dc2f3b45ecca299158b4f/ruff-0.16.1-py3-none-win_amd64.whl", hash = "sha256:09b05e8b90c2cb06ad63464350e7a45e8e44a2dfe52072ebfba6666ca8d3f596", size = 11961111, upload-time = "2026-07-30T19:36:56.107Z" }, + { url = "https://files.pythonhosted.org/packages/cb/46/240ea004bf6dc4feb40e9832f2205a476a47dd5b8a3f8211a5fc5f95e20e/ruff-0.16.1-py3-none-win_arm64.whl", hash = "sha256:dbaadaac38c70239f056d306b7476f246b0bf000fa6b3876402acbf5b227eaf8", size = 11309414, upload-time = "2026-07-30T19:36:58.79Z" }, ] [[package]] @@ -1321,12 +1321,12 @@ dev = [ [package.metadata] requires-dist = [ - { name = "beautifulsoup4", specifier = "==4.14.3" }, + { name = "beautifulsoup4", specifier = "==4.15.0" }, { name = "brotli", marker = "platform_python_implementation == 'CPython'", specifier = "==1.2.0" }, { name = "brotlicffi", marker = "platform_python_implementation != 'CPython'", specifier = "==1.2.0.1" }, { name = "gitpython", specifier = "==3.1.57" }, { name = "hatch", marker = "extra == 'dev'", specifier = "==1.17.1" }, - { name = "markdown", specifier = "==3.10.2" }, + { name = "markdown", specifier = "==3.10.3" }, { name = "mdutils", specifier = "==1.8.1" }, { name = "packaging", specifier = "==26.2" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = "==4.6.1" }, @@ -1338,7 +1338,7 @@ requires-dist = [ { name = "pytest-watch", marker = "extra == 'test'", specifier = "==4.2.0" }, { name = "python-dotenv", specifier = "==1.2.2" }, { name = "requests", specifier = "==2.34.2" }, - { name = "ruff", marker = "extra == 'dev'", specifier = "==0.16.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.16.1" }, { name = "socketdev", specifier = "==3.5.0" }, { name = "twine", marker = "extra == 'dev'", specifier = "==7.0.0" }, { name = "uv", marker = "extra == 'dev'", specifier = "==0.12.0" }, From 974f65671108390088abd7bb0dd2698aa2958175 Mon Sep 17 00:00:00 2001 From: "socket-pr-bot[bot]" <294242679+socket-pr-bot[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 22:03:24 -0400 Subject: [PATCH 08/11] Bump pinned @coana-tech/cli to 15.10.4 (#291) * Bump pinned @coana-tech/cli to 15.10.4 * Rev version to 2.6.3, consolidating the unpublished 2.6.2 notes 2.6.1 shipped while this PR was in flight, and the 2.6.2 version bump on main was never published. Fold the 2.6.2 changelog entry, the Dependabot updates, and the dev/test dependency pinning into the 2.6.3 entry. Co-Authored-By: Claude Fable 5 * Trim dependency-update specifics in the 2.6.3 changelog entry Co-Authored-By: Claude Fable 5 --------- Co-authored-by: socket-pr-bot[bot] <294242679+socket-pr-bot[bot]@users.noreply.github.com> Co-authored-by: lelia <2418071+lelia@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- CHANGELOG.md | 17 ++++++++++++++++- docs/cli-reference.md | 2 +- pyproject.toml | 2 +- socketsecurity/__init__.py | 2 +- socketsecurity/core/tools/reachability.py | 6 +++--- uv.lock | 2 +- 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53adb48d..eabbddf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 2.6.2 +## 2.6.3 ### Fixed: SBOM fetch failures no longer produce empty reports @@ -13,6 +13,21 @@ instead of raising, and individual unparseable artifacts are skipped rather than failing the whole response. +### Changed: bump pinned @coana-tech/cli to 15.10.4 + +- Bumped the pinned reachability engine (`@coana-tech/cli`) from `15.10.3` to + `15.10.4`. See the [Coana changelogs](https://docs.coana.tech/changelogs) for + engine changes. + +### Changed: dependency updates and dev/test dependency pinning + +- Bumped pinned runtime dependencies (`beautifulsoup4` `4.15.0`, + `markdown` `3.10.3`). +- Pinned the `test` and `dev` dependency groups to exact versions, extending + the exact-pinning policy introduced in 2.6.0 to all dependency groups. +- Updated GitHub Actions used in CI workflows. No effect on the installed + CLI. + ## 2.6.1 ### Changed: scan comparison now polls the diff-scans endpoints diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 9f4ca88f..7d4f85c0 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -275,7 +275,7 @@ If you don't want to provide the Socket API Token every time then you can use th | Parameter | Required | Default | Description | |:---------------------------------|:---------|:--------|:---------------------------------------------------------------------------------------------------------------------------| | `--reach` | False | False | Enable reachability analysis to identify which vulnerable functions are actually called by your code. Creates a full application reachability scan (`scan_type=socket_tier1`). | -| `--reach-version` | False | 15.10.3 | Version of @coana-tech/cli to use. Defaults to the pinned version that ships with this CLI release, so the engine only changes when you upgrade the Socket CLI. Pass `latest` to always use the newest published version (opt-in auto-update), or an explicit version (e.g. `1.2.3`) to pin it. | +| `--reach-version` | False | 15.10.4 | Version of @coana-tech/cli to use. Defaults to the pinned version that ships with this CLI release, so the engine only changes when you upgrade the Socket CLI. Pass `latest` to always use the newest published version (opt-in auto-update), or an explicit version (e.g. `1.2.3`) to pin it. | | `--reach-analysis-timeout` | False | 10m | Timeout for each reachability analysis run, e.g. `90s`, `10m` or `1h`. Omitted by default, so coana applies its own default (`10m`). Alias: `--reach-timeout` | | `--reach-analysis-memory-limit` | False | 8GB | Memory limit for each reachability analysis run, e.g. `512MB` or `8GB`. Omitted by default, so coana applies its own default (`8GB`). Alias: `--reach-memory-limit` | | `--reach-concurrency` | False | 1 | Control parallel analysis execution (must be >= 1). Omitted by default, so coana applies its own default. | diff --git a/pyproject.toml b/pyproject.toml index 8275fdb2..f325c07b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.6.2" +version = "2.6.3" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index c4c8bad0..88ac8237 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,3 +1,3 @@ __author__ = 'socket.dev' -__version__ = '2.6.2' +__version__ = '2.6.3' USER_AGENT = f'SocketPythonCLI/{__version__}' diff --git a/socketsecurity/core/tools/reachability.py b/socketsecurity/core/tools/reachability.py index 34d79099..d0fdd07e 100644 --- a/socketsecurity/core/tools/reachability.py +++ b/socketsecurity/core/tools/reachability.py @@ -18,7 +18,7 @@ # Pinned @coana-tech/cli version. Bumped deliberately per Python CLI release so the # reachability engine version only changes through a standard pip upgrade (advance notice). # Pass --reach-version latest to opt into the newest published version instead. -DEFAULT_COANA_CLI_VERSION: Final = "15.10.3" +DEFAULT_COANA_CLI_VERSION: Final = "15.10.4" # Resolved @coana-tech/cli script paths from the npm-install fallback, keyed by version. # Lives for the process lifetime so repeated fallback invocations install only once @@ -55,7 +55,7 @@ def __init__(self, sdk: socketdev, api_token: str): def _resolve_coana_package_spec(self, version: Optional[str] = None) -> str: """ - Resolve the @coana-tech/cli package spec to run (e.g. '@coana-tech/cli@15.10.3'). + Resolve the @coana-tech/cli package spec to run (e.g. '@coana-tech/cli@15.10.4'). Args: version: Coana CLI version to use. @@ -64,7 +64,7 @@ def _resolve_coana_package_spec(self, version: Optional[str] = None) -> str: - '': that exact version. Returns: - str: The package specifier to use with npx (e.g. '@coana-tech/cli@15.10.3'). + str: The package specifier to use with npx (e.g. '@coana-tech/cli@15.10.4'). """ return f"@coana-tech/cli@{self._resolve_coana_version(version)}" diff --git a/uv.lock b/uv.lock index 58516703..ffc1224d 100644 --- a/uv.lock +++ b/uv.lock @@ -1282,7 +1282,7 @@ wheels = [ [[package]] name = "socketsecurity" -version = "2.6.2" +version = "2.6.3" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, From 0d3937d71257f068ec89bfd7ae8a8e43b433fc7f Mon Sep 17 00:00:00 2001 From: "socket-pr-bot[bot]" <294242679+socket-pr-bot[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:29:33 -0700 Subject: [PATCH 09/11] Bump pinned @coana-tech/cli to 15.10.13 (#300) * Bump pinned @coana-tech/cli to 15.10.13 * Bump pinned GitPython to 3.1.59 GitPython 3.1.57 is affected by six advisories fixed in 3.1.58 (published 2026-08-07) and five more fixed in 3.1.59 (published 2026-08-10). Pin the latest so the pip-audit gate stays green once the newer advisories propagate to the audit databases. None of the affected GitPython APIs are used by this CLI. Co-Authored-By: Claude Fable 5 Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> --------- Signed-off-by: lelia <2418071+lelia@users.noreply.github.com> Co-authored-by: socket-pr-bot[bot] <294242679+socket-pr-bot[bot]@users.noreply.github.com> Co-authored-by: lelia <2418071+lelia@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- CHANGELOG.md | 14 ++++++++++++++ docs/cli-reference.md | 2 +- pyproject.toml | 4 ++-- socketsecurity/__init__.py | 2 +- socketsecurity/core/tools/reachability.py | 6 +++--- uv.lock | 10 +++++----- 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eabbddf5..c87837da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 2.6.4 + +### Changed: bump pinned @coana-tech/cli to 15.10.13 + +- Bumped the pinned reachability engine (`@coana-tech/cli`) from `15.10.4` to + `15.10.13`. See the [Coana changelogs](https://docs.coana.tech/changelogs) for + engine changes. + +### Changed: bump pinned GitPython to 3.1.59 + +- Bumped `GitPython` from `3.1.57` to `3.1.59`, picking up the security fixes + released in GitPython 3.1.58 and 3.1.59. None of the affected GitPython APIs + are used by this CLI. + ## 2.6.3 ### Fixed: SBOM fetch failures no longer produce empty reports diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 7d4f85c0..49be8dfa 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -275,7 +275,7 @@ If you don't want to provide the Socket API Token every time then you can use th | Parameter | Required | Default | Description | |:---------------------------------|:---------|:--------|:---------------------------------------------------------------------------------------------------------------------------| | `--reach` | False | False | Enable reachability analysis to identify which vulnerable functions are actually called by your code. Creates a full application reachability scan (`scan_type=socket_tier1`). | -| `--reach-version` | False | 15.10.4 | Version of @coana-tech/cli to use. Defaults to the pinned version that ships with this CLI release, so the engine only changes when you upgrade the Socket CLI. Pass `latest` to always use the newest published version (opt-in auto-update), or an explicit version (e.g. `1.2.3`) to pin it. | +| `--reach-version` | False | 15.10.13 | Version of @coana-tech/cli to use. Defaults to the pinned version that ships with this CLI release, so the engine only changes when you upgrade the Socket CLI. Pass `latest` to always use the newest published version (opt-in auto-update), or an explicit version (e.g. `1.2.3`) to pin it. | | `--reach-analysis-timeout` | False | 10m | Timeout for each reachability analysis run, e.g. `90s`, `10m` or `1h`. Omitted by default, so coana applies its own default (`10m`). Alias: `--reach-timeout` | | `--reach-analysis-memory-limit` | False | 8GB | Memory limit for each reachability analysis run, e.g. `512MB` or `8GB`. Omitted by default, so coana applies its own default (`8GB`). Alias: `--reach-memory-limit` | | `--reach-concurrency` | False | 1 | Control parallel analysis execution (must be >= 1). Omitted by default, so coana applies its own default. | diff --git a/pyproject.toml b/pyproject.toml index f325c07b..313a8fae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,14 +6,14 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.6.3" +version = "2.6.4" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ "requests==2.34.2", "mdutils==1.8.1", "prettytable==3.18.0", - "GitPython==3.1.57", + "GitPython==3.1.59", "packaging==26.2", "python-dotenv==1.2.2", "socketdev==3.5.0", diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 88ac8237..9d9963d7 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,3 +1,3 @@ __author__ = 'socket.dev' -__version__ = '2.6.3' +__version__ = '2.6.4' USER_AGENT = f'SocketPythonCLI/{__version__}' diff --git a/socketsecurity/core/tools/reachability.py b/socketsecurity/core/tools/reachability.py index d0fdd07e..37458c3d 100644 --- a/socketsecurity/core/tools/reachability.py +++ b/socketsecurity/core/tools/reachability.py @@ -18,7 +18,7 @@ # Pinned @coana-tech/cli version. Bumped deliberately per Python CLI release so the # reachability engine version only changes through a standard pip upgrade (advance notice). # Pass --reach-version latest to opt into the newest published version instead. -DEFAULT_COANA_CLI_VERSION: Final = "15.10.4" +DEFAULT_COANA_CLI_VERSION: Final = "15.10.13" # Resolved @coana-tech/cli script paths from the npm-install fallback, keyed by version. # Lives for the process lifetime so repeated fallback invocations install only once @@ -55,7 +55,7 @@ def __init__(self, sdk: socketdev, api_token: str): def _resolve_coana_package_spec(self, version: Optional[str] = None) -> str: """ - Resolve the @coana-tech/cli package spec to run (e.g. '@coana-tech/cli@15.10.4'). + Resolve the @coana-tech/cli package spec to run (e.g. '@coana-tech/cli@15.10.13'). Args: version: Coana CLI version to use. @@ -64,7 +64,7 @@ def _resolve_coana_package_spec(self, version: Optional[str] = None) -> str: - '': that exact version. Returns: - str: The package specifier to use with npx (e.g. '@coana-tech/cli@15.10.4'). + str: The package specifier to use with npx (e.g. '@coana-tech/cli@15.10.13'). """ return f"@coana-tech/cli@{self._resolve_coana_version(version)}" diff --git a/uv.lock b/uv.lock index ffc1224d..8fd80913 100644 --- a/uv.lock +++ b/uv.lock @@ -574,14 +574,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.57" +version = "3.1.59" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ba/0d/132ed135c871b6bf91adf16a0e43797cd535b81d4973b5d09291c54fc5ee/gitpython-3.1.57.tar.gz", hash = "sha256:c493ec57c0ef6b19743798b6a5af859c71814b524e7e6f97baa2f8e658961488", size = 225898, upload-time = "2026-07-26T07:33:26.351Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/dc/126b28e76b24a9268ba931ad3e012f71ebdadf62fd9f17758f7074bb0b20/gitpython-3.1.59.tar.gz", hash = "sha256:0a1475cfdc38a5bfba1a3e9a4a9da52a39749ecec322b772915c019f94e5b7e4", size = 230445, upload-time = "2026-08-10T12:03:20.271Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/6e/2139de986d9c7c3ac86f1f8be43858ce90bdfe2f7175e6c80c650ba15242/gitpython-3.1.57-py3-none-any.whl", hash = "sha256:4ccf7d73c10f5c9e76043fbb2675ac5a1b3ff5b41e648f56bcbed5f63792ecaf", size = 217151, upload-time = "2026-07-26T07:33:24.838Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ed/ae57eb7d344f43f87b74b3a281ead6ec7d6394eef72a7b1dcb28dd089550/gitpython-3.1.59-py3-none-any.whl", hash = "sha256:67a82f537384578643624c8b2c531938a9b82be431663e575dcf638526631d4c", size = 220996, upload-time = "2026-08-10T12:03:18.804Z" }, ] [[package]] @@ -1282,7 +1282,7 @@ wheels = [ [[package]] name = "socketsecurity" -version = "2.6.3" +version = "2.6.4" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, @@ -1324,7 +1324,7 @@ requires-dist = [ { name = "beautifulsoup4", specifier = "==4.15.0" }, { name = "brotli", marker = "platform_python_implementation == 'CPython'", specifier = "==1.2.0" }, { name = "brotlicffi", marker = "platform_python_implementation != 'CPython'", specifier = "==1.2.0.1" }, - { name = "gitpython", specifier = "==3.1.57" }, + { name = "gitpython", specifier = "==3.1.59" }, { name = "hatch", marker = "extra == 'dev'", specifier = "==1.17.1" }, { name = "markdown", specifier = "==3.10.3" }, { name = "mdutils", specifier = "==1.8.1" }, From 13651d762b7147b751480810e53cb79b9554ea58 Mon Sep 17 00:00:00 2001 From: "socket-pr-bot[bot]" <294242679+socket-pr-bot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:42:10 +0200 Subject: [PATCH 10/11] Bump pinned @coana-tech/cli to 15.10.16 (#308) Co-authored-by: socket-pr-bot[bot] <294242679+socket-pr-bot[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 ++++++++ docs/cli-reference.md | 2 +- pyproject.toml | 2 +- socketsecurity/__init__.py | 2 +- socketsecurity/core/tools/reachability.py | 6 +++--- uv.lock | 2 +- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c87837da..ecd3585d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 2.6.5 + +### Changed: bump pinned @coana-tech/cli to 15.10.16 + +- Bumped the pinned reachability engine (`@coana-tech/cli`) from `15.10.13` to + `15.10.16`. See the [Coana changelogs](https://docs.coana.tech/changelogs) for + engine changes. + ## 2.6.4 ### Changed: bump pinned @coana-tech/cli to 15.10.13 diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 49be8dfa..736fe8be 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -275,7 +275,7 @@ If you don't want to provide the Socket API Token every time then you can use th | Parameter | Required | Default | Description | |:---------------------------------|:---------|:--------|:---------------------------------------------------------------------------------------------------------------------------| | `--reach` | False | False | Enable reachability analysis to identify which vulnerable functions are actually called by your code. Creates a full application reachability scan (`scan_type=socket_tier1`). | -| `--reach-version` | False | 15.10.13 | Version of @coana-tech/cli to use. Defaults to the pinned version that ships with this CLI release, so the engine only changes when you upgrade the Socket CLI. Pass `latest` to always use the newest published version (opt-in auto-update), or an explicit version (e.g. `1.2.3`) to pin it. | +| `--reach-version` | False | 15.10.16 | Version of @coana-tech/cli to use. Defaults to the pinned version that ships with this CLI release, so the engine only changes when you upgrade the Socket CLI. Pass `latest` to always use the newest published version (opt-in auto-update), or an explicit version (e.g. `1.2.3`) to pin it. | | `--reach-analysis-timeout` | False | 10m | Timeout for each reachability analysis run, e.g. `90s`, `10m` or `1h`. Omitted by default, so coana applies its own default (`10m`). Alias: `--reach-timeout` | | `--reach-analysis-memory-limit` | False | 8GB | Memory limit for each reachability analysis run, e.g. `512MB` or `8GB`. Omitted by default, so coana applies its own default (`8GB`). Alias: `--reach-memory-limit` | | `--reach-concurrency` | False | 1 | Control parallel analysis execution (must be >= 1). Omitted by default, so coana applies its own default. | diff --git a/pyproject.toml b/pyproject.toml index 313a8fae..48ae5907 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.6.4" +version = "2.6.5" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 9d9963d7..5875d7cb 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,3 +1,3 @@ __author__ = 'socket.dev' -__version__ = '2.6.4' +__version__ = '2.6.5' USER_AGENT = f'SocketPythonCLI/{__version__}' diff --git a/socketsecurity/core/tools/reachability.py b/socketsecurity/core/tools/reachability.py index 37458c3d..dc9ac74f 100644 --- a/socketsecurity/core/tools/reachability.py +++ b/socketsecurity/core/tools/reachability.py @@ -18,7 +18,7 @@ # Pinned @coana-tech/cli version. Bumped deliberately per Python CLI release so the # reachability engine version only changes through a standard pip upgrade (advance notice). # Pass --reach-version latest to opt into the newest published version instead. -DEFAULT_COANA_CLI_VERSION: Final = "15.10.13" +DEFAULT_COANA_CLI_VERSION: Final = "15.10.16" # Resolved @coana-tech/cli script paths from the npm-install fallback, keyed by version. # Lives for the process lifetime so repeated fallback invocations install only once @@ -55,7 +55,7 @@ def __init__(self, sdk: socketdev, api_token: str): def _resolve_coana_package_spec(self, version: Optional[str] = None) -> str: """ - Resolve the @coana-tech/cli package spec to run (e.g. '@coana-tech/cli@15.10.13'). + Resolve the @coana-tech/cli package spec to run (e.g. '@coana-tech/cli@15.10.16'). Args: version: Coana CLI version to use. @@ -64,7 +64,7 @@ def _resolve_coana_package_spec(self, version: Optional[str] = None) -> str: - '': that exact version. Returns: - str: The package specifier to use with npx (e.g. '@coana-tech/cli@15.10.13'). + str: The package specifier to use with npx (e.g. '@coana-tech/cli@15.10.16'). """ return f"@coana-tech/cli@{self._resolve_coana_version(version)}" diff --git a/uv.lock b/uv.lock index 8fd80913..156b9936 100644 --- a/uv.lock +++ b/uv.lock @@ -1282,7 +1282,7 @@ wheels = [ [[package]] name = "socketsecurity" -version = "2.6.4" +version = "2.6.5" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, From 699a9a1d471b05cc4c9c8a53fc8b9ed9a046a15e Mon Sep 17 00:00:00 2001 From: lelia <2418071+lelia@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:06:16 -0400 Subject: [PATCH 11/11] Reduce scan startup time in large repositories (#301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(core): discover manifests in a single filesystem walk find_files() started a separate recursive rglob traversal for every expanded manifest pattern, so a scan re-walked each root once per pattern and only filtered excluded directories after descending into them. Replace that with one os.walk() per scan root: - Expand and case-fold all active patterns once, then match in memory. - Prune excluded directories, including .git, before descending. - Reject non-manifests on the basename alone (one set lookup plus one compiled glob alternation) before building a relative path or running a path match. - Cache supported manifest patterns per Core instance, but only when the API lookup succeeds, so a transient failure does not pin the run to the smaller local fallback pattern set. - Emit INFO durations for organization setup, pattern retrieval and discovery, with files/directories visited, directories pruned and manifests found. Matching behaviour is unchanged apart from intentionally excluding .git metadata. Adds parity tests against the previous rglob implementation for every built-in ecosystem and pattern, covering case-insensitivity, brace expansion, nested patterns, dot-directories, exclusions, inclusions, symlinks, excluded ecosystems, multiple roots, sorting and deduplication, plus an opt-in benchmark that asserts old/new result equality on a synthetic large-monorepo fixture. Ref: CE-379 * perf(git): fetch only the refs a comparison needs Git.__init__() ran `git fetch --all` on every invocation, pulling every remote branch and tag before changed-file detection even began. Resolve commit and branch metadata locally instead, and for pull-request comparisons prefer refs already present in the checkout, fetching a single base or head ref only when it is missing. Also recognises Buildkite's native BUILDKITE_COMMIT, BUILDKITE_BRANCH, BUILDKITE_PULL_REQUEST and BUILDKITE_PULL_REQUEST_BASE_BRANCH so Buildkite jobs can calculate a complete base-to-head changed-file range without mapping their environment onto GitHub Actions variable names. Buildkite is checked before GitHub because some pipelines deliberately export GitHub-compatible variables. Adds INFO durations for Git initialisation, changed-file detection and each fetch, including the ref requested and why. Existing GitHub Actions, GitLab CI, Bitbucket Pipelines and local behaviour is preserved; tests cover local-ref preference, absence of an unconditional fetch, the targeted-fetch fallback, all four CI providers, and non-PR and detached-HEAD execution. Ref: CE-379 * feat(buildkite): derive GitHub comment context natively `--scm github` read its configuration solely from GITHUB_* variables, so Buildkite users had to shim every one of them to get PR comments. Fall back to Buildkite's own variables when the GITHUB_* equivalents are absent: PR number, commit, branch, checkout path, commit message, build creator, and owner/repository parsed from BUILDKITE_REPO (preferring the pipeline repository over a contributor's fork). Explicit GITHUB_* and PR_NUMBER values still take priority, and GitHub Enterprise remains configurable via GITHUB_API_URL. A running Buildkite PR build maps to the supported `synchronize` comment path, and a non-PR build maps to `push`, so event routing is unchanged. Default-branch detection requires an actual branch name rather than treating two unset variables as a match, which would otherwise mark any build as the default branch and overwrite the repository baseline. Ref: CE-379 * refactor(cli): reuse sub-path discovery results and clarify scan routing The --sub-path routing pre-check walked every selected path to decide whether any manifests existed, then discarded the result so scan creation walked the same paths again. Retain and reuse it. Apply --excluded-ecosystems before the pre-check rather than after, so every find_files() call in a run sees the same ecosystem filter. Add an INFO duration for CLI run registration, and replace the "No Manifest files changed" line with wording that describes the decision being made: no supported manifest was detected in the changed-file set, so a full report is created. Scan-routing semantics are unchanged. Ref: CE-379 * docs(changelog): note faster local scan setup for large repositories Ref: CE-379 * chore(release): bump version to 2.6.5 Bumped via .hooks/sync_version.py so __init__.py, pyproject.toml and uv.lock stay in sync, and moved the changelog entry under a 2.6.5 heading. Ref: CE-379 * fix(ci): build the Docker preview from the checked-out workspace The publish-docker job downloads the built wheel to ./dist, but the build step omitted `context`, so docker/build-push-action used its default Git context. Buildx then cloned the repository as the build context, where ./dist does not exist, and `COPY dist/socketsecurity-*.whl` failed with "lstat /dist: no such file or directory". Set `context: .` so the build uses the workspace the artifact was downloaded into. This also makes the job's existing trust boundary hold as documented: the context is now the default-branch checkout rather than the pull-request ref, so Dockerfile.preview is read from trusted code and the pull request still enters the image only through the built wheel. Pre-existing; the TestPyPI half of the workflow is unaffected. * ci(preview): build Docker previews for arm64 as well as amd64 The preview image was amd64-only while the release and stable images are built for linux/amd64,linux/arm64, so a preview tag could not stand in for socketdev/cli:latest on arm64 hosts without emulation. Match the release arch matrix and enable QEMU so the arm64 layer can be built on an amd64 runner. Previews are opt-in via label, so the extra build time is an acceptable tradeoff for making the tag a drop-in replacement. * perf(diff): tighten diff-scan poll ceiling and make its timing attributable A finished comparison could sit unobserved for up to 30s between polls, which is dead time on every PR job. Lower the ceiling to 10s: a multi-minute comparison costs roughly 2x the polls while cutting worst-case dead time to 10s. Diff scans now log their ID, poll count, and the wait before the final poll at INFO. Previously the ID was debug-only, so a slow comparison in a customer CI log could not be tied back to a server-side diff scan, and there was no way to tell backend comparison time apart from time the result spent ready-but-unpolled. Also document the diff-scans token scopes. A token missing them still completes the scan, silently falling back to the streaming comparison, which differs in both transport and payload (cached diff-scan responses always embed per-package license details; the streaming path requests a lean payload). Ref: CE-379 * feat(diff): log the diff report URL and cover discovery memory PR/MR runs logged the head and new scan IDs but no link to the result, so a CI log gave no way to reach the report. Log the diff report URL where it is computed, so every diff flow gets it rather than only the full-scan-only branches. Also add a regression test asserting manifest discovery's peak allocation stays bounded by the widest single directory and the result set rather than by repository size. Measured against the per-pattern rglob approach this replaced, on a tree of 59,300 files including one 50,000-entry directory: 3.25 MB peak vs 10.72 MB. os.walk keeps a list of names per directory where rglob materialised DirEntry objects and a Path per candidate, so the single-pass walk allocates strictly less. Ref: CE-379 * docs(diff): record verified cached diff-scan param behaviour Probed the live API against an existing diff scan to confirm what the polling path can and cannot ask for: - omit_license_details is ignored when cached=true, as the existing comment said. License fields remain in the response. - omit_unchanged IS honored and removes unchanged artifacts entirely, measured at ~1.1 KB per artifact (225,542 B -> 78,003 B when dropping 135 of 192 artifacts). Record why the CLI still does not send omit_unchanged: unchanged artifacts feed diff.unchanged_alerts, which create_security_comment_gitlab and the FOSSA compat issue list read unconditionally, not only under --strict-blocking. Omitting them would silently shrink those outputs, so this needs proper gating in its own change rather than a param tweak here. Ref: CE-379 * perf(diff): skip unchanged artifacts when no output reads them Cached diff-scan responses embed every unchanged artifact at roughly 1 KB each. On a large dependency tree that is nearly the whole response — measured at ~11 MB for a tree with ~10k unchanged packages — downloaded, deserialised into Package objects and then discarded on every pull request. omit_unchanged is honored by the API (unlike omit_license_details, which cached responses ignore), so request it whenever no enabled output reads that half of the comparison. Verified against the live API through the SDK: 192 artifacts -> 57. Every consumer is behind an opt-in flag, so the gate is centralised in Core._requires_unchanged_artifacts with the reasoning recorded there: - --strict-blocking blocks on pre-existing issues via diff.unchanged_alerts - --enable-gitlab-security includes them in the dependency scanning report - --generate-license enumerates diff.packages, which must list every dependency - --legal-format fossa reports all currently-present issues Diff.to_dict serialises them too but has no callers. When cli_config is absent the caller is unknown, so the full payload is kept. Tests parametrise over every flag in that list so a new reader of diff.unchanged_alerts or diff.packages cannot be added without also updating the gate. The completion log reports omit_unchanged so it is visible whether the optimisation engaged on a given run. Ref: CE-379 * chore(release): bump version to 2.6.6 * fix: always filter diff scan artifacts * fix: preserve directory-only manifest patterns Address peer review feedback by retaining pathlib.rglob trailing-slash semantics, trimming the release notes, and removing redundant implementation commentary. --- .github/workflows/pr-preview.yml | 12 +- CHANGELOG.md | 21 ++ benchmarks/manifest_discovery.py | 101 ++++++ docs/ci-cd.md | 13 + docs/troubleshooting.md | 29 ++ pyproject.toml | 2 +- socketsecurity/__init__.py | 2 +- socketsecurity/core/__init__.py | 401 +++++++++++++++++---- socketsecurity/core/git_interface.py | 271 ++++++++++---- socketsecurity/core/scm/github.py | 99 ++++- socketsecurity/core/streaming.py | 6 + socketsecurity/socketcli.py | 42 ++- tests/core/test_diff_scan_polling.py | 152 ++++++++ tests/unit/test_git_interface.py | 251 +++++++++++++ tests/unit/test_github_buildkite_config.py | 173 +++++++++ tests/unit/test_manifest_discovery.py | 349 ++++++++++++++++++ tests/unit/test_streaming.py | 15 +- uv.lock | 2 +- 18 files changed, 1768 insertions(+), 173 deletions(-) create mode 100644 benchmarks/manifest_discovery.py create mode 100644 tests/unit/test_git_interface.py create mode 100644 tests/unit/test_github_buildkite_config.py create mode 100644 tests/unit/test_manifest_discovery.py diff --git a/.github/workflows/pr-preview.yml b/.github/workflows/pr-preview.yml index 3a2ab1a9..d3ef1c8b 100644 --- a/.github/workflows/pr-preview.yml +++ b/.github/workflows/pr-preview.yml @@ -254,17 +254,25 @@ jobs: - name: Set up Docker publishing uses: ./.github/actions/setup-docker with: - enable-qemu: "false" + # QEMU is required to build the arm64 layer on an amd64 runner. + enable-qemu: "true" dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push Docker preview uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: + # Build from the checked-out workspace, not the default Git context. The + # wheel is only present here as a downloaded artifact, and this also keeps + # Dockerfile.preview on the default-branch checkout rather than the PR ref. + context: . file: Dockerfile.preview push: true pull: true - platforms: linux/amd64 + # Match the arch matrix of the release and stable images so a preview is a + # drop-in replacement for socketdev/cli:latest on arm64 runners too. The + # arm64 layer builds under emulation, so expect roughly double the runtime. + platforms: linux/amd64,linux/arm64 tags: socketdev/cli:pr-${{ needs.context.outputs.pr_number }} build-args: | SDK_PREVIEW_VERSION=${{ inputs.sdk_preview_version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index ecd3585d..efa4fd2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## 2.6.6 + +### Changed: faster local scan setup for large repositories + +- Manifest discovery now uses one filesystem walk per scan root and prunes + excluded directories before descent. +- Pull request scans use local Git refs first and fetch only missing history. + Buildkite pull request metadata is now supported directly. +- Supported manifest patterns are cached per invocation, and discovered + manifests are reused during scan creation. +- Added timings for initialization, Git operations, changed-file detection, + pattern lookup, and manifest discovery. + +### Changed: scan comparisons no longer fetch unused artifacts + +- Scan comparisons omit unchanged artifacts unless an enabled output needs them. +- Diff scans poll more frequently and log identifiers and timing details for + easier troubleshooting. +- Documented the `diff-scans:create`, `diff-scans:list` and `full-scans:list` + token scopes required by the optimized comparison path. + ## 2.6.5 ### Changed: bump pinned @coana-tech/cli to 15.10.16 diff --git a/benchmarks/manifest_discovery.py b/benchmarks/manifest_discovery.py new file mode 100644 index 00000000..abc0e8bb --- /dev/null +++ b/benchmarks/manifest_discovery.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Compare legacy per-pattern rglob discovery with the single-pass walker. + +This is an opt-in developer benchmark, not a timing assertion in the test +suite. It creates a synthetic monorepo so filesystem or CI-agent changes do not +make regular tests flaky. +""" + +import argparse +import tempfile +import time +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +from socketsecurity.core import Core +from socketsecurity.core.socket_config import SocketConfig +from socketsecurity.core.utils import socket_globs + + +def seed_tree(root: Path, directories: int, files_per_directory: int) -> None: + for directory_index in range(directories): + directory = root / "packages" / f"package-{directory_index:05d}" + directory.mkdir(parents=True) + (directory / "package.json").write_text("{}\n", encoding="utf-8") + for file_index in range(files_per_directory): + (directory / f"source-{file_index:03d}.txt").write_text( + "not a manifest\n", + encoding="utf-8", + ) + + # These trees model the expensive directories that the new walker prunes + # before descent rather than visiting once for every manifest pattern. + for excluded in (".git/objects", "node_modules/example", ".venv/site-packages"): + directory = root / excluded + directory.mkdir(parents=True) + for index in range(files_per_directory * 10): + (directory / f"object-{index:05d}").write_text("x", encoding="utf-8") + + +def legacy_discover(root: Path) -> set[str]: + results = set() + excluded_dirs = SocketConfig(api_key="benchmark").excluded_dirs + for ecosystem_patterns in socket_globs.values(): + for details in ecosystem_patterns.values(): + for pattern in Core.expand_brace_pattern(details["pattern"]): + insensitive = Core.to_case_insensitive_regex(pattern) + for candidate in root.rglob(insensitive): + if candidate.is_file() and not Core.is_excluded( + str(candidate), + excluded_dirs, + ): + results.add(candidate.as_posix()) + return results + + +def new_core() -> Core: + core = Core.__new__(Core) + core.config = SocketConfig(api_key="benchmark") + core.cli_config = SimpleNamespace(exclude_paths=None) + core.sdk = MagicMock() + core._supported_patterns = socket_globs + return core + + +def timed(function, root: Path) -> tuple[set[str], float]: + start = time.perf_counter() + results = set(function(root)) + return results, time.perf_counter() - start + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--directories", type=int, default=500) + parser.add_argument("--files-per-directory", type=int, default=20) + args = parser.parse_args() + + with tempfile.TemporaryDirectory(prefix="socket-manifest-benchmark-") as temp: + root = Path(temp) + seed_tree(root, args.directories, args.files_per_directory) + legacy_results, legacy_seconds = timed(legacy_discover, root) + new_results, new_seconds = timed( + lambda path: new_core().find_files(str(path)), + root, + ) + + if legacy_results != new_results: + raise SystemExit( + "Manifest result mismatch: " + f"legacy={len(legacy_results)}, single_pass={len(new_results)}" + ) + + speedup = legacy_seconds / new_seconds if new_seconds else float("inf") + print(f"Manifests: {len(new_results)}") + print(f"Legacy per-pattern rglob: {legacy_seconds:.3f}s") + print(f"Single-pass walk: {new_seconds:.3f}s") + print(f"Speedup: {speedup:.1f}x") + + +if __name__ == "__main__": + main() diff --git a/docs/ci-cd.md b/docs/ci-cd.md index 66193f38..061d18ea 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -81,6 +81,19 @@ steps: SOCKET_SECURITY_API_TOKEN: "${SOCKET_SECURITY_API_TOKEN}" ``` +The CLI reads Buildkite's native `BUILDKITE_COMMIT`, `BUILDKITE_BRANCH`, +`BUILDKITE_PULL_REQUEST`, and `BUILDKITE_PULL_REQUEST_BASE_BRANCH` variables. +For pull-request builds, ensure the checkout contains the base branch and the +checked-out head commit. The CLI uses those local refs first and performs a +targeted fetch only when a required ref or its comparison history is missing; +it does not fetch every remote ref and tag during startup. + +When `--scm github` is used from Buildkite, the CLI also derives GitHub comment +context from `BUILDKITE_REPO`, `BUILDKITE_BUILD_CHECKOUT_PATH`, and the variables +above. Set `GH_API_TOKEN` to a GitHub token with the required repository access. +GitHub Enterprise users should also set `GITHUB_API_URL`; GitHub.com defaults to +`https://api.github.com`. + #### Merge-base baselines in Buildkite (dynamic pipelines) Notes for using `--base-commit-sha` (see the diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 531099cd..9d6d06df 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1,5 +1,34 @@ # Troubleshooting +## API token scopes for scan comparisons + +PR/MR runs compare the new scan against the repository's head scan. That comparison +first uses the diff-scans endpoints, which require an organization token with these +scopes in addition to whatever the scan itself needs: + +- `diff-scans:create` +- `diff-scans:list` +- `full-scans:list` + +If the token is missing them the scan still succeeds, so this is easy to miss. The only +signal is a warning, after which the CLI falls back to the older streaming comparison: + +``` +Diff scan comparison failed with APIAccessDenied(Insufficient permissions), falling back to the streaming scan comparison +``` + +Grant the scopes to use the diff-scans path. It polls with short, bounded requests +rather than holding one connection open while the backend computes, which is what lets +large comparisons survive network idle timeouts — notably Azure NAT gateways, which +reap idle connections after four minutes and surface as an intermittent +`ConnectionResetError`. + +The two paths can take noticeably different amounts of time on the same repository, +because cached diff-scan responses always embed per-package license details while the +streaming comparison requests a lean payload. On a large dependency tree, compare the +`Diff scan comparison ready in ...` timing against the `Diff Report Gathered in ...` +total before assuming either path is at fault. + ## Common gotchas - In diff scope, `--strict-blocking` uses a stricter alert set (`new + unchanged`) for blocking checks and diff-based output selection. diff --git a/pyproject.toml b/pyproject.toml index 48ae5907..6910e7cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "socketsecurity" -version = "2.6.5" +version = "2.6.6" requires-python = ">= 3.11" license = {"file" = "LICENSE"} dependencies = [ diff --git a/socketsecurity/__init__.py b/socketsecurity/__init__.py index 5875d7cb..4ee959b2 100644 --- a/socketsecurity/__init__.py +++ b/socketsecurity/__init__.py @@ -1,3 +1,3 @@ __author__ = 'socket.dev' -__version__ = '2.6.5' +__version__ = '2.6.6' USER_AGENT = f'SocketPythonCLI/{__version__}' diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 7be24858..daab5fdd 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -1,3 +1,7 @@ +import copy +import fnmatch +import importlib +import json import logging import os import random @@ -6,10 +10,9 @@ import tarfile import tempfile import time -import json from dataclasses import asdict -from pathlib import Path, PurePath -from typing import Dict, List, Tuple, Set, TYPE_CHECKING, Optional +from pathlib import PurePath +from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Set, Tuple if TYPE_CHECKING: from socketsecurity.config import CliConfig @@ -18,21 +21,15 @@ from socketdev.fullscans import DiffArtifacts, FullScanParams, SocketArtifact from socketdev.org import Organization from socketdev.repos import RepositoryInfo -import copy -from socketsecurity import __version__, USER_AGENT -from socketsecurity.core.classes import ( - Alert, - Diff, - FullScan, - Issue, - Package, - Purl -) + +from socketsecurity import USER_AGENT, __version__ +from socketsecurity.core.classes import Alert, Diff, FullScan, Issue, Package, Purl from socketsecurity.core.exceptions import APIResourceNotFound + +from .resource_utils import check_file_count_against_ulimit from .socket_config import SocketConfig from .utils import socket_globs -from .resource_utils import check_file_count_against_ulimit -import importlib + logging_std = importlib.import_module("logging") @@ -107,7 +104,7 @@ # completes; on expiry (or any other failure of this flow) the caller falls back to the # legacy streaming comparison rather than failing the scan outright. DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS = 5.0 -DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS = 30.0 +DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS = 10.0 DIFF_SCAN_POLL_BACKOFF_MULTIPLIER = 1.5 DIFF_SCAN_POLL_TIMEOUT_SECONDS = 30 * 60.0 @@ -125,6 +122,30 @@ def _humanize_alert_type(alert_type: str) -> str: return " ".join(part[:1].upper() + part[1:] for part in parts if part) +class ManifestPatterns(NamedTuple): + """Manifest patterns prepared once per scan root, case-folded for matching. + + The first three fields are the authoritative matchers used by + Core._matches_manifest_pattern. The candidate_* fields are a prefilter over + basenames alone: a manifest-discovery walk visits every file in the repository + but only a few hundred are manifests, so rejecting a name up front avoids + building a relative path and running a path match for the rest. The prefilter's + globs are pre-compiled into one alternation so the cost per rejected file stays + flat as the API's pattern list grows. + """ + + literal_basenames: Set[str] + basename_globs: List[str] + path_globs: List[str] + candidate_basenames: Set[str] + candidate_basename_regex: Optional["re.Pattern"] + + @property + def is_empty(self) -> bool: + """True when every ecosystem was filtered out, so the walk can be skipped.""" + return not (self.literal_basenames or self.basename_globs or self.path_globs) + + class Core: """Main class for interacting with Socket Security API and processing scan results.""" @@ -146,7 +167,13 @@ def __init__(self, config: SocketConfig, sdk: socketdev, cli_config: Optional['C self.config = config self.sdk = sdk self.cli_config = cli_config + self._supported_patterns: Optional[Dict] = None + org_start_time = time.perf_counter() self.set_org_vars() + log.info( + "Organization initialization completed in " + f"{time.perf_counter() - org_start_time:.2f}s" + ) def set_org_vars(self) -> None: """Sets the main shared configuration variables for organization access.""" @@ -421,6 +448,112 @@ def format_bytes(bytes_value): except Exception as e: log.error(f"Failed to save manifest tar.gz to {output_path}: {e}") + @staticmethod + def _prepare_manifest_patterns( + patterns: Dict, + ecosystems: Optional[List[str]], + excluded_ecosystems: List[str] + ) -> "ManifestPatterns": + """Prepare case-folded manifest patterns for a single filesystem walk. + + Literal basenames are kept in a set for the common fast path. Basename + globs and path-shaped globs are kept separately so the latter retain + pathlib's path-segment-aware matching behavior. The candidate basename + collections are derived here so the walker can reject a file on its name + alone; see ManifestPatterns. + """ + included_ecosystems = set(ecosystems) if ecosystems is not None else None + excluded = set(excluded_ecosystems) + literal_basenames: Set[str] = set() + basename_globs: Set[str] = set() + path_globs: Set[str] = set() + + for ecosystem, ecosystem_patterns in patterns.items(): + if included_ecosystems is not None and ecosystem not in included_ecosystems: + continue + if ecosystem in excluded: + continue + log.debug(f"Scanning ecosystem: {ecosystem}") + for details in ecosystem_patterns.values(): + original_pattern = details["pattern"] + for expanded in Core.expand_brace_pattern(original_pattern): + normalized = expanded.replace("\\", "/").casefold() + if "/" in normalized: + path_globs.add(normalized) + elif any(character in normalized for character in "*?["): + basename_globs.add(normalized) + else: + literal_basenames.add(normalized) + + # PurePath.match compares pattern segments right to left, so a path-shaped glob + # can only match a file whose basename matches the glob's final segment. Folding + # those final segments into the basename prefilter lets the walk skip the path + # match for everything else. A trailing "/" is directory-only under the legacy + # rglob behavior, so its empty final segment intentionally admits no files. + candidate_basenames = set(literal_basenames) + candidate_basename_globs = set(basename_globs) + for pattern in path_globs: + final_segment = pattern.rsplit("/", 1)[-1] + if any(character in final_segment for character in "*?["): + candidate_basename_globs.add(final_segment) + else: + candidate_basenames.add(final_segment) + + return ManifestPatterns( + literal_basenames=literal_basenames, + basename_globs=sorted(basename_globs), + path_globs=sorted(path_globs), + candidate_basenames=candidate_basenames, + candidate_basename_regex=Core._compile_basename_globs(candidate_basename_globs), + ) + + @staticmethod + def _compile_basename_globs(globs: Set[str]) -> Optional["re.Pattern"]: + """Compile basename globs into a single alternation, or None if there are none. + + fnmatch.translate anchors the tail with ``\\Z`` and re.match anchors the head, + so each alternative matches exactly what fnmatch.fnmatchcase would. + """ + if not globs: + return None + return re.compile( + "|".join(f"(?:{fnmatch.translate(glob)})" for glob in sorted(globs)) + ) + + @staticmethod + def _basename_could_match(normalized_name: str, patterns: "ManifestPatterns") -> bool: + """Cheap prefilter: could a file with this basename match any manifest pattern? + + False is authoritative; True still has to be confirmed by + _matches_manifest_pattern against the scan-root-relative path. + """ + if normalized_name in patterns.candidate_basenames: + return True + return ( + patterns.candidate_basename_regex is not None + and patterns.candidate_basename_regex.match(normalized_name) is not None + ) + + @staticmethod + def _matches_manifest_pattern(relative_path: str, patterns: "ManifestPatterns") -> bool: + """Return whether a scan-root-relative path matches a manifest pattern.""" + normalized_path = relative_path.replace("\\", "/").casefold() + basename = normalized_path.rsplit("/", 1)[-1] + if basename in patterns.literal_basenames: + return True + if any(fnmatch.fnmatchcase(basename, pattern) for pattern in patterns.basename_globs): + return True + if not patterns.path_globs: + return False + + candidate = PurePath(normalized_path) + return any(candidate.match(pattern) for pattern in patterns.path_globs) + + @staticmethod + def _matches_excluded_directory(directory_name: str, excluded_dirs: Set[str]) -> bool: + """Match configured directory exclusions, including entries such as ``*.egg-info``.""" + return any(fnmatch.fnmatchcase(directory_name, pattern) for pattern in excluded_dirs) + def find_files(self, path: str, ecosystems: Optional[List[str]] = None) -> List[str]: """ Finds supported manifest files in the given path. @@ -432,8 +565,8 @@ def find_files(self, path: str, ecosystems: Optional[List[str]] = None) -> List[ Returns: List of found manifest file paths. """ - log.debug("Starting Find Files") - start_time = time.time() + log.debug("Starting manifest discovery") + start_time = time.perf_counter() files: Set[str] = set() # Unified --exclude-paths: filter discovered manifests by the same paths/globs that are @@ -447,50 +580,92 @@ def find_files(self, path: str, ecosystems: Optional[List[str]] = None) -> List[ exclude_paths = getattr(self.cli_config, "exclude_paths", None) if self.cli_config else None exclude_regexes = Core.compile_exclude_paths(exclude_paths) if exclude_paths else [] - # Get supported patterns from the API patterns = self.get_supported_patterns() + manifest_patterns = self._prepare_manifest_patterns( + patterns, + ecosystems, + self.config.excluded_ecosystems, + ) - for ecosystem in patterns: - # If ecosystems filter is provided, only include specified ecosystems - if ecosystems is not None and ecosystem not in ecosystems: - continue - if ecosystem in self.config.excluded_ecosystems: - continue - log.debug(f'Scanning ecosystem: {ecosystem}') - ecosystem_patterns = patterns[ecosystem] - for file_name in ecosystem_patterns: - original_pattern = ecosystem_patterns[file_name]["pattern"] - - # Expand brace patterns - expanded_patterns = Core.expand_brace_pattern(original_pattern) - - for pattern in expanded_patterns: - case_insensitive_pattern = Core.to_case_insensitive_regex(pattern) - - log.debug(f"Searching for pattern: {case_insensitive_pattern}") - glob_start = time.time() - - # Use pathlib.Path.rglob() instead of glob.glob() to properly match dotfiles/dotdirs - base_path = Path(path) - glob_files = base_path.rglob(case_insensitive_pattern) - - for glob_file in glob_files: - glob_file_str = str(glob_file) - if not os.path.isfile(glob_file_str): - continue - if Core.is_excluded(glob_file_str, self.config.excluded_dirs): - continue - if exclude_regexes: - rel = os.path.relpath(glob_file_str, path) - if Core.path_matches_exclude_regexes(rel, exclude_regexes): - continue - files.add(glob_file_str.replace("\\", "/")) - - glob_end = time.time() - log.debug(f"Globbing took {glob_end - glob_start:.4f} seconds") + if manifest_patterns.is_empty: + elapsed = time.perf_counter() - start_time + log.info( + "Manifest discovery completed in " + f"{elapsed:.2f}s: root={os.path.abspath(path)}, " + "directories_visited=0, directories_pruned=0, " + "files_visited=0, manifests_found=0" + ) + log.info("Total files found: 0") + return [] + + directories_visited = 0 + directories_pruned = 0 + files_visited = 0 + excluded_dirs = set(self.config.excluded_dirs) + + def handle_walk_error(error: OSError) -> None: + log.debug(f"Unable to inspect path during manifest discovery: {error}") + + for current_root, directory_names, file_names in os.walk( + path, + topdown=True, + followlinks=False, + onerror=handle_walk_error, + ): + directories_visited += 1 + + kept_directories = [] + for directory_name in directory_names: + if directory_name == ".git" or Core._matches_excluded_directory( + directory_name, + excluded_dirs, + ): + directories_pruned += 1 + continue + # Only --exclude-paths needs a scan-root-relative path, so build one + # lazily rather than for every directory in the repository. + if exclude_regexes: + relative_directory = os.path.relpath( + os.path.join(current_root, directory_name), + path, + ) + if Core.path_matches_exclude_regexes(relative_directory, exclude_regexes): + directories_pruned += 1 + continue + kept_directories.append(directory_name) + directory_names[:] = kept_directories + + files_visited += len(file_names) + for file_name in file_names: + # Reject on the basename first: os.walk already hands us the name, so + # non-manifests cost one set lookup instead of a relative path plus a + # path match. Exclusions are then only evaluated for real candidates. + if not Core._basename_could_match(file_name.casefold(), manifest_patterns): + continue + file_path = os.path.join(current_root, file_name) + relative_path = os.path.relpath(file_path, path) + if not Core._matches_manifest_pattern(relative_path, manifest_patterns): + continue + if exclude_regexes and Core.path_matches_exclude_regexes( + relative_path, + exclude_regexes, + ): + continue + if os.path.isfile(file_path): + files.add(file_path.replace("\\", "/")) file_list = sorted(files) file_count = len(file_list) + elapsed = time.perf_counter() - start_time + log.info( + "Manifest discovery completed in " + f"{elapsed:.2f}s: root={os.path.abspath(path)}, " + f"directories_visited={directories_visited}, " + f"directories_pruned={directories_pruned}, " + f"files_visited={files_visited}, manifests_found={file_count}" + ) + # Retain the established count-only message for log consumers while they + # transition to the stage-level timing above. log.info(f"Total files found: {file_count}") # Check if the number of manifest files might exceed ulimit -n @@ -532,19 +707,36 @@ def get_supported_patterns(self) -> Dict: Returns: Dictionary of supported file patterns with 'general' key removed """ + cached_patterns = getattr(self, "_supported_patterns", None) + if cached_patterns is not None: + log.debug("Using cached supported manifest patterns") + return cached_patterns + + start_time = time.perf_counter() response = self.sdk.report.supported() + source = "api" if not response: log.error("Failed to get supported patterns from API") - # Import the old patterns as fallback - from .utils import socket_globs - return socket_globs - - # Remove the 'general' key if it exists - if 'general' in response: - response.pop('general') - - # The response is already in the format we need - return response + response = socket_globs + source = "local-fallback" + + # Do not mutate the SDK response, which may be shared by its own cache. + patterns = { + ecosystem: ecosystem_patterns + for ecosystem, ecosystem_patterns in response.items() + if ecosystem != "general" + } + # Only cache a successful lookup. The local fallback covers far fewer ecosystems + # than the API, so one transient failure must not pin the rest of the run to it — + # has_manifest_files() runs before find_files() and would poison the cache. + if source == "api": + self._supported_patterns = patterns + elapsed = time.perf_counter() - start_time + log.info( + "Supported manifest patterns loaded in " + f"{elapsed:.2f}s: source={source}, ecosystems={len(patterns)}" + ) + return patterns def has_manifest_files(self, files: list) -> bool: """ @@ -627,7 +819,7 @@ def empty_head_scan_file() -> List[str]: temp_path = os.path.join(temp_dir, '.socket.facts.json') # Create the empty file - with open(temp_path, 'w') as f: + with open(temp_path, 'w'): pass # Creates an empty file log.debug(f"Created temporary empty file for baseline scan: {temp_path}") @@ -1392,7 +1584,16 @@ def get_diff_scan_artifacts( "Error creating or resolving diff scan: " f"unexpected response: {str(response_summary)[:500]}" ) - artifacts_dict = diff_scan.get("artifacts") + # Logged at INFO, not debug: this is the only identifier that ties a slow or + # failed comparison in a CI log back to a server-side diff scan, and it is + # needed even when the run later falls back to the streaming comparison. + log.info(f"Diff scan created: id={diff_scan_id}") + + # The create and list endpoints are metadata-only. Always fetch artifacts + # through GET below, even if an unexpected/legacy response happens to embed + # them, so omit_unchanged and the bounded cached-polling contract cannot be + # bypassed by an eager response. + artifacts_dict = None # cached=true is the polling contract (202 while computing, 200 when # ready). The API ignores omit_license_details when cached=true - cached @@ -1402,9 +1603,24 @@ def get_diff_scan_artifacts( # payload ever gets a response truncated on a huge dependency tree, # response.json() fails and the caller falls back to the legacy # streaming comparison, which still requests the lean payload. + # + # Verified against the live API: passing omit_license_details alongside + # cached=true leaves the license fields in the response, but omit_unchanged + # IS honored and drops the unchanged artifacts entirely (~1.1 KB each), so + # it is requested whenever no enabled output reads them. See + # _requires_unchanged_artifacts. poll_params = {"cached": "true"} - deadline = time.monotonic() + DIFF_SCAN_POLL_TIMEOUT_SECONDS + omit_unchanged = not self._requires_unchanged_artifacts() + if omit_unchanged: + poll_params["omit_unchanged"] = "true" + poll_start = time.monotonic() + deadline = poll_start + DIFF_SCAN_POLL_TIMEOUT_SECONDS interval = DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS + # Tracked so the completion log can separate backend compute time from time the + # result spent ready-but-unpolled: the wait before the final poll bounds the + # latter, which is otherwise invisible in a CI log. + polls = 0 + last_interval = 0.0 while artifacts_dict is None: try: response = self.sdk.diffscans.get(self.config.org_slug, diff_scan_id, params=poll_params) @@ -1418,6 +1634,7 @@ def get_diff_scan_artifacts( f"({type(error).__name__}), retrying in {interval:.0f}s" ) response = {"status": "processing"} + polls += 1 if response.get("status") != "processing": scan = response.get("diff_scan") or {} if scan.get("artifacts") is None: @@ -1425,6 +1642,12 @@ def get_diff_scan_artifacts( f"Error fetching diff scan {diff_scan_id}: unexpected response: {str(response)[:500]}" ) artifacts_dict = scan["artifacts"] + log.info( + "Diff scan comparison ready in " + f"{time.monotonic() - poll_start:.2f}s: id={diff_scan_id}, " + f"polls={polls}, wait_before_final_poll={last_interval:.0f}s, " + f"omit_unchanged={str(omit_unchanged).lower()}" + ) break if time.monotonic() >= deadline: raise Exception( @@ -1433,6 +1656,7 @@ def get_diff_scan_artifacts( ) log.debug(f"Diff scan {diff_scan_id} still processing, polling again in {interval:.0f}s") time.sleep(interval) + last_interval = interval interval = min(interval * DIFF_SCAN_POLL_BACKOFF_MULTIPLIER, DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS) return DiffArtifacts.from_dict({ @@ -1440,6 +1664,39 @@ def get_diff_scan_artifacts( for key in ("added", "removed", "unchanged", "replaced", "updated") }) + def _requires_unchanged_artifacts(self) -> bool: + """Whether any enabled output reads the unchanged half of a comparison. + + A cached diff-scan response embeds every unchanged artifact at roughly 1 KB + each, so on a large dependency tree they are almost the entire payload + (~11 MB for a tree of ~10k unchanged packages) even though most runs never + look at them. Every consumer is behind an opt-in flag: + + - ``--strict-blocking`` reads ``diff.unchanged_alerts`` to block on + pre-existing issues (socketcli, output, alert_selection, slack plugin). + - ``--enable-gitlab-security`` includes them in the GitLab dependency + scanning report (Messages.create_security_comment_gitlab). + - ``--generate-license`` enumerates ``diff.packages``, which must list every + dependency, not just the changed ones. + - ``--legal-format fossa`` reports all currently-present issues, matching + FOSSA's point-in-time snapshot semantics. + + ``Diff.to_dict`` also serializes them but has no callers. When cli_config is + absent the caller is unknown, so the full payload is kept. + + Keep this in sync with those consumers; test_unchanged_artifacts_gating + pins the list. + """ + config = self.cli_config + if config is None: + return True + return bool( + getattr(config, "strict_blocking", False) + or getattr(config, "enable_gitlab_security", False) + or getattr(config, "generate_license", False) + or getattr(config, "legal_format", "socket") == "fossa" + ) + def get_added_and_removed_packages( self, head_full_scan_id: str, @@ -1754,6 +2011,10 @@ def create_new_diff( else: diff.diff_url = diff.report_url + # PR/MR runs previously logged only the scan IDs, so a CI log had no link to the + # result. Logged here rather than at each call site so every diff flow gets it. + log.info(f"Diff report URL: {diff.diff_url}") + return diff def create_diff_report( diff --git a/socketsecurity/core/git_interface.py b/socketsecurity/core/git_interface.py index da614063..b3c53bdc 100644 --- a/socketsecurity/core/git_interface.py +++ b/socketsecurity/core/git_interface.py @@ -1,6 +1,7 @@ +import os import re +import time import urllib.parse -import os from git import Repo @@ -12,34 +13,33 @@ class Git: path: str def __init__(self, path: str): + initialization_start = time.perf_counter() self.path = path + self._fetched_ref_commits = {} self.ensure_safe_directory(path) self.repo = Repo(path) assert self.repo self.head = self.repo.head - - # Always fetch all remote refs to ensure branches exist for diffing - try: - self.repo.git.fetch('--all') - log.debug("Fetched all remote refs for diffing.") - except Exception as fetch_error: - log.debug(f"Failed to fetch all remote refs: {fetch_error}") # Use CI environment SHA if available, otherwise fall back to current HEAD commit github_sha = os.getenv('GITHUB_SHA') gitlab_sha = os.getenv('CI_COMMIT_SHA') bitbucket_sha = os.getenv('BITBUCKET_COMMIT') - ci_sha = github_sha or gitlab_sha or bitbucket_sha + buildkite_sha = os.getenv('BUILDKITE_COMMIT') + ci_commits = ( + ("BUILDKITE_COMMIT", buildkite_sha), + ("GITHUB_SHA", github_sha), + ("CI_COMMIT_SHA", gitlab_sha), + ("BITBUCKET_COMMIT", bitbucket_sha), + ) + env_source, ci_sha = next( + ((source, sha) for source, sha in ci_commits if sha), + (None, None), + ) if ci_sha: try: self.commit = self.repo.commit(ci_sha) - if github_sha: - env_source = "GITHUB_SHA" - elif gitlab_sha: - env_source = "CI_COMMIT_SHA" - else: - env_source = "BITBUCKET_COMMIT" log.debug(f"Using commit from {env_source}: {ci_sha}") except Exception as error: log.debug(f"Failed to get commit from CI environment: {error}") @@ -82,13 +82,19 @@ def __init__(self, path: str): # Bitbucket Pipelines variables bitbucket_branch = os.getenv('BITBUCKET_BRANCH') + + # Buildkite branch (the source branch for pull-request builds) + buildkite_branch = os.getenv('BUILDKITE_BRANCH') - # Select CI branch with priority: GitLab -> GitHub -> Bitbucket - ci_branch = gitlab_branch or github_branch or bitbucket_branch + # Prefer the native environment when Buildkite is driving the job. This + # also avoids requiring Buildkite users to emulate GitHub Actions vars. + ci_branch = buildkite_branch or gitlab_branch or github_branch or bitbucket_branch if ci_branch: self.branch = ci_branch - if gitlab_branch: + if buildkite_branch: + env_source = "Buildkite" + elif gitlab_branch: env_source = "GitLab CI" elif github_branch: env_source = "GitHub Actions" @@ -141,40 +147,39 @@ def __init__(self, path: str): self.commit_sha = self.commit.binsha self.commit_message = self.commit.message self.committer = self.commit.committer - # Detect changed files in PR/MR context for GitHub, GitLab, Bitbucket; fallback to git show + + # Detect changed files in PR/MR context, using local refs first and + # fetching only a required ref when the checkout does not contain it. + changed_files_start = time.perf_counter() self.show_files = [] detected = False - # GitHub Actions PR context + detection_source = "single-commit" + github_base_ref = os.getenv('GITHUB_BASE_REF') github_head_ref = os.getenv('GITHUB_HEAD_REF') github_event_name = os.getenv('GITHUB_EVENT_NAME') github_before_sha = os.getenv('GITHUB_EVENT_BEFORE') # previous commit for push github_sha = os.getenv('GITHUB_SHA') # current commit - if github_event_name == 'pull_request' and github_base_ref and github_head_ref: - try: - # Fetch both branches individually - self.repo.git.fetch('origin', github_base_ref) - self.repo.git.fetch('origin', github_head_ref) - # Try remote diff first - diff_range = f"origin/{github_base_ref}...origin/{github_head_ref}" - try: - diff_files = self.repo.git.diff('--name-only', diff_range) - self.show_files = diff_files.splitlines() - log.debug(f"Changed files detected via git diff (GitHub PR remote): {self.show_files}") - detected = True - except Exception as remote_error: - log.debug(f"Remote diff failed: {remote_error}") - # Try local branch diff - local_diff_range = f"{github_base_ref}...{github_head_ref}" - try: - diff_files = self.repo.git.diff('--name-only', local_diff_range) - self.show_files = diff_files.splitlines() - log.debug(f"Changed files detected via git diff (GitHub PR local): {self.show_files}") - detected = True - except Exception as local_error: - log.debug(f"Local diff failed: {local_error}") - except Exception as error: - log.debug(f"Failed to fetch branches or diff for GitHub PR: {error}") + + buildkite_pr = os.getenv('BUILDKITE_PULL_REQUEST') + buildkite_base_ref = os.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') + buildkite_head_ref = os.getenv('BUILDKITE_BRANCH') + if self._is_buildkite_pull_request(buildkite_pr) and buildkite_base_ref: + detected = self._detect_pull_request_changes( + provider="Buildkite", + base_ref=buildkite_base_ref, + head_ref=buildkite_head_ref, + ) + if detected: + detection_source = "buildkite-pr" + elif github_event_name == 'pull_request' and github_base_ref: + detected = self._detect_pull_request_changes( + provider="GitHub", + base_ref=github_base_ref, + head_ref=github_head_ref, + ) + if detected: + detection_source = "github-pr" # Commits to default branch (push events) elif github_event_name == 'push' and github_before_sha and github_sha: try: @@ -182,6 +187,7 @@ def __init__(self, path: str): self.show_files = diff_files.splitlines() log.debug(f"Changed files detected via git diff (GitHub push): {self.show_files}") detected = True + detection_source = "github-push" except Exception as error: log.debug(f"Failed to get changed files via git diff (GitHub push): {error}") elif github_event_name == 'push': @@ -189,6 +195,7 @@ def __init__(self, path: str): self.show_files = self.repo.git.show(self.commit, name_only=True, format="%n").splitlines() log.debug(f"Changed files detected via git show (GitHub push fallback): {self.show_files}") detected = True + detection_source = "github-push-fallback" except Exception as error: log.debug(f"Failed to get changed files via git show (GitHub push fallback): {error}") # GitLab CI Merge Request context @@ -196,15 +203,13 @@ def __init__(self, path: str): gitlab_target = os.getenv('CI_MERGE_REQUEST_TARGET_BRANCH_NAME') gitlab_source = os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME') if gitlab_target and gitlab_source: - try: - self.repo.git.fetch('origin', gitlab_target, gitlab_source) - diff_range = f"origin/{gitlab_target}...origin/{gitlab_source}" - diff_files = self.repo.git.diff('--name-only', diff_range) - self.show_files = diff_files.splitlines() - log.debug(f"Changed files detected via git diff (GitLab): {self.show_files}") - detected = True - except Exception as error: - log.debug(f"Failed to get changed files via git diff (GitLab): {error}") + detected = self._detect_pull_request_changes( + provider="GitLab", + base_ref=gitlab_target, + head_ref=gitlab_source, + ) + if detected: + detection_source = "gitlab-mr" # Bitbucket Pipelines PR context if not detected: bitbucket_pr_id = os.getenv('BITBUCKET_PR_ID') @@ -212,15 +217,13 @@ def __init__(self, path: str): bitbucket_dest = os.getenv('BITBUCKET_PR_DESTINATION_BRANCH') # BITBUCKET_BRANCH is the source branch in PR builds if bitbucket_pr_id and bitbucket_source and bitbucket_dest: - try: - self.repo.git.fetch('origin', bitbucket_dest, bitbucket_source) - diff_range = f"origin/{bitbucket_dest}...origin/{bitbucket_source}" - diff_files = self.repo.git.diff('--name-only', diff_range) - self.show_files = diff_files.splitlines() - log.debug(f"Changed files detected via git diff (Bitbucket): {self.show_files}") - detected = True - except Exception as error: - log.debug(f"Failed to get changed files via git diff (Bitbucket): {error}") + detected = self._detect_pull_request_changes( + provider="Bitbucket", + base_ref=bitbucket_dest, + head_ref=bitbucket_source, + ) + if detected: + detection_source = "bitbucket-pr" # Fallback to git show for single commit if not detected: # Check if this is a merge commit first @@ -233,20 +236,132 @@ def __init__(self, path: str): self.show_files = self.repo.git.show(self.commit, name_only=True, format="%n").splitlines() log.debug(f"Changed files detected via git show (merge commit fallback): {self.show_files}") detected = True + detection_source = "merge-commit-fallback" + if detected and detection_source == "single-commit": + detection_source = "merge-diff" else: # Regular single commit self.show_files = self.repo.git.show(self.commit, name_only=True, format="%n").splitlines() log.debug(f"Changed files detected via git show: {self.show_files}") detected = True + detection_source = "single-commit" self.changed_files = [] for item in self.show_files: if item != "": # Use relative path for glob matching self.changed_files.append(item) + + log.info( + "Changed-file detection completed in " + f"{time.perf_counter() - changed_files_start:.2f}s: " + f"source={detection_source}, files={len(self.changed_files)}" + ) # Determine if this commit is on the default branch # This considers both GitHub Actions detached HEAD and regular branch situations self.is_default_branch = self._is_commit_and_branch_default() + log.info( + "Git initialization completed in " + f"{time.perf_counter() - initialization_start:.2f}s" + ) + + @staticmethod + def _is_buildkite_pull_request(pull_request: str | None) -> bool: + return bool(pull_request and pull_request.casefold() != "false") + + def _resolve_ref(self, ref: str | None) -> str | None: + """Resolve a branch, tag, or SHA without accessing the network.""" + if not ref: + return None + if ref in self._fetched_ref_commits: + return self._fetched_ref_commits[ref] + + candidates = [ref] + if not ref.startswith("refs/"): + candidates = [f"origin/{ref}", ref] + for candidate in candidates: + try: + return self.repo.commit(candidate).hexsha + except Exception: + continue + return None + + def _fetch_ref(self, ref: str, reason: str) -> str | None: + """Fetch one required ref and return its commit without broadening scope.""" + if ref in self._fetched_ref_commits: + return self._fetched_ref_commits[ref] + + fetch_start = time.perf_counter() + try: + self.repo.git.fetch("origin", ref) + commit_sha = self.repo.commit("FETCH_HEAD").hexsha + self._fetched_ref_commits[ref] = commit_sha + log.info( + "Git fetch completed in " + f"{time.perf_counter() - fetch_start:.2f}s: " + f"remote=origin, ref={ref}, reason={reason}" + ) + return commit_sha + except Exception as error: + log.info( + "Git fetch failed in " + f"{time.perf_counter() - fetch_start:.2f}s: " + f"remote=origin, ref={ref}, reason={reason}" + ) + log.debug(f"Targeted fetch failed for {ref}: {error}") + return None + + def _detect_pull_request_changes( + self, + provider: str, + base_ref: str, + head_ref: str | None, + ) -> bool: + """Detect a full PR range locally, fetching only refs needed to complete it.""" + base_commit = self._resolve_ref(base_ref) + if base_commit is None: + base_commit = self._fetch_ref(base_ref, f"{provider} pull-request base ref missing") + if base_commit is None: + log.debug(f"Unable to resolve {provider} pull-request base ref: {base_ref}") + return False + + head_commit = self.commit.hexsha + diff_range = f"{base_commit}...{head_commit}" + try: + diff_files = self.repo.git.diff("--name-only", diff_range) + self.show_files = diff_files.splitlines() + log.debug( + f"Changed files detected via local git diff ({provider}): {self.show_files}" + ) + return True + except Exception as local_error: + log.debug(f"Local {provider} pull-request diff failed: {local_error}") + + # A shallow checkout can contain both tips but not their merge base. In + # that case refresh only the two relevant branch histories and retry. + base_commit = self._fetch_ref( + base_ref, + f"{provider} pull-request history incomplete", + ) or base_commit + if head_ref: + self._fetch_ref( + head_ref, + f"{provider} pull-request history incomplete", + ) + + try: + diff_files = self.repo.git.diff( + "--name-only", + f"{base_commit}...{head_commit}", + ) + self.show_files = diff_files.splitlines() + log.debug( + f"Changed files detected after targeted fetch ({provider}): {self.show_files}" + ) + return True + except Exception as retry_error: + log.debug(f"Targeted {provider} pull-request diff failed: {retry_error}") + return False def _is_commit_and_branch_default(self) -> bool: """ @@ -268,9 +383,29 @@ def _is_commit_and_branch_default(self) -> bool: gitlab_mr_branch = os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME') gitlab_default_branch = os.getenv('CI_DEFAULT_BRANCH', '') bitbucket_branch = os.getenv('BITBUCKET_BRANCH') + buildkite_branch = os.getenv('BUILDKITE_BRANCH') + buildkite_pr = os.getenv('BUILDKITE_PULL_REQUEST') + buildkite_default_branch = os.getenv('BUILDKITE_PIPELINE_DEFAULT_BRANCH') + # Handle Buildkite before GitHub because some Buildkite pipelines + # intentionally provide GitHub-compatible environment variables. + if buildkite_branch: + if self._is_buildkite_pull_request(buildkite_pr): + log.debug( + f"Processing Buildkite pull request from branch: {buildkite_branch}, " + "not default branch" + ) + return False + default_branch_name = buildkite_default_branch or self.get_default_branch_name() + is_default = buildkite_branch == default_branch_name + log.debug( + f"Buildkite branch: {buildkite_branch}, Default: {default_branch_name}, " + f"Is default: {is_default}" + ) + return is_default + # Handle GitHub Actions - if github_ref: + elif github_ref: log.debug(f"GitHub ref: {github_ref}") # Handle pull requests - they're not on the default branch @@ -483,7 +618,7 @@ def get_default_branch_name(self) -> str: if f'origin/{branch_name}' in [str(ref) for ref in self.repo.remotes.origin.refs]: log.debug(f"Using fallback default branch: {branch_name}") return branch_name - except: + except Exception: continue # Last fallback: assume 'main' @@ -505,12 +640,12 @@ def is_commit_on_default_branch(self) -> bool: # Try remote branch first default_branch_ref = self.repo.remotes.origin.refs[default_branch] default_branch_commit = default_branch_ref.commit - except: + except Exception: # Fallback to local branch try: default_branch_ref = self.repo.heads[default_branch] default_branch_commit = default_branch_ref.commit - except: + except Exception: log.debug(f"Could not find default branch '{default_branch}' locally or remotely") return False @@ -572,4 +707,4 @@ def ensure_safe_directory(path: str) -> None: else: log.debug(f"{abs_path} already present in git safe.directory config.") except Exception as safe_error: - log.debug(f"Failed to set safe.directory for git: {safe_error}") \ No newline at end of file + log.debug(f"Failed to set safe.directory for git: {safe_error}") diff --git a/socketsecurity/core/scm/github.py b/socketsecurity/core/scm/github.py index 7d5905d2..7504a46c 100644 --- a/socketsecurity/core/scm/github.py +++ b/socketsecurity/core/scm/github.py @@ -1,6 +1,7 @@ import json import os import sys +import urllib.parse from dataclasses import dataclass from git import Optional @@ -34,6 +35,31 @@ class GithubConfig: event_action: Optional[str] headers: dict + @staticmethod + def _repository_from_buildkite() -> tuple[str, str]: + """Return ``(owner, repository)`` from Buildkite's Git repository URL.""" + repository_url = ( + # Comments and statuses belong to the pipeline/base repository, + # not a contributor's fork from BUILDKITE_PULL_REQUEST_REPO. + os.getenv("BUILDKITE_REPO") + or os.getenv("BUILDKITE_PULL_REQUEST_REPO") + or "" + ).strip() + if not repository_url: + return "", "" + + if "://" in repository_url: + repository_path = urllib.parse.urlparse(repository_url).path + elif ":" in repository_url: + # SCP-style SSH URL: git@github.com:owner/repository.git + repository_path = repository_url.split(":", 1)[1] + else: + repository_path = repository_url + parts = repository_path.strip("/").removesuffix(".git").split("/") + if len(parts) < 2: + return "", "" + return parts[-2], parts[-1] + @classmethod def from_env(cls, pr_number: Optional[str] = None) -> 'GithubConfig': """Create config from environment variables with optional overrides""" @@ -42,12 +68,24 @@ def from_env(cls, pr_number: Optional[str] = None) -> 'GithubConfig': log.error("Unable to get Github API Token from GH_API_TOKEN") sys.exit(2) - # Use provided PR number if available, otherwise fall back to env var + is_buildkite = os.getenv("BUILDKITE") == "true" + buildkite_pr = os.getenv("BUILDKITE_PULL_REQUEST") + is_buildkite_pr = bool( + is_buildkite + and buildkite_pr + and buildkite_pr.casefold() != "false" + ) + + # Use explicit/GitHub-compatible values first, then native Buildkite PR context. pr_number = pr_number or os.getenv('PR_NUMBER') + if not pr_number and is_buildkite_pr: + pr_number = buildkite_pr # Add debug logging - sha = os.getenv('GITHUB_SHA', '') - log.debug(f"Loading SHA from GITHUB_SHA: {sha}") + sha = os.getenv('GITHUB_SHA') or ( + os.getenv("BUILDKITE_COMMIT", "") if is_buildkite else "" + ) + log.debug(f"Loading GitHub integration SHA: {sha}") event_action = os.getenv('EVENT_ACTION', None) if not event_action: event_path = os.getenv('GITHUB_EVENT_PATH') @@ -55,29 +93,66 @@ def from_env(cls, pr_number: Optional[str] = None) -> 'GithubConfig': with open(event_path, 'r') as f: event = json.load(f) event_action = event.get('action') + if not event_action and is_buildkite_pr: + # Buildkite provides the current PR state, not the originating + # GitHub webhook action. A running PR build is equivalent to the + # supported synchronize path for comment updates. + event_action = "synchronize" repository = os.getenv('GITHUB_REPOSITORY', '') owner = os.getenv('GITHUB_REPOSITORY_OWNER', '') if '/' in repository: owner = repository.split('/')[0] repository = repository.split('/')[1] + elif is_buildkite: + buildkite_owner, buildkite_repository = cls._repository_from_buildkite() + owner = owner or buildkite_owner + repository = repository or buildkite_repository default_branch_env = os.getenv('DEFAULT_BRANCH') # Consider the variable truthy if it exists and isn't explicitly 'false' - is_default = default_branch_env is not None and default_branch_env.lower() != 'false' + if default_branch_env is not None: + is_default = default_branch_env.lower() != 'false' + elif is_buildkite: + # Require a branch name: comparing two unset variables would otherwise report + # every build as the default branch and overwrite the repository's baseline. + buildkite_branch = os.getenv("BUILDKITE_BRANCH") + is_default = bool( + not is_buildkite_pr + and buildkite_branch + and buildkite_branch == os.getenv("BUILDKITE_PIPELINE_DEFAULT_BRANCH") + ) + else: + is_default = False + + event_name = os.getenv('GITHUB_EVENT_NAME', '') + if not event_name and is_buildkite: + event_name = "pull_request" if is_buildkite_pr else "push" return cls( - sha=os.getenv('GITHUB_SHA', ''), - api_url=os.getenv('GITHUB_API_URL', ''), - ref_type=os.getenv('GITHUB_REF_TYPE', ''), - event_name=os.getenv('GITHUB_EVENT_NAME', ''), - workspace=os.getenv('GITHUB_WORKSPACE', ''), + sha=sha, + api_url=os.getenv('GITHUB_API_URL') or ( + "https://api.github.com" if is_buildkite else "" + ), + ref_type=os.getenv('GITHUB_REF_TYPE') or ( + "branch" if is_buildkite else "" + ), + event_name=event_name, + workspace=os.getenv('GITHUB_WORKSPACE') or ( + os.getenv("BUILDKITE_BUILD_CHECKOUT_PATH", "") if is_buildkite else "" + ), repository=repository, - ref_name=os.getenv('GITHUB_REF_NAME', ''), + ref_name=os.getenv('GITHUB_REF_NAME') or ( + os.getenv("BUILDKITE_BRANCH", "") if is_buildkite else "" + ), default_branch=is_default, is_default_branch=is_default, pr_number=pr_number, pr_name=os.getenv('PR_NAME'), - commit_message=os.getenv('COMMIT_MESSAGE'), - actor=os.getenv('GITHUB_ACTOR', ''), + commit_message=os.getenv('COMMIT_MESSAGE') or ( + os.getenv("BUILDKITE_MESSAGE") if is_buildkite else None + ), + actor=os.getenv('GITHUB_ACTOR') or ( + os.getenv("BUILDKITE_BUILD_CREATOR", "") if is_buildkite else "" + ), env=os.getenv('GITHUB_ENV', ''), token=token, owner=owner, diff --git a/socketsecurity/core/streaming.py b/socketsecurity/core/streaming.py index 20b45eac..e6910b44 100644 --- a/socketsecurity/core/streaming.py +++ b/socketsecurity/core/streaming.py @@ -13,6 +13,7 @@ """ import logging +import time from typing import Optional from .cli_client import CliClient @@ -49,12 +50,17 @@ def set_report_run_id(self, report_run_id: Optional[str]) -> None: self._report_run_id = report_run_id def __enter__(self) -> "StreamingLogs": + registration_start = time.perf_counter() self._run_id = register_cli_run( self._client, client_version=self._client_version, upload_logs=self._upload_logs, ) cli_logger = self._loggers[0] + cli_logger.info( + "CLI run registration completed in " + f"{time.perf_counter() - registration_start:.2f}s" + ) if not self._run_id: cli_logger.debug("server log streaming not active for this run") return self diff --git a/socketsecurity/socketcli.py b/socketsecurity/socketcli.py index 0d8bcccb..24e8e966 100644 --- a/socketsecurity/socketcli.py +++ b/socketsecurity/socketcli.py @@ -207,6 +207,8 @@ def main_code(): if dirs_to_include: core.config.excluded_dirs = set(core.config.excluded_dirs) - dirs_to_include log.debug(f"Re-including normally-excluded directories in scan: {sorted(dirs_to_include)}") + if config.excluded_ecosystems: + core.config.excluded_ecosystems = list(config.excluded_ecosystems) # Check for required dependencies if reachability analysis is enabled if config.reach: @@ -292,6 +294,9 @@ def main_code(): facts_file_to_submit = None # Variable to track SBOM files to submit when using --reach-use-only-pregenerated-sboms sbom_files_to_submit = None + # Manifest results retained from the --sub-path routing pre-check. Reusing + # these avoids walking every selected sub-path again during scan creation. + discovered_scan_files = None # Git setup is_repo = False @@ -534,14 +539,18 @@ def main_code(): # Override file checking to look in the scan paths instead # Get manifest files from all scan paths try: - all_scan_files = [] + discovered_scan_files = [] for scan_path in scan_paths: scan_files = core.find_files(scan_path) - all_scan_files.extend(scan_files) - has_supported_files = len(all_scan_files) > 0 - log.debug(f"Found {len(all_scan_files)} manifest files across {len(scan_paths)} scan paths") + discovered_scan_files.extend(scan_files) + has_supported_files = len(discovered_scan_files) > 0 + log.debug( + f"Found {len(discovered_scan_files)} manifest files across " + f"{len(scan_paths)} scan paths" + ) except Exception as e: log.debug(f"Error finding files in scan paths: {e}") + discovered_scan_files = None has_supported_files = False # Case 3: If no supported files or files are empty, force API mode (no PR comments) @@ -564,8 +573,6 @@ def main_code(): org_slug = core.config.org_slug if config.repo_is_public: core.config.repo_visibility = "public" - if config.excluded_ecosystems and len(config.excluded_ecosystems) > 0: - core.config.excluded_ecosystems = config.excluded_ecosystems integration_type = config.integration_type integration_org_slug = config.integration_org_slug or org_slug try: @@ -613,6 +620,12 @@ def main_code(): diff.diff_url = "" diff.report_url = "" + scan_explicit_files = ( + sbom_files_to_submit + if sbom_files_to_submit is not None + else discovered_scan_files + ) + # Handle SCM-specific flows log.debug(f"Flow decision: scm={scm is not None}, force_diff_mode={force_diff_mode}, force_api_mode={force_api_mode}, enable_diff={config.enable_diff}") @@ -684,7 +697,7 @@ def _is_unprocessed(c): log.info("Push initiated flow") if scm.check_event_type() == "diff": log.info("Starting comment logic for PR/MR event") - diff = core.create_new_diff(scan_paths, params, no_change=should_skip_scan, save_files_list_path=config.save_submitted_files_list, save_manifest_tar_path=config.save_manifest_tar, base_paths=base_paths, explicit_files=sbom_files_to_submit) + diff = core.create_new_diff(scan_paths, params, no_change=should_skip_scan, save_files_list_path=config.save_submitted_files_list, save_manifest_tar_path=config.save_manifest_tar, base_paths=base_paths, explicit_files=scan_explicit_files) comments = scm.get_comments_for_pr() # FIXME: this overwrites diff.new_alerts, which was previously populated by Core.create_issue_alerts @@ -807,14 +820,14 @@ def _is_unprocessed(c): ) else: log.info("Starting non-PR/MR flow") - diff = core.create_new_diff(scan_paths, params, no_change=should_skip_scan, save_files_list_path=config.save_submitted_files_list, save_manifest_tar_path=config.save_manifest_tar, base_paths=base_paths, explicit_files=sbom_files_to_submit) + diff = core.create_new_diff(scan_paths, params, no_change=should_skip_scan, save_files_list_path=config.save_submitted_files_list, save_manifest_tar_path=config.save_manifest_tar, base_paths=base_paths, explicit_files=scan_explicit_files) output_handler.handle_output(diff) elif (config.enable_diff or force_diff_mode) and not force_api_mode: # New logic: --enable-diff or force_diff_mode (from --ignore-commit-files in git repos) forces diff mode log.info("Diff mode enabled without SCM integration") - diff = core.create_new_diff(scan_paths, params, no_change=should_skip_scan, save_files_list_path=config.save_submitted_files_list, save_manifest_tar_path=config.save_manifest_tar, base_paths=base_paths, explicit_files=sbom_files_to_submit) + diff = core.create_new_diff(scan_paths, params, no_change=should_skip_scan, save_files_list_path=config.save_submitted_files_list, save_manifest_tar_path=config.save_manifest_tar, base_paths=base_paths, explicit_files=scan_explicit_files) output_handler.handle_output(diff) elif (config.enable_diff or force_diff_mode) and force_api_mode: @@ -834,7 +847,7 @@ def _is_unprocessed(c): save_files_list_path=config.save_submitted_files_list, save_manifest_tar_path=config.save_manifest_tar, base_paths=base_paths, - explicit_files=sbom_files_to_submit + explicit_files=scan_explicit_files ) log.info(f"Full scan created with ID: {diff.id}") log.info(f"Full scan report URL: {diff.report_url}") @@ -842,7 +855,10 @@ def _is_unprocessed(c): else: if force_api_mode: - log.info("No Manifest files changed, creating Socket Report") + log.info( + "No supported manifest detected in the changed-file set; " + "creating a full Socket report" + ) serializable_params = { key: value if isinstance(value, (int, float, str, list, dict, bool, type(None))) else str(value) for key, value in params.__dict__.items() @@ -855,7 +871,7 @@ def _is_unprocessed(c): save_files_list_path=config.save_submitted_files_list, save_manifest_tar_path=config.save_manifest_tar, base_paths=base_paths, - explicit_files=sbom_files_to_submit + explicit_files=scan_explicit_files ) log.info(f"Full scan created with ID: {diff.id}") log.info(f"Full scan report URL: {diff.report_url}") @@ -868,7 +884,7 @@ def _is_unprocessed(c): save_files_list_path=config.save_submitted_files_list, save_manifest_tar_path=config.save_manifest_tar, base_paths=base_paths, - explicit_files=sbom_files_to_submit + explicit_files=scan_explicit_files ) output_handler.handle_output(diff) diff --git a/tests/core/test_diff_scan_polling.py b/tests/core/test_diff_scan_polling.py index be369429..d8c0e396 100644 --- a/tests/core/test_diff_scan_polling.py +++ b/tests/core/test_diff_scan_polling.py @@ -104,6 +104,63 @@ def test_duplicate_conflict_uses_cached_polling(core, diff_scan_get_response): assert len(artifacts.added) > 0 +def test_eager_create_artifacts_do_not_bypass_filtered_get(core, diff_scan_get_response): + """Unexpected create artifacts are ignored so the filtered GET remains canonical.""" + from types import SimpleNamespace + + core.cli_config = SimpleNamespace( + strict_blocking=False, + enable_gitlab_security=False, + generate_license=False, + legal_format="socket", + ) + core.sdk.diffscans.create_from_ids.return_value = { + "diff_scan": { + "id": "diff-scan-123", + "artifacts": diff_scan_get_response["diff_scan"]["artifacts"], + } + } + + core.get_diff_scan_artifacts("head", "new") + + core.sdk.diffscans.get.assert_called_once_with( + core.config.org_slug, + "diff-scan-123", + params={"cached": "true", "omit_unchanged": "true"}, + ) + + +def test_eager_list_artifacts_do_not_bypass_filtered_get(core, diff_scan_get_response): + """Unexpected duplicate-list artifacts cannot skip the filtered GET either.""" + from types import SimpleNamespace + + core.cli_config = SimpleNamespace( + strict_blocking=False, + enable_gitlab_security=False, + generate_license=False, + legal_format="socket", + ) + core.sdk.diffscans.create_from_ids.side_effect = APIFailure( + "duplicate", status_code=409 + ) + core.sdk.diffscans.list.return_value = { + "results": [ + { + "id": "existing-diff-scan", + "artifacts": diff_scan_get_response["diff_scan"]["artifacts"], + } + ], + } + + core.get_diff_scan_artifacts("head", "new") + + core.sdk.diffscans.get.assert_called_once_with( + core.config.org_slug, + "existing-diff-scan", + params={"cached": "true", "omit_unchanged": "true"}, + ) + + def test_fallback_to_streaming_diff_on_failure(core): """If the diff-scans flow fails (e.g. token missing the diff-scans scopes), the comparison falls back to the legacy streaming diff transparently.""" @@ -120,3 +177,98 @@ def test_fallback_to_streaming_diff_on_failure(core): ) assert "dp3" in added assert "dp2" in removed + + +def test_completion_log_reports_id_polls_and_final_wait( + core, diff_scan_get_response, no_sleep, caplog, monkeypatch +): + """The completion log must let a CI log separate backend compute time from the + time a finished comparison sat unnoticed between polls.""" + import logging + + monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS", 4.0) + monkeypatch.setattr(core_module, "DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS", 6.0) + processing = {"status": "processing", "id": "diff-scan-123"} + core.sdk.diffscans.get.side_effect = [processing, processing, diff_scan_get_response] + + with caplog.at_level(logging.INFO, logger="socketdev"): + core.get_diff_scan_artifacts("head", "new") + + messages = [record.message for record in caplog.records] + assert any("Diff scan created: id=" in message for message in messages) + ready = next(message for message in messages if "Diff scan comparison ready" in message) + assert "polls=3" in ready + # Waits were 4s then 6s (capped); the final poll followed the 6s wait, which is + # the upper bound on how long the result was ready before being observed. + assert "wait_before_final_poll=6s" in ready + + +def test_max_poll_interval_bounds_dead_time_for_ci_budgets(): + """A finished comparison is never left unobserved longer than the max interval.""" + assert core_module.DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS <= 10.0 + assert ( + core_module.DIFF_SCAN_POLL_INITIAL_INTERVAL_SECONDS + <= core_module.DIFF_SCAN_POLL_MAX_INTERVAL_SECONDS + ) + + +UNCHANGED_ARTIFACT_CONSUMERS = [ + # flag name, value that makes the flag active + ("strict_blocking", True), + ("enable_gitlab_security", True), + ("generate_license", True), + ("legal_format", "fossa"), +] + + +@pytest.mark.parametrize(("flag", "value"), UNCHANGED_ARTIFACT_CONSUMERS) +def test_unchanged_artifacts_gating(core, diff_scan_get_response, flag, value): + """Any output that reads unchanged artifacts must keep them in the response. + + This pins the consumer list in Core._requires_unchanged_artifacts: adding a new + reader of diff.unchanged_alerts or diff.packages without adding it here (and to + that method) would silently ship an empty result to that output. + """ + from types import SimpleNamespace + + defaults = {name: (False if name != "legal_format" else "socket") + for name, _ in UNCHANGED_ARTIFACT_CONSUMERS} + core.cli_config = SimpleNamespace(**{**defaults, flag: value}) + core.sdk.diffscans.get.side_effect = None + core.sdk.diffscans.get.return_value = diff_scan_get_response + + core.get_diff_scan_artifacts("head", "new") + + params = core.sdk.diffscans.get.call_args.kwargs["params"] + assert "omit_unchanged" not in params, f"{flag}={value} still needs unchanged artifacts" + + +def test_unchanged_artifacts_omitted_when_no_output_reads_them(core, diff_scan_get_response): + """With no such flag set, the ~1 KB-per-artifact unchanged half is not fetched.""" + from types import SimpleNamespace + + core.cli_config = SimpleNamespace( + strict_blocking=False, + enable_gitlab_security=False, + generate_license=False, + legal_format="socket", + ) + core.sdk.diffscans.get.side_effect = None + core.sdk.diffscans.get.return_value = diff_scan_get_response + + core.get_diff_scan_artifacts("head", "new") + + params = core.sdk.diffscans.get.call_args.kwargs["params"] + assert params["cached"] == "true" + assert params["omit_unchanged"] == "true" + + +def test_unknown_caller_keeps_full_payload(core, diff_scan_get_response): + """cli_config is optional; without it, do not assume unchanged is unused.""" + core.cli_config = None + core.sdk.diffscans.get.side_effect = None + core.sdk.diffscans.get.return_value = diff_scan_get_response + + core.get_diff_scan_artifacts("head", "new") + + assert "omit_unchanged" not in core.sdk.diffscans.get.call_args.kwargs["params"] diff --git a/tests/unit/test_git_interface.py b/tests/unit/test_git_interface.py new file mode 100644 index 00000000..a22cf634 --- /dev/null +++ b/tests/unit/test_git_interface.py @@ -0,0 +1,251 @@ +import logging +import subprocess +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from socketsecurity.core.git_interface import Git + +CI_ENVIRONMENT_VARIABLES = ( + "BUILDKITE", + "BUILDKITE_BRANCH", + "BUILDKITE_COMMIT", + "BUILDKITE_PIPELINE_DEFAULT_BRANCH", + "BUILDKITE_PULL_REQUEST", + "BUILDKITE_PULL_REQUEST_BASE_BRANCH", + "GITHUB_BASE_REF", + "GITHUB_EVENT_BEFORE", + "GITHUB_EVENT_NAME", + "GITHUB_HEAD_REF", + "GITHUB_REF", + "GITHUB_SHA", + "CI_COMMIT_BRANCH", + "CI_COMMIT_SHA", + "CI_DEFAULT_BRANCH", + "CI_MERGE_REQUEST_SOURCE_BRANCH_NAME", + "CI_MERGE_REQUEST_TARGET_BRANCH_NAME", + "BITBUCKET_BRANCH", + "BITBUCKET_COMMIT", + "BITBUCKET_PR_DESTINATION_BRANCH", + "BITBUCKET_PR_ID", +) + + +@pytest.fixture(autouse=True) +def clear_ci_environment(monkeypatch): + for variable in CI_ENVIRONMENT_VARIABLES: + monkeypatch.delenv(variable, raising=False) + + +def _git(path, *args): + return subprocess.run( + ["git", *args], + cwd=path, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +@pytest.fixture +def pull_request_repo(tmp_path): + path = tmp_path / "repo" + path.mkdir() + _git(path, "init", "-b", "main") + _git(path, "config", "user.name", "Socket Test") + _git(path, "config", "user.email", "socket@example.com") + (path / "README.md").write_text("base\n", encoding="utf-8") + _git(path, "add", "README.md") + _git(path, "commit", "-m", "base") + _git(path, "checkout", "-b", "feature") + (path / "package.json").write_text("{}\n", encoding="utf-8") + _git(path, "add", "package.json") + _git(path, "commit", "-m", "add manifest") + return path + + +@pytest.mark.parametrize( + ("environment", "expected_branch", "expected_source"), + [ + ( + { + "BUILDKITE": "true", + "BUILDKITE_BRANCH": "feature", + "BUILDKITE_PULL_REQUEST": "123", + "BUILDKITE_PULL_REQUEST_BASE_BRANCH": "main", + }, + "feature", + "buildkite-pr", + ), + ( + { + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_BASE_REF": "main", + "GITHUB_HEAD_REF": "feature", + "GITHUB_REF": "refs/pull/123/merge", + }, + "feature", + "github-pr", + ), + ( + { + "CI_MERGE_REQUEST_SOURCE_BRANCH_NAME": "feature", + "CI_MERGE_REQUEST_TARGET_BRANCH_NAME": "main", + }, + "feature", + "gitlab-mr", + ), + ( + { + "BITBUCKET_BRANCH": "feature", + "BITBUCKET_PR_DESTINATION_BRANCH": "main", + "BITBUCKET_PR_ID": "123", + }, + "feature", + "bitbucket-pr", + ), + ], +) +def test_pull_request_context_uses_local_refs_without_fetch( + pull_request_repo, monkeypatch, mocker, caplog, + environment, expected_branch, expected_source, +): + head_sha = _git(pull_request_repo, "rev-parse", "HEAD") + sha_variable = { + "buildkite-pr": "BUILDKITE_COMMIT", + "github-pr": "GITHUB_SHA", + "gitlab-mr": "CI_COMMIT_SHA", + "bitbucket-pr": "BITBUCKET_COMMIT", + }[expected_source] + environment[sha_variable] = head_sha + for name, value in environment.items(): + monkeypatch.setenv(name, value) + + fetch = mocker.patch.object( + Git, + "_fetch_ref", + side_effect=AssertionError("unexpected fetch"), + ) + mocker.patch.object(Git, "ensure_safe_directory") + + with caplog.at_level(logging.INFO, logger="socketdev"): + repository = Git(str(pull_request_repo)) + + assert repository.branch == expected_branch + assert repository.changed_files == ["package.json"] + assert repository.is_default_branch is False + fetch.assert_not_called() + assert any( + f"source={expected_source}" in record.message + for record in caplog.records + ) + assert any( + "Git initialization completed" in record.message + for record in caplog.records + ) + + +def test_buildkite_native_context_wins_over_github_compatibility_shims( + pull_request_repo, monkeypatch, mocker +): + head_sha = _git(pull_request_repo, "rev-parse", "HEAD") + monkeypatch.setenv("BUILDKITE", "true") + monkeypatch.setenv("BUILDKITE_BRANCH", "feature") + monkeypatch.setenv("BUILDKITE_COMMIT", head_sha) + monkeypatch.setenv("BUILDKITE_PULL_REQUEST", "123") + monkeypatch.setenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + monkeypatch.setenv("GITHUB_BASE_REF", "wrong-base") + monkeypatch.setenv("GITHUB_HEAD_REF", "wrong-head") + mocker.patch.object( + Git, + "_fetch_ref", + side_effect=AssertionError("unexpected fetch"), + ) + mocker.patch.object(Git, "ensure_safe_directory") + + repository = Git(str(pull_request_repo)) + + assert repository.changed_files == ["package.json"] + + +def test_regular_initialization_never_fetches_all(pull_request_repo, mocker): + fetch = mocker.patch.object( + Git, + "_fetch_ref", + side_effect=AssertionError("unexpected fetch"), + ) + mocker.patch.object(Git, "ensure_safe_directory") + + repository = Git(str(pull_request_repo)) + + assert repository.commit_str == _git(pull_request_repo, "rev-parse", "HEAD") + assert repository.changed_files == ["package.json"] + fetch.assert_not_called() + + +def test_detached_head_uses_buildkite_branch_and_commit(pull_request_repo, monkeypatch, mocker): + head_sha = _git(pull_request_repo, "rev-parse", "HEAD") + _git(pull_request_repo, "checkout", "--detach", head_sha) + monkeypatch.setenv("BUILDKITE", "true") + monkeypatch.setenv("BUILDKITE_BRANCH", "feature") + monkeypatch.setenv("BUILDKITE_COMMIT", head_sha) + monkeypatch.setenv("BUILDKITE_PULL_REQUEST", "123") + monkeypatch.setenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main") + mocker.patch.object( + Git, + "_fetch_ref", + side_effect=AssertionError("unexpected fetch"), + ) + mocker.patch.object(Git, "ensure_safe_directory") + + repository = Git(str(pull_request_repo)) + + assert repository.commit_str == head_sha + assert repository.branch == "feature" + assert repository.changed_files == ["package.json"] + + +def test_missing_base_ref_fetches_only_that_ref( + pull_request_repo, monkeypatch, mocker, caplog +): + head_sha = _git(pull_request_repo, "rev-parse", "HEAD") + monkeypatch.setenv("BUILDKITE_BRANCH", "feature") + monkeypatch.setenv("BUILDKITE_COMMIT", head_sha) + monkeypatch.setenv("BUILDKITE_PULL_REQUEST", "123") + monkeypatch.setenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "remote-main") + mocker.patch.object(Git, "ensure_safe_directory") + base_sha = _git(pull_request_repo, "rev-parse", "main") + fetch = mocker.patch.object(Git, "_fetch_ref", return_value=base_sha) + + with caplog.at_level(logging.INFO, logger="socketdev"): + repository = Git(str(pull_request_repo)) + + fetch.assert_called_once_with( + "remote-main", + "Buildkite pull-request base ref missing", + ) + assert repository.changed_files == ["package.json"] + + +def test_targeted_fetch_never_uses_all(): + repository = Git.__new__(Git) + repository.repo = MagicMock() + repository._fetched_ref_commits = {} + main_sha = "a" * 40 + repository.repo.commit.return_value = SimpleNamespace(hexsha=main_sha) + + result = repository._fetch_ref("main", "test") + + repository.repo.git.fetch.assert_called_once_with("origin", "main") + assert "--all" not in repository.repo.git.fetch.call_args.args + assert result == main_sha + + +@pytest.mark.parametrize( + ("value", "expected"), + [(None, False), ("", False), ("false", False), ("False", False), ("0", True), ("123", True)], +) +def test_buildkite_pull_request_detection(value, expected): + assert Git._is_buildkite_pull_request(value) is expected diff --git a/tests/unit/test_github_buildkite_config.py b/tests/unit/test_github_buildkite_config.py new file mode 100644 index 00000000..23f41507 --- /dev/null +++ b/tests/unit/test_github_buildkite_config.py @@ -0,0 +1,173 @@ +import pytest + +from socketsecurity.core.scm.github import Github, GithubConfig + +CONTEXT_VARIABLES = ( + "BUILDKITE", + "BUILDKITE_BRANCH", + "BUILDKITE_BUILD_CHECKOUT_PATH", + "BUILDKITE_BUILD_CREATOR", + "BUILDKITE_COMMIT", + "BUILDKITE_MESSAGE", + "BUILDKITE_PIPELINE_DEFAULT_BRANCH", + "BUILDKITE_PULL_REQUEST", + "BUILDKITE_PULL_REQUEST_REPO", + "BUILDKITE_REPO", + "DEFAULT_BRANCH", + "EVENT_ACTION", + "GH_API_TOKEN", + "GITHUB_ACTOR", + "GITHUB_API_URL", + "GITHUB_EVENT_NAME", + "GITHUB_EVENT_PATH", + "GITHUB_REF_NAME", + "GITHUB_REF_TYPE", + "GITHUB_REPOSITORY", + "GITHUB_REPOSITORY_OWNER", + "GITHUB_SHA", + "GITHUB_WORKSPACE", + "PR_NUMBER", +) + + +@pytest.fixture(autouse=True) +def clear_context(monkeypatch): + for variable in CONTEXT_VARIABLES: + monkeypatch.delenv(variable, raising=False) + monkeypatch.setenv("GH_API_TOKEN", "test-token") + + +def test_github_config_uses_native_buildkite_pull_request_context(monkeypatch): + values = { + "BUILDKITE": "true", + "BUILDKITE_BRANCH": "feature/socket", + "BUILDKITE_BUILD_CHECKOUT_PATH": "/workspace/repo", + "BUILDKITE_BUILD_CREATOR": "octocat", + "BUILDKITE_COMMIT": "a" * 40, + "BUILDKITE_MESSAGE": "Update dependencies", + "BUILDKITE_PIPELINE_DEFAULT_BRANCH": "main", + "BUILDKITE_PULL_REQUEST": "123", + "BUILDKITE_PULL_REQUEST_REPO": "git@github.com:acme/widgets.git", + "BUILDKITE_REPO": "git@github.com:acme/widgets.git", + } + for name, value in values.items(): + monkeypatch.setenv(name, value) + + config = GithubConfig.from_env() + + assert config.sha == "a" * 40 + assert config.api_url == "https://api.github.com" + assert config.ref_type == "branch" + assert config.event_name == "pull_request" + assert config.event_action == "synchronize" + assert config.workspace == "/workspace/repo" + assert config.owner == "acme" + assert config.repository == "widgets" + assert config.ref_name == "feature/socket" + assert config.pr_number == "123" + assert config.commit_message == "Update dependencies" + assert config.actor == "octocat" + assert config.is_default_branch is False + assert Github(client=object(), config=config).check_event_type() == "diff" + + +def test_buildkite_non_pr_build_uses_push_and_default_branch(monkeypatch): + values = { + "BUILDKITE": "true", + "BUILDKITE_BRANCH": "main", + "BUILDKITE_COMMIT": "b" * 40, + "BUILDKITE_PIPELINE_DEFAULT_BRANCH": "main", + "BUILDKITE_PULL_REQUEST": "false", + "BUILDKITE_REPO": "https://github.com/acme/widgets.git", + } + for name, value in values.items(): + monkeypatch.setenv(name, value) + + config = GithubConfig.from_env() + + assert config.event_name == "push" + assert config.pr_number is None + assert config.owner == "acme" + assert config.repository == "widgets" + assert config.is_default_branch is True + assert Github(client=object(), config=config).check_event_type() == "main" + + +@pytest.mark.parametrize( + "branch_variables", + [ + {}, + {"BUILDKITE_BRANCH": "feature/socket"}, + {"BUILDKITE_PIPELINE_DEFAULT_BRANCH": "main"}, + ], +) +def test_buildkite_default_branch_requires_a_matching_branch_name( + monkeypatch, branch_variables +): + """Absent branch context must not be read as 'this build is the default branch'.""" + monkeypatch.setenv("BUILDKITE", "true") + for name, value in branch_variables.items(): + monkeypatch.setenv(name, value) + + config = GithubConfig.from_env() + + assert config.is_default_branch is False + assert config.default_branch is False + + +def test_explicit_github_values_take_priority_in_buildkite(monkeypatch): + values = { + "BUILDKITE": "true", + "BUILDKITE_BRANCH": "buildkite-branch", + "BUILDKITE_COMMIT": "b" * 40, + "BUILDKITE_PULL_REQUEST": "123", + "BUILDKITE_REPO": "git@github.com:buildkite/repository.git", + "EVENT_ACTION": "opened", + "GITHUB_API_URL": "https://github.example/api/v3", + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_REF_NAME": "github-branch", + "GITHUB_REF_TYPE": "branch", + "GITHUB_REPOSITORY": "github/repository", + "GITHUB_SHA": "c" * 40, + "GITHUB_WORKSPACE": "/github/workspace", + "PR_NUMBER": "456", + } + for name, value in values.items(): + monkeypatch.setenv(name, value) + + config = GithubConfig.from_env() + + assert config.sha == "c" * 40 + assert config.api_url == "https://github.example/api/v3" + assert config.workspace == "/github/workspace" + assert config.owner == "github" + assert config.repository == "repository" + assert config.ref_name == "github-branch" + assert config.pr_number == "456" + assert config.event_action == "opened" + + +@pytest.mark.parametrize( + ("repository_url", "expected"), + [ + ("git@github.com:acme/widgets.git", ("acme", "widgets")), + ("https://github.com/acme/widgets.git", ("acme", "widgets")), + ("ssh://git@github.com/acme/widgets.git", ("acme", "widgets")), + ("", ("", "")), + ("not-a-repository", ("", "")), + ], +) +def test_buildkite_repository_url_parsing(monkeypatch, repository_url, expected): + monkeypatch.setenv("BUILDKITE_REPO", repository_url) + + assert GithubConfig._repository_from_buildkite() == expected + + +def test_buildkite_pipeline_repository_wins_over_pull_request_fork(monkeypatch): + monkeypatch.setenv("BUILDKITE_REPO", "git@github.com:acme/widgets.git") + monkeypatch.setenv( + "BUILDKITE_PULL_REQUEST_REPO", + "git@github.com:contributor/widgets.git", + ) + + assert GithubConfig._repository_from_buildkite() == ("acme", "widgets") diff --git a/tests/unit/test_manifest_discovery.py b/tests/unit/test_manifest_discovery.py new file mode 100644 index 00000000..2bdc2bbd --- /dev/null +++ b/tests/unit/test_manifest_discovery.py @@ -0,0 +1,349 @@ +import logging +import os +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from socketsecurity.core import Core +from socketsecurity.core.socket_config import SocketConfig, default_exclude_dirs +from socketsecurity.core.utils import socket_globs + + +def _make_core(*, patterns=socket_globs, excluded_dirs=None, exclude_paths=None): + core = Core.__new__(Core) + core.config = SocketConfig( + api_key="test-key", + excluded_dirs=set(default_exclude_dirs if excluded_dirs is None else excluded_dirs), + ) + core.cli_config = SimpleNamespace(exclude_paths=exclude_paths) + core.sdk = MagicMock() + core._supported_patterns = patterns + return core + + +def _write_files(root: Path, relative_paths): + for relative_path in relative_paths: + target = root / relative_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("test\n", encoding="utf-8") + + +def _relative_results(root: Path, results): + return {Path(result).relative_to(root).as_posix() for result in results} + + +ALL_PATTERN_EXAMPLES = { + "app.spdx.json", + "bom.json", + "nested/app-cdx.json", + "nested/app-cyclonedx.xml", + "package.json", + "nested/package-lock.json", + "npm-shrinkwrap.json", + "yarn.lock", + "pnpm-lock.yaml", + "pnpm-lock.yml", + "pnpm-workspace.yaml", + "pnpm-workspace.yml", + "bun.lock", + "bun.lockb", + "vlt-lock.json", + "PIPFILE", + "pyproject.toml", + "poetry.lock", + "requirements.txt", + "dev-requirements.txt", + "requirements-dev.txt", + "requirements_test.txt", + "requirements.frozen", + "requirements/base.txt", + "nested/requirements/constraints.txt", + "setup.py", + "go.mod", + "go.sum", + "pom.xml", + "src/Project.CSPROJ", + "Directory.Build.Props", + "build.targets", + "project.nuspec", + "nuget.CONFIG", + "packages.config", + "packages.lock.json", +} + + +def test_all_builtin_manifest_patterns_match_in_one_walk(tmp_path, mocker): + _write_files( + tmp_path, + ALL_PATTERN_EXAMPLES + | { + "README.md", + "requirements/deep/not-a-direct-child.txt", + "src/package.json.backup", + }, + ) + original_walk = os.walk + walk = mocker.patch("socketsecurity.core.os.walk", wraps=original_walk) + + found = _relative_results(tmp_path, _make_core().find_files(str(tmp_path))) + + assert found == ALL_PATTERN_EXAMPLES + walk.assert_called_once() + + +def test_single_walk_matches_legacy_rglob_results_for_builtin_patterns(tmp_path): + _write_files( + tmp_path, + ALL_PATTERN_EXAMPLES + | { + ".hidden/package.json", + "nested/Requirements.TXT", + "src/not-a-manifest.json", + }, + ) + core = _make_core(excluded_dirs=set()) + + legacy_results = set() + for ecosystem_patterns in socket_globs.values(): + for details in ecosystem_patterns.values(): + for expanded in Core.expand_brace_pattern(details["pattern"]): + case_insensitive = Core.to_case_insensitive_regex(expanded) + for result in tmp_path.rglob(case_insensitive): + if result.is_file(): + legacy_results.add(result.as_posix()) + + assert set(core.find_files(str(tmp_path))) == legacy_results + + +def test_directory_only_pattern_does_not_match_same_named_file(tmp_path): + """A trailing slash keeps pathlib.rglob's directory-only semantics.""" + _write_files( + tmp_path, + { + "manifests/package.json", + "nested/manifests", + }, + ) + patterns = { + "test": { + "directory-only": {"pattern": "manifests/"}, + }, + } + + assert _make_core(patterns=patterns).find_files(str(tmp_path)) == [] + + +def test_prunes_git_default_globs_and_exclude_paths_before_descent( + tmp_path, mocker, caplog +): + _write_files( + tmp_path, + { + "package.json", + ".git/objects/package.json", + "node_modules/pkg/package.json", + "generated.egg-info/package.json", + "legacy/nested/package.json", + ".hidden/package.json", + }, + ) + scanned_directories = [] + original_scandir = os.scandir + + def tracking_scandir(path): + scanned_directories.append(Path(path).relative_to(tmp_path).as_posix()) + return original_scandir(path) + + mocker.patch("socketsecurity.core.os.scandir", side_effect=tracking_scandir) + core = _make_core(exclude_paths=["legacy"]) + + with caplog.at_level(logging.INFO, logger="socketdev"): + found = _relative_results(tmp_path, core.find_files(str(tmp_path))) + + assert found == {"package.json", ".hidden/package.json"} + assert ".git" not in scanned_directories + assert "node_modules" not in scanned_directories + assert "generated.egg-info" not in scanned_directories + assert "legacy" not in scanned_directories + assert any( + "directories_pruned=4" in record.message + and "manifests_found=2" in record.message + for record in caplog.records + ) + + +def test_include_dirs_and_excluded_ecosystems_are_preserved(tmp_path): + _write_files( + tmp_path, + { + "build/package.json", + "build/requirements.txt", + "dist/package.json", + }, + ) + core = _make_core(excluded_dirs=set(default_exclude_dirs) - {"build"}) + core.config.excluded_ecosystems = ["npm"] + + found = _relative_results(tmp_path, core.find_files(str(tmp_path))) + + assert found == {"build/requirements.txt"} + + +def test_excluding_every_ecosystem_skips_the_filesystem_walk(tmp_path, mocker): + core = _make_core() + core.config.excluded_ecosystems = list(socket_globs) + walk = mocker.patch( + "socketsecurity.core.os.walk", + side_effect=AssertionError("unexpected walk"), + ) + + assert core.find_files(str(tmp_path)) == [] + walk.assert_not_called() + + +def test_symlinked_file_is_included_but_symlinked_directory_is_not_followed(tmp_path): + if not hasattr(os, "symlink"): + pytest.skip("symlinks are not supported") + + source_file = tmp_path / "source.txt" + source_file.write_text("{}", encoding="utf-8") + source_directory = tmp_path / "external" + _write_files(source_directory, {"package.json"}) + try: + (tmp_path / "package.json").symlink_to(source_file) + (tmp_path / "linked-directory").symlink_to(source_directory, target_is_directory=True) + except OSError as error: + pytest.skip(f"symlinks are unavailable: {error}") + + found = _relative_results(tmp_path, _make_core().find_files(str(tmp_path))) + + assert "package.json" in found + assert "linked-directory/package.json" not in found + assert "external/package.json" in found + + +def test_supported_patterns_are_cached_without_mutating_sdk_response(): + response = { + "general": {"ignored": {"pattern": "ignored"}}, + "npm": {"package.json": {"pattern": "package.json"}}, + } + core = _make_core(patterns=None) + core.sdk.report.supported.return_value = response + + first = core.get_supported_patterns() + second = core.get_supported_patterns() + + assert first is second + assert first == {"npm": {"package.json": {"pattern": "package.json"}}} + assert "general" in response + core.sdk.report.supported.assert_called_once_with() + + +def test_failed_pattern_lookup_is_not_cached(): + """A transient API failure must not pin the run to the smaller local fallback.""" + api_response = {"npm": {"package.json": {"pattern": "package.json"}}} + core = _make_core(patterns=None) + core.sdk.report.supported.side_effect = [None, api_response] + + fallback = core.get_supported_patterns() + assert set(fallback) == set(socket_globs) + + recovered = core.get_supported_patterns() + assert set(recovered) == {"npm"} + # The successful lookup is still cached, so the API is not re-queried again. + assert core.get_supported_patterns() is recovered + assert core.sdk.report.supported.call_count == 2 + + +def test_basename_prefilter_admits_every_supported_manifest(): + """The cheap prefilter must never reject a path the authoritative matcher accepts.""" + patterns = Core._prepare_manifest_patterns(socket_globs, None, []) + + for relative_path in ALL_PATTERN_EXAMPLES: + basename = relative_path.rsplit("/", 1)[-1].casefold() + assert Core._matches_manifest_pattern(relative_path, patterns), relative_path + assert Core._basename_could_match(basename, patterns), relative_path + + +def test_results_are_sorted_and_deduplicated_across_overlapping_patterns(tmp_path): + _write_files(tmp_path, {"z/package.json", "a/package.json"}) + overlapping_patterns = { + "npm": { + "literal": {"pattern": "package.json"}, + "wildcard": {"pattern": "package*.json"}, + } + } + + found = _make_core(patterns=overlapping_patterns).find_files(str(tmp_path)) + + assert found == sorted(found) + assert len(found) == 2 + + +def test_explicit_discovery_results_prevent_a_second_walk(tmp_path): + manifest = tmp_path / "package.json" + manifest.write_text("{}", encoding="utf-8") + core = _make_core() + core.config.org_slug = "example" + core.cli_config = None + core.find_files = MagicMock(side_effect=AssertionError("unexpected second walk")) + core.create_full_scan = MagicMock(return_value=SimpleNamespace(id="scan-123")) + params = MagicMock() + + diff = core.create_full_scan_with_report_url( + [str(tmp_path)], + params, + explicit_files=[manifest.as_posix()], + ) + + core.find_files.assert_not_called() + core.create_full_scan.assert_called_once_with( + [manifest.as_posix()], + params, + base_paths=None, + ) + assert diff.id == "scan-123" + + +def test_core_initialization_logs_organization_timing(caplog): + sdk = MagicMock() + sdk.org.get.return_value = { + "organizations": {"org-id": {"slug": "example"}}, + } + + with caplog.at_level(logging.INFO, logger="socketdev"): + core = Core(SocketConfig(api_key="test-key"), sdk) + + assert core.config.org_slug == "example" + assert any( + "Organization initialization completed" in record.message + for record in caplog.records + ) + + +def test_discovery_does_not_build_a_repository_sized_index(tmp_path): + """Peak memory must stay bounded by the widest directory and the result set, not + by repository size. This is the property that keeps discovery viable on small + runners; the per-pattern rglob approach it replaced allocated strictly more. + """ + import tracemalloc + + wide_directory = tmp_path / "wide" + wide_directory.mkdir() + for index in range(20000): + (wide_directory / f"source{index:05d}.ts").write_text("x", encoding="utf-8") + (tmp_path / "package.json").write_text("{}", encoding="utf-8") + + core = _make_core() + tracemalloc.start() + try: + found = core.find_files(str(tmp_path)) + _, peak_bytes = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert _relative_results(tmp_path, found) == {"package.json"} + # 20k files in one directory; a repo-sized index would be far larger than this. + assert peak_bytes < 8_000_000, f"peak allocation was {peak_bytes / 1e6:.1f} MB" diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py index 999b44a8..3b9cbd9b 100644 --- a/tests/unit/test_streaming.py +++ b/tests/unit/test_streaming.py @@ -18,14 +18,19 @@ def _make(**overrides): return StreamingLogs(**kwargs) -def test_setup_streaming_is_noop_when_register_fails(): +def test_setup_streaming_is_noop_when_register_fails(caplog): finalize_calls = [] - with patch("socketsecurity.core.streaming.register_cli_run", return_value=None), \ - patch("socketsecurity.core.streaming.finalize_cli_run", side_effect=lambda *a, **k: finalize_calls.append(k)): - with _make(cli_name="t-fail-cli", sdk_name="t-fail-sdk") as streaming: - assert isinstance(streaming, StreamingLogs) + with caplog.at_level(logging.INFO, logger="t-fail-cli"): + with patch("socketsecurity.core.streaming.register_cli_run", return_value=None), \ + patch("socketsecurity.core.streaming.finalize_cli_run", side_effect=lambda *a, **k: finalize_calls.append(k)): + with _make(cli_name="t-fail-cli", sdk_name="t-fail-sdk") as streaming: + assert isinstance(streaming, StreamingLogs) # No run was registered → finalize must not be called. assert finalize_calls == [] + assert any( + "CLI run registration completed" in record.message + for record in caplog.records + ) def test_clean_exit_reports_success(): diff --git a/uv.lock b/uv.lock index 156b9936..17326035 100644 --- a/uv.lock +++ b/uv.lock @@ -1282,7 +1282,7 @@ wheels = [ [[package]] name = "socketsecurity" -version = "2.6.5" +version = "2.6.6" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" },