From 27a99d42c3dd060e77ce9b9100acc0aeda8a53d3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 15 Jul 2026 00:27:30 +0000 Subject: [PATCH 01/22] docs: sync README --help and Neo4j schema for v1.0.0 --- README.md | 465 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 280 insertions(+), 185 deletions(-) diff --git a/README.md b/README.md index 5ee1d44..8c8acf9 100644 --- a/README.md +++ b/README.md @@ -160,191 +160,286 @@ $ canpy --help Static Analysis on Python source code using Jedi, PyCG and Tree sitter. -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ -│ --version Show the canpy version │ -│ and exit. │ -│ --input -i PATH Path to the project root │ -│ directory (not required │ -│ for --emit schema). │ -│ --output -o PATH Output directory for │ -│ artifacts. │ -│ --format -f [json|msgpack] Output format for --emit │ -│ json: json or msgpack. │ -│ [default: json] │ -│ --emit [json|neo4j|schema] Output target: json │ -│ (analysis.json, default) │ -│ | neo4j (graph.cypher or │ -│ live Bolt push) | schema │ -│ (the Neo4j schema.json │ -│ contract). │ -│ [default: json] │ -│ --app-name TEXT Logical application name │ -│ for the graph │ -│ :PyApplication anchor │ -│ (default: input dir │ -│ name). │ -│ --neo4j-uri TEXT Push the graph to a live │ -│ Neo4j over Bolt │ -│ (incremental); omit to │ -│ write graph.cypher. │ -│ [env var: NEO4J_URI] │ -│ --neo4j-user TEXT Neo4j username. │ -│ [env var: NEO4J_USERNAME] │ -│ [default: neo4j] │ -│ --neo4j-password TEXT Neo4j password. Prefer │ -│ the env var over the flag │ -│ (the flag is visible in │ -│ shell history / process │ -│ list). │ -│ [env var: NEO4J_PASSWORD] │ -│ [default: neo4j] │ -│ --neo4j-database TEXT Neo4j database name │ -│ (default: server │ -│ default). │ -│ [env var: NEO4J_DATABASE] │ -│ --analysis-level -a INTEGER RANGE [1<=x<=4] Analysis depth: 1=symbol │ -│ table+Jedi call graph, │ -│ 2=+PyCG call graph, │ -│ 3=+native intraprocedural │ -│ dataflow (CFG/PDG), │ -│ 4=+interprocedural SDG │ -│ (param/summary edges, │ -│ alias-aware DDG). │ -│ [default: 1] │ -│ --graphs TEXT Level 3+ only: │ -│ comma-separated │ -│ program-graph sections to │ -│ emit (cfg, dfg, pdg, │ -│ sdg). Default: │ -│ cfg,dfg,pdg. `dfg` emits │ -│ the PDG's data edges │ -│ only; `sdg` requires -a │ -│ 4. │ -│ [default: cfg,dfg,pdg] │ -│ --graph-field-depth INTEGER RANGE [x>=1] Level 3 only: k-limit on │ -│ access-path depth │ -│ (x.f.g.h with k=3 becomes │ -│ x.f.g.*). Mandatory bound │ -│ — it is what guarantees │ -│ the interprocedural │ -│ fixpoint terminates. │ -│ [default: 3] │ -│ --ray --no-ray Enable Ray for │ -│ distributed analysis. │ -│ [default: no-ray] │ -│ --eager --lazy Enable eager or lazy │ -│ analysis. Defaults to │ -│ lazy. │ -│ [default: lazy] │ -│ --skip-tests --include-tests Skip test files in │ -│ analysis. │ -│ [default: skip-tests] │ -│ --no-venv --venv Skip virtualenv creation │ -│ and dependency │ -│ installation; resolve │ -│ imports against the │ -│ ambient Python │ -│ environment instead. │ -│ [default: venv] │ -│ --file-name PATH Analyze only the │ -│ specified file (relative │ -│ to input directory). │ -│ --cache-dir -c PATH Directory to store │ -│ analysis cache. Defaults │ -│ to '.codeanalyzer' in the │ -│ input directory. │ -│ --clear-cache --keep-cache Clear cache after │ -│ analysis. By default, │ -│ cache is retained. │ -│ [default: keep-cache] │ -│ -v INTEGER Increase verbosity: -v, │ -│ -vv, -vvv │ -│ [default: 0] │ -│ --pycg-shard --no-pycg-shard Shard PyCG call-graph │ -│ analysis by Python │ -│ package (level 2 only). │ -│ When the project exceeds │ -│ the 500-file ceiling, │ -│ PyCG is run independently │ -│ per top-level package │ -│ with cross-package │ -│ imports treated as ghost │ -│ nodes. Without this flag, │ -│ projects over the ceiling │ -│ fall back to Jedi-only │ -│ edges. │ -│ [default: no-pycg-shard] │ -│ --pycg-shard-ceiling INTEGER RANGE [x>=1] Maximum files per shard │ -│ when --pycg-shard is │ -│ active (default 100). │ -│ Shards exceeding this │ -│ limit are skipped; their │ -│ call edges are omitted │ -│ from the call graph (Jedi │ -│ edges for those packages │ -│ are still included). │ -│ Lower values are safer │ -│ for packages with deep │ -│ class hierarchies or │ -│ heavy import graphs. │ -│ [default: 100] │ -│ --pycg-shard-timeout INTEGER RANGE [x>=0] Per-shard wall-clock │ -│ timeout in seconds when │ -│ --pycg-shard is active │ -│ (default 120). A shard │ -│ that exceeds this limit │ -│ is skipped gracefully. │ -│ PyCG's fixpoint is │ -│ bimodal: it either │ -│ converges quickly or │ -│ diverges indefinitely, so │ -│ the timeout acts as a │ -│ final safety net after │ -│ the file-count ceiling. │ -│ Set to 0 to disable. │ -│ POSIX only (macOS / │ -│ Linux); ignored on │ -│ Windows. │ -│ [default: 120] │ -│ --pycg-shard-strategy [jedi|package] How --pycg-shard groups │ -│ files (level 2 only). │ -│ 'jedi' (default) │ -│ partitions the Jedi │ -│ module-dependency graph │ -│ (SCC + Louvain) so │ -│ tightly-coupled modules │ -│ co-compute and few call │ -│ edges are severed between │ -│ shards; import cycles are │ -│ never split. 'package' │ -│ uses the legacy │ -│ one-shard-per-package-di… │ -│ grouping. │ -│ [default: jedi] │ -│ --pycg-max-iter INTEGER RANGE [x>=-1] Cap on PyCG's fixpoint │ -│ passes per shard/project │ -│ (level 2; default 50). │ -│ PyCG iterates until its │ -│ points-to state stops │ -│ changing, but its │ -│ access-path domain has no │ -│ convergence bound, so │ -│ heavy metaclass/mixin │ -│ code (e.g. an ORM) can │ -│ loop with each pass │ -│ costing seconds. The cap │ -│ returns a │ -│ sound-but-incomplete call │ -│ graph instead of looping │ -│ until the timeout kills │ -│ it. Set to -1 for PyCG's │ -│ unbounded │ -│ run-to-convergence │ -│ behaviour. │ -│ [default: 50] │ -│ --help Show this message and │ -│ exit. │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ --version Show the canpy │ +│ version and │ +│ exit. │ +│ --input -i PATH Path to the │ +│ project root │ +│ directory (not │ +│ required for │ +│ --emit schema). │ +│ --output -o PATH Output directory │ +│ for artifacts. │ +│ --format -f [json|msgpack] Output format │ +│ for --emit json: │ +│ json or msgpack. │ +│ [default: json] │ +│ --emit [json|neo4j|sche Output target: │ +│ ma] json │ +│ (analysis.json, │ +│ default) | neo4j │ +│ (graph.cypher or │ +│ live Bolt push) │ +│ | schema (the │ +│ Neo4j │ +│ schema.json │ +│ contract). │ +│ [default: json] │ +│ --app-name TEXT Logical │ +│ application name │ +│ for the graph │ +│ :PyApplication │ +│ anchor (default: │ +│ input dir name). │ +│ --neo4j-uri TEXT Push the graph │ +│ to a live Neo4j │ +│ over Bolt │ +│ (incremental); │ +│ omit to write │ +│ graph.cypher. │ +│ [env var: │ +│ NEO4J_URI] │ +│ --neo4j-user TEXT Neo4j username. │ +│ [env var: │ +│ NEO4J_USERNAME] │ +│ [default: neo4j] │ +│ --neo4j-password TEXT Neo4j password. │ +│ Prefer the env │ +│ var over the │ +│ flag (the flag │ +│ is visible in │ +│ shell history / │ +│ process list). │ +│ [env var: │ +│ NEO4J_PASSWORD] │ +│ [default: neo4j] │ +│ --neo4j-database TEXT Neo4j database │ +│ name (default: │ +│ server default). │ +│ [env var: │ +│ NEO4J_DATABASE] │ +│ --analysis-level -a INTEGER RANGE Analysis depth: │ +│ [1<=x<=4] 1=symbol │ +│ table+Jedi call │ +│ graph, 2=+PyCG │ +│ call graph, │ +│ 3=+native │ +│ intraprocedural │ +│ dataflow │ +│ (CFG/PDG), │ +│ 4=+interprocedu… │ +│ SDG │ +│ (param/summary │ +│ edges, │ +│ alias-aware │ +│ DDG). │ +│ [default: 1] │ +│ --graphs TEXT Level 3+ only: │ +│ comma-separated │ +│ program-graph │ +│ sections to emit │ +│ (cfg, dfg, pdg, │ +│ sdg). Default: │ +│ cfg,dfg,pdg. │ +│ `dfg` emits the │ +│ PDG's data edges │ +│ only; `sdg` │ +│ requires -a 4. │ +│ [default: │ +│ cfg,dfg,pdg] │ +│ --graph-field-de… INTEGER RANGE Level 3 only: │ +│ [x>=1] k-limit on │ +│ access-path │ +│ depth (x.f.g.h │ +│ with k=3 becomes │ +│ x.f.g.*). │ +│ Mandatory bound │ +│ — it is what │ +│ guarantees the │ +│ interprocedural │ +│ fixpoint │ +│ terminates. │ +│ [default: 3] │ +│ --ray --no-ray Enable Ray for │ +│ distributed │ +│ analysis. │ +│ [default: │ +│ no-ray] │ +│ --eager --lazy Enable eager or │ +│ lazy analysis. │ +│ Defaults to │ +│ lazy. │ +│ [default: lazy] │ +│ --skip-tests --include-tests Skip test files │ +│ in analysis. │ +│ [default: │ +│ skip-tests] │ +│ --no-venv --venv Skip virtualenv │ +│ creation and │ +│ dependency │ +│ installation; │ +│ resolve imports │ +│ against the │ +│ ambient Python │ +│ environment │ +│ instead. │ +│ [default: venv] │ +│ --file-name PATH Analyze only the │ +│ specified file │ +│ (relative to │ +│ input │ +│ directory). │ +│ --cache-dir -c PATH Directory to │ +│ store analysis │ +│ cache. Defaults │ +│ to │ +│ '.codeanalyzer' │ +│ in the input │ +│ directory. │ +│ --clear-cache --keep-cache Clear cache │ +│ after analysis. │ +│ By default, │ +│ cache is │ +│ retained. │ +│ [default: │ +│ keep-cache] │ +│ -v INTEGER Increase │ +│ verbosity: -v, │ +│ -vv, -vvv │ +│ [default: 0] │ +│ --pycg-shard --no-pycg-shard Shard PyCG │ +│ call-graph │ +│ analysis by │ +│ Python package │ +│ (level 2 only). │ +│ When the project │ +│ exceeds the │ +│ 500-file │ +│ ceiling, PyCG is │ +│ run │ +│ independently │ +│ per top-level │ +│ package with │ +│ cross-package │ +│ imports treated │ +│ as ghost nodes. │ +│ Without this │ +│ flag, projects │ +│ over the ceiling │ +│ fall back to │ +│ Jedi-only edges. │ +│ [default: │ +│ no-pycg-shard] │ +│ --pycg-shard-cei… INTEGER RANGE Maximum files │ +│ [x>=1] per shard when │ +│ --pycg-shard is │ +│ active (default │ +│ 100). Shards │ +│ exceeding this │ +│ limit are │ +│ skipped; their │ +│ call edges are │ +│ omitted from the │ +│ call graph (Jedi │ +│ edges for those │ +│ packages are │ +│ still included). │ +│ Lower values are │ +│ safer for │ +│ packages with │ +│ deep class │ +│ hierarchies or │ +│ heavy import │ +│ graphs. │ +│ [default: 100] │ +│ --pycg-shard-tim… INTEGER RANGE Per-shard │ +│ [x>=0] wall-clock │ +│ timeout in │ +│ seconds when │ +│ --pycg-shard is │ +│ active (default │ +│ 120). A shard │ +│ that exceeds │ +│ this limit is │ +│ skipped │ +│ gracefully. │ +│ PyCG's fixpoint │ +│ is bimodal: it │ +│ either converges │ +│ quickly or │ +│ diverges │ +│ indefinitely, so │ +│ the timeout acts │ +│ as a final │ +│ safety net after │ +│ the file-count │ +│ ceiling. Set to │ +│ 0 to disable. │ +│ POSIX only │ +│ (macOS / Linux); │ +│ ignored on │ +│ Windows. │ +│ [default: 120] │ +│ --pycg-shard-str… [jedi|package] How --pycg-shard │ +│ groups files │ +│ (level 2 only). │ +│ 'jedi' (default) │ +│ partitions the │ +│ Jedi │ +│ module-dependen… │ +│ graph (SCC + │ +│ Louvain) so │ +│ tightly-coupled │ +│ modules │ +│ co-compute and │ +│ few call edges │ +│ are severed │ +│ between shards; │ +│ import cycles │ +│ are never split. │ +│ 'package' uses │ +│ the legacy │ +│ one-shard-per-p… │ +│ grouping. │ +│ [default: jedi] │ +│ --pycg-max-iter INTEGER RANGE Cap on PyCG's │ +│ [x>=-1] fixpoint passes │ +│ per │ +│ shard/project │ +│ (level 2; │ +│ default 50). │ +│ PyCG iterates │ +│ until its │ +│ points-to state │ +│ stops changing, │ +│ but its │ +│ access-path │ +│ domain has no │ +│ convergence │ +│ bound, so heavy │ +│ metaclass/mixin │ +│ code (e.g. an │ +│ ORM) can loop │ +│ with each pass │ +│ costing seconds. │ +│ The cap returns │ +│ a │ +│ sound-but-incom… │ +│ call graph │ +│ instead of │ +│ looping until │ +│ the timeout │ +│ kills it. Set to │ +│ -1 for PyCG's │ +│ unbounded │ +│ run-to-converge… │ +│ behaviour. │ +│ [default: 50] │ +│ --help Show this │ +│ message and │ +│ exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ ``` From ae2df7473387af4e9174c22b21ae455b0e1237ae Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Tue, 14 Jul 2026 22:16:45 -0400 Subject: [PATCH 02/22] fix(determinism): byte-identical analysis output for identical runs (#99) + L2 audit gate (#87) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same flags on the same input now emit byte-identical analysis.json. Four systematic nondeterminism sources closed: - CLI re-execs once with PYTHONHASHSEED=0 (export the var to opt out or pin another seed): PyCG's capped fixpoint iterates hash-ordered sets keyed on module/access-path strings, so an unpinned per-interpreter seed shifted the iteration frontier arbitrarily — observed 1,527 vs 4,712 edges on identical input. Guarded so in-process invocations (Typer CliRunner in the test suite) are never exec'd; Ray workers inherit the pinned seed via an explicit ray.init runtime env. - The PyCG mini-project root is content-derived (sha1 of project + shard files) instead of a random mkdtemp — PyCG state keys on absolute module paths, so a fresh random suffix changed the analysis input every run. An exclusive sidecar flock serializes concurrent analyses of the same shard (a test suite and a manual run on one project would otherwise rmtree each other's tree mid-analysis); distinct projects hash to distinct roots. - Entry points sorted (was filesystem order); emitted call_graph canonically ordered by (src, dst) so identical edge sets always serialize identically. - Jedi inference candidates are tie-broken deterministically (sorted on (full_name, name), not set order) — a union-typed receiver previously resolved to a different member per run; NoneType results are dropped from return-type candidates (Jedi flaps between yielding {NoneType} and {} for the None arm of an Optional, and a real type in the union is the informative pick anyway). Verified: repeated -a 2 runs on the requests fixture are identical in every edge and all but one field; the residue is a single call site whose compiled introspection (an os.environ-derived dict) depends on jedi's helper subprocess surviving the run — environmental and upstream, documented on issue #99. Regression gate: test_l2_runs_are_byte_identical. Also lands the #87 acceptance gate (test_l2_audit_gate_callee_name_equality): >=95% of resolved non-constructor callsites must bind to the invoked attribute name, and no callsite may fall back to a class id when the class declares the exact method. The underlying anchoring fix shipped in 1.0.0. --- CHANGELOG.md | 21 ++++ codeanalyzer/__main__.py | 35 +++++++ codeanalyzer/core.py | 22 +++++ .../semantic_analysis/pycg/pycg_analysis.py | 73 ++++++++++++-- .../symbol_table_builder.py | 53 ++++++---- test/test_v2_l2.py | 97 +++++++++++++++++++ 6 files changed, 278 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 119a876..a1905b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- **Deterministic output** (#99): the same flags on the same input now emit byte-identical + `analysis.json`. Four independent nondeterminism sources were closed: + - the CLI re-execs once with `PYTHONHASHSEED=0` (export the variable to opt out or pin another + seed) — PyCG's capped fixpoint iterates hash-ordered sets, so an unpinned per-interpreter + seed shifted its frontier arbitrarily (observed 1,527 vs 4,712 edges on the same input); + Ray workers get the pinned seed via the runtime env; + - the PyCG mini-project root is content-derived instead of a random `mkdtemp` (PyCG state keys + on absolute module paths), with an exclusive sidecar lock so concurrent analyses of the same + project serialize instead of deleting each other's tree mid-run; + - entry points are sorted (was filesystem order) and the emitted `call_graph` is canonically + ordered by `(src, dst)`; + - Jedi inference candidates are tie-broken deterministically (sorted, not set order) — a + union-typed receiver (e.g. `IOBase | BufferedRandom | TextIOWrapper`) previously resolved to + a different member per run. + Regression gates: `test_l2_runs_are_byte_identical` plus the #87 audit gate below. +- **L2 audit gate** (#87 acceptance): ≥95% of resolved non-constructor callsites must bind to the + invoked attribute name, and no callsite may fall back to a class id when the class declares the + exact method (`test_l2_audit_gate_callee_name_equality`). The underlying anchoring fix shipped + in 1.0.0 via the v0.3.1 merge. + ## [1.0.0] - 2026-07-14 ### Added diff --git a/codeanalyzer/__main__.py b/codeanalyzer/__main__.py index 69ab67a..3f8ae54 100644 --- a/codeanalyzer/__main__.py +++ b/codeanalyzer/__main__.py @@ -1,9 +1,40 @@ +import os +import sys from importlib.metadata import version as _pkg_version, PackageNotFoundError from pathlib import Path from typing import Optional, Annotated import typer + +def _pin_hash_seed() -> None: + """Re-exec once with ``PYTHONHASHSEED=0`` unless the caller pinned one. + + PyCG's capped fixpoint (``--pycg-max-iter``) iterates hash-ordered sets + keyed on module/access-path strings, so an unpinned per-interpreter hash + seed makes the emitted L2+ call graph vary run to run (issue #99). The + seed cannot be set after interpreter start, hence the exec. Export + PYTHONHASHSEED (any value) to opt out or pin a different seed. + + Only fires when this process really is the CLI (canpy / python -m + codeanalyzer): in-process invocations — e.g. Typer's CliRunner in the + test suite, or a host app calling the callback — must never have their + own process exec'd out from under them.""" + if os.environ.get("PYTHONHASHSEED") is not None: + return + argv0 = os.path.basename(sys.argv[0]) if sys.argv else "" + is_cli = argv0 in ("canpy", "codeanalyzer") or sys.argv[0].endswith( + os.path.join("codeanalyzer", "__main__.py") + ) + if not is_cli: + return + env = dict(os.environ, PYTHONHASHSEED="0") + os.execvpe( + sys.executable, + [sys.executable, "-m", "codeanalyzer", *sys.argv[1:]], + env, + ) + from codeanalyzer.core import Codeanalyzer from codeanalyzer.utils import _set_log_level, logger from codeanalyzer.config import OutputFormat @@ -264,6 +295,10 @@ def main( ), ] = 50, ): + # Determinism: pin the interpreter hash seed before any analysis (no-op + # when PYTHONHASHSEED is already set; --version exits before this). + _pin_hash_seed() + # Flag validation (strict: unrecognized values error out, never fall back). selected_graphs = [g.strip() for g in graphs.split(",") if g.strip()] from codeanalyzer.dataflow.builder import VALID_GRAPHS diff --git a/codeanalyzer/core.py b/codeanalyzer/core.py index a8f03c3..a533d0f 100644 --- a/codeanalyzer/core.py +++ b/codeanalyzer/core.py @@ -37,6 +37,22 @@ from codeanalyzer.options import AnalysisOptions from codeanalyzer.provenance import analyzer_info, repository_info +def _ensure_ray() -> None: + """Initialize Ray with the driver's pinned hash seed in the workers. + + An implicit auto-init would not carry PYTHONHASHSEED into worker + interpreters, so PyCG shards (and Jedi inference) run there with random + set-iteration order and the emitted edges vary run to run (issue #99).""" + if not ray.is_initialized(): + ray.init( + runtime_env={ + "env_vars": { + "PYTHONHASHSEED": os.environ.get("PYTHONHASHSEED", "0") + } + }, + ) + + @ray.remote def _process_file_with_ray(py_file: Union[Path, str], project_dir: Union[Path, str], virtualenv: Union[Path, str, None]) -> Dict[str, PyModule]: """Processes files in the project directory using Ray for distributed processing. @@ -438,6 +454,11 @@ def analyze(self) -> Analysis: call_graph = merge_edges(call_graph, pycg_edges) call_graph = filter_external_edges(call_graph, symbol_table) + # Canonical edge order: backend iteration order (PyCG dicts, Counter + # insertion) is not a contract — sort so identical edge SETS always + # serialize identically (issue #99 determinism gate), and so the + # external-symbol homing below assigns ids in a stable order. + call_graph.sort(key=lambda e: (e.src, e.dst)) # Recreate pyapplication app = ( @@ -716,6 +737,7 @@ def _build_symbol_table(self, cached_symbol_table: Optional[Dict[str, PyModule]] # Process only new/changed files with Ray if files_to_process: + _ensure_ray() futures = [_process_file_with_ray.remote(py_file, self.project_dir, str(self.virtualenv) if self.virtualenv else None) for py_file in files_to_process] with ProgressBar(len(futures), "Building symbol table (parallel)") as progress: diff --git a/codeanalyzer/semantic_analysis/pycg/pycg_analysis.py b/codeanalyzer/semantic_analysis/pycg/pycg_analysis.py index 93b36e9..3a19915 100644 --- a/codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +++ b/codeanalyzer/semantic_analysis/pycg/pycg_analysis.py @@ -39,9 +39,12 @@ # re-enters PyCG's hook before its import graph is ready. Pre-importing # these modules at import time ensures they're already in sys.modules when # PyCG's hook is active, preventing the re-entrant ImportManagerError. +import fcntl +import hashlib import importlib.metadata # noqa: F401 import importlib.util # noqa: F401 import contextlib +import os import json # noqa: F401 import shutil import signal @@ -85,6 +88,15 @@ def _handler(signum: int, frame: object) -> None: from codeanalyzer.utils import ProgressBar, logger +def _shard_root_path(files: List[str], project_dir: Path) -> Path: + """Content-derived mini-project root for a shard: same project + same file + set → same path on every run (determinism, issue #99).""" + digest = hashlib.sha1( + "\0".join([str(project_dir), *sorted(files)]).encode("utf-8") + ).hexdigest()[:16] + return Path(tempfile.gettempdir()) / f"canpy_pycg_shard_{digest}" + + def _materialize_shard_root( files: List[str], project_dir: Path, @@ -103,10 +115,21 @@ def _materialize_shard_root( The caller owns the returned *root* and must ``shutil.rmtree`` it. """ - root = Path(tempfile.mkdtemp(prefix="canpy_pycg_shard_")) + # Deterministic root: PyCG's capped fixpoint (--pycg-max-iter) is + # order-sensitive, and its internal state keys on absolute module paths — + # a random mkdtemp suffix changes those strings every run and shifts the + # iteration frontier, making the emitted edge set vary run-to-run + # (issue #99). Deriving the directory name from the shard's content keeps + # the path (and thus the analysis input) identical across runs. Callers + # that may run concurrently on the same shard serialize on the sidecar + # lock (see _shard_symlink_root). + root = _shard_root_path(files, project_dir) + if root.exists(): + shutil.rmtree(root, ignore_errors=True) + root.mkdir(parents=True, exist_ok=True) entry_points: List[str] = [] linked_inits: Set[Path] = set() - for f in files: + for f in sorted(files): src = Path(f).resolve() try: rel = src.relative_to(project_dir) @@ -142,12 +165,27 @@ def _shard_symlink_root( """Context-manager wrapper around :func:`_materialize_shard_root`. Yields ``(root, entry_points)`` and removes the temp tree on exit. + + The root path is content-derived (determinism, issue #99), so two + concurrent analyses of the same shard — e.g. a test suite and a manual + run on one project — would collide on it (one rmtree's the tree the + other is mid-analysis on). An exclusive flock on a sidecar lockfile + serializes them; distinct projects/shards hash to distinct roots and + never contend. """ - root, entry_points = _materialize_shard_root(files, project_dir) + digest_root = _shard_root_path(files, project_dir) + lock_path = digest_root.with_name(digest_root.name + ".lock") + lock_fd = os.open(str(lock_path), os.O_CREAT | os.O_RDWR) try: - yield root, entry_points + fcntl.flock(lock_fd, fcntl.LOCK_EX) + root, entry_points = _materialize_shard_root(files, project_dir) + try: + yield root, entry_points + finally: + shutil.rmtree(root, ignore_errors=True) finally: - shutil.rmtree(root, ignore_errors=True) + fcntl.flock(lock_fd, fcntl.LOCK_UN) + os.close(lock_fd) def _pycg_shard_worker( @@ -469,7 +507,9 @@ def _collect_entry_points(self) -> List[str]: ): continue paths.append(str(p)) - return paths + # Sorted for run-to-run stability: rglob yields filesystem order, and + # PyCG's capped fixpoint is sensitive to entry-point order (issue #99). + return sorted(paths) # ------------------------------------------------------------------ # Package-root helpers for sharding @@ -712,11 +752,14 @@ def _run_fileset_shards_ray( """ import os import ray + from codeanalyzer.core import _ensure_ray + _ensure_ray() os.environ.setdefault("RAY_IGNORE_UNHANDLED_ERRORS", "1") remote_fn = ray.remote(_pycg_shard_worker) roots: List[Path] = [] + lock_fds: List[int] = [] futures: List[Any] = [] meta: Dict[Any, List[str]] = {} # ObjectRef -> shard file list edges_all: List[PyCallEdge] = [] @@ -724,6 +767,16 @@ def _run_fileset_shards_ray( try: with ProgressBar(len(shards), "Building call graph shards (parallel)", item_label="shards") as progress: for files in shards: + # Deterministic roots can collide across concurrent + # analyses of the same project — the driver holds each + # shard's sidecar lock for the whole Ray fan-out (released + # in the finally below with the root cleanup). + lock_fd = os.open( + str(_shard_root_path(files, self.project_dir).with_suffix(".lock")), + os.O_CREAT | os.O_RDWR, + ) + fcntl.flock(lock_fd, fcntl.LOCK_EX) + lock_fds.append(lock_fd) root, eps = _materialize_shard_root(files, self.project_dir) roots.append(root) fut = remote_fn.remote(eps, str(root), "", self.max_iter) @@ -765,6 +818,12 @@ def _run_fileset_shards_ray( finally: for root in roots: shutil.rmtree(root, ignore_errors=True) + for fd in lock_fds: + try: + fcntl.flock(fd, fcntl.LOCK_UN) + os.close(fd) + except OSError: + pass return edges_all, runaways def _build_sharded( @@ -870,6 +929,8 @@ def _build_sharded_ray(self, shards: Dict[Path, List[str]]) -> List[PyCallEdge]: """ import os import ray + from codeanalyzer.core import _ensure_ray + _ensure_ray() # force-cancel kills worker processes; suppress Ray's "worker died # unexpectedly" noise since the death is intentional here. diff --git a/codeanalyzer/syntactic_analysis/symbol_table_builder.py b/codeanalyzer/syntactic_analysis/symbol_table_builder.py index 5e33c8c..bc0ca9f 100644 --- a/codeanalyzer/syntactic_analysis/symbol_table_builder.py +++ b/codeanalyzer/syntactic_analysis/symbol_table_builder.py @@ -57,13 +57,26 @@ def _fallback_signature(self, script_path: Union[Path, str], name: str) -> str: relative = Path(script_path).relative_to(self.project_dir) return ".".join(relative.with_suffix("").parts) + f".{name}" + @staticmethod + def _first_definition(definitions): + """Deterministic pick from Jedi's inference candidates. + + On a union-typed receiver Jedi may return several candidates whose + ORDER varies run to run (issue #99 — e.g. ``o.seek`` on + ``_IOBase | BufferedRandom | TextIOWrapper``); taking whichever came + first made the emitted call graph nondeterministic. Sort on the + stable identity (full_name, then name) and take the smallest.""" + if not definitions: + return None + return min(definitions, key=lambda d: (d.full_name or "", d.name or "")) + @staticmethod def _infer_type(script: Script, line: int, column: int) -> str: """Tries to infer the type at a given position using Jedi.""" try: - inference = script.infer(line=line, column=column) - if inference: - return inference[0].name # or .full_name + d = SymbolTableBuilder._first_definition(script.infer(line=line, column=column)) + if d is not None: + return d.name # or .full_name except Exception: pass return None @@ -82,9 +95,9 @@ def _infer_qualified_name(script: Script, line: int, column: int) -> Optional[st Optional[str]: The fully qualified name if available, else None. """ try: - definitions = script.infer(line=line, column=column) - if definitions: - return definitions[0].full_name + d = SymbolTableBuilder._first_definition(script.infer(line=line, column=column)) + if d is not None: + return d.full_name except Exception: pass return None @@ -103,10 +116,9 @@ def _infer_callee( the call graph. """ try: - definitions = script.infer(line=line, column=column) - if not definitions: + d = SymbolTableBuilder._first_definition(script.infer(line=line, column=column)) + if d is None: return None, False - d = definitions[0] is_class = (d.type == "class") full = d.full_name if is_class and full: @@ -144,11 +156,17 @@ def _infer_call_return_type(script: Script, line: int, column: int) -> Optional[ as the callee's own name. """ try: - definitions = script.infer(line=line, column=column) - if definitions: - results = definitions[0].execute() - if results: - return results[0].name + d = SymbolTableBuilder._first_definition(script.infer(line=line, column=column)) + if d is not None: + # Drop NoneType results before picking: Jedi flaps between + # yielding {NoneType} and {} for the None arm of an Optional + # return (issue #99), and when a real type is also in the + # union it is the informative choice — a bare NoneType says + # nothing a missing return_type doesn't. + results = [r for r in d.execute() if r.name != "NoneType"] + r = SymbolTableBuilder._first_definition(results) + if r is not None: + return r.name except Exception: pass return None @@ -974,9 +992,10 @@ def _symbol_from_name_node( if script: try: - definitions = script.infer(line=lineno, column=col_offset) - if definitions: - d = definitions[0] + d = SymbolTableBuilder._first_definition( + script.infer(line=lineno, column=col_offset) + ) + if d is not None: inferred_type = d.name qname = d.full_name if d.type == "function": diff --git a/test/test_v2_l2.py b/test/test_v2_l2.py index 49c402c..cf7e206 100644 --- a/test/test_v2_l2.py +++ b/test/test_v2_l2.py @@ -41,3 +41,100 @@ def test_call_graph_external_target_unchanged(): reidentify_call_graph(app, {"m.f": "can://python/app/m.py/f()"}) assert app.call_graph[0].src == "can://python/app/m.py/f()" assert app.call_graph[0].dst == "requests.get" + + +def test_l2_audit_gate_callee_name_equality(tmp_path): + """Issue #87 acceptance (CLDK-001/002 v2 counterpart): for resolvable + non-constructor calls, >=95% of resolved callee bindings must agree with + the invoked attribute name, and no callsite may fall back to a class id + when that class declares the exact method (the receiver-anchor defect + bound `self.env['x'].search(...)` to the caller's own class).""" + import json + import shutil + import subprocess + import sys + from pathlib import Path + + fixture = Path(__file__).parent / "fixtures" / "single_functionalities" / "method_call_resolution" + proj = tmp_path / "proj" + shutil.copytree(fixture, proj) + out = subprocess.run( + [sys.executable, "-m", "codeanalyzer", "-i", str(proj), "-a", "2", "--no-venv"], + capture_output=True, text=True, check=True, + ).stdout + app = json.loads(out)["application"] + + # collect declared types (name -> their method names) for fallback detection + class_methods = {} + def walk_type(t): + class_methods[t["id"]] = set() + for mname, m in (t.get("callables") or {}).items(): + class_methods[t["id"]].add(m["name"]) + for it in (t.get("types") or {}).values(): + walk_type(it) + sites = [] + for mod in app["symbol_table"].values(): + for t in (mod.get("types") or {}).values(): + walk_type(t) + def walk_callable(c): + for cs in c.get("call_sites") or []: + sites.append(cs) + for ic in (c.get("callables") or {}).values(): + walk_callable(ic) + for fn in (mod.get("functions") or {}).values(): + walk_callable(fn) + for t in (mod.get("types") or {}).values(): + for m in (t.get("callables") or {}).values(): + walk_callable(m) + + resolved = [ + cs for cs in sites + if cs.get("callee_signature") and not cs.get("is_constructor_call") + ] + assert resolved, "fixture must produce resolved non-constructor callsites" + agree = sum( + 1 for cs in resolved + if cs["callee_signature"].rsplit(".", 1)[-1] == cs["method_name"] + ) + ratio = agree / len(resolved) + assert ratio >= 0.95, ( + f"only {agree}/{len(resolved)} resolved callsites bind to the invoked " + f"name: {[(cs['method_name'], cs['callee_signature']) for cs in resolved if cs['callee_signature'].rsplit('.', 1)[-1] != cs['method_name']]}" + ) + # no class fallback when the exact method exists: a callee binding that + # names a class which declares the invoked method is the defect's signature + for cs in resolved: + for cls_id, methods in class_methods.items(): + cls_sig_name = cls_id.rsplit("/", 1)[-1] + if cs["callee_signature"].rsplit(".", 1)[-1] == cls_sig_name and cs["method_name"] in methods: + raise AssertionError( + f"callsite {cs['method_name']!r} fell back to class {cls_sig_name!r} " + f"which declares the exact method" + ) + + +def test_l2_runs_are_byte_identical(tmp_path): + """Issue #99: same flags, same input → byte-identical analysis.json. + Exercises the deterministic PyCG shard root, sorted entry points, the + canonical call_graph sort, the Jedi union tie-break, and the CLI's + self-pinned PYTHONHASHSEED (each subprocess re-execs with seed 0).""" + import filecmp + import shutil + import subprocess + import sys + from pathlib import Path + + fixture = Path(__file__).parent / "fixtures" / "single_functionalities" / "decorators_and_hof" + proj = tmp_path / "proj" + shutil.copytree(fixture, proj) + outs = [] + for i in (1, 2): + out = tmp_path / f"out{i}" + subprocess.run( + [sys.executable, "-m", "codeanalyzer", "-i", str(proj), "-a", "2", + "--no-venv", "-o", str(out), "-c", str(tmp_path / f"cache{i}")], + capture_output=True, text=True, check=True, + ) + outs.append(out / "analysis.json") + assert filecmp.cmp(outs[0], outs[1], shallow=False), \ + "two identical -a 2 runs must emit byte-identical analysis.json" From 16e765aa86fc1e266757675c50f852220278ab13 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 15 Jul 2026 17:30:04 -0400 Subject: [PATCH 03/22] chore(release): 1.0.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deterministic-output batch (#101) released as a patch. Handoff goldens regenerated against the shipping emitter — the 1.0.0 bundle predated the v0.3.1 reconciliation merge, so it lacked the envelope analyzer.config, structured call-site arguments, and the callee-anchoring improvements the released binary actually emits; the bundle now round-trips 1.0.1 exactly (and reproducibly, courtesy of #101 itself). --- CHANGELOG.md | 2 + docs/handoff/README.md | 4 +- docs/handoff/analysis.l1.json | 31 ++++++--- docs/handoff/analysis.l2.json | 46 ++++++++------ docs/handoff/analysis.l3.json | 46 ++++++++------ docs/handoff/analysis.l4.json | 114 +++++++++++++++++++++++++++------- docs/handoff/graph.cypher | 10 +-- pyproject.toml | 2 +- 8 files changed, 177 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1905b5..1c2ab57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.0.1] - 2026-07-15 + ### Fixed - **Deterministic output** (#99): the same flags on the same input now emit byte-identical `analysis.json`. Four independent nondeterminism sources were closed: diff --git a/docs/handoff/README.md b/docs/handoff/README.md index 945ac8b..9397ba9 100644 --- a/docs/handoff/README.md +++ b/docs/handoff/README.md @@ -13,14 +13,14 @@ the contract lives in the spec, the decision log, and `schema.neo4j.json`. Pin the analyzer that produced these samples: ``` -pip install "codeanalyzer-python==1.0.0" +pip install "codeanalyzer-python==1.0.1" ``` For L4 alias analysis (the Scalpel points-to oracle), install the optional extra — without it, L4 degrades to the type-based fallback oracle (still sound, coarser): ``` -pip install "codeanalyzer-python[scalpel]==1.0.0" +pip install "codeanalyzer-python[scalpel]==1.0.1" ``` ## Schema contract diff --git a/docs/handoff/analysis.l1.json b/docs/handoff/analysis.l1.json index 4945170..421d462 100644 --- a/docs/handoff/analysis.l1.json +++ b/docs/handoff/analysis.l1.json @@ -4,7 +4,10 @@ "max_level": 1, "analyzer": { "name": "codeanalyzer-python", - "version": "1.0.0" + "version": "1.0.1", + "config": { + "analysis_level": 1 + } }, "application": { "symbol_table": { @@ -162,7 +165,11 @@ "argument_types": [ "Name" ], - "return_type": "build", + "arguments": [ + { + "ast_kind": "Name" + } + ], "callee_signature": "service.build", "is_constructor_call": false, "start_line": 6, @@ -368,6 +375,7 @@ { "method_name": "Service", "argument_types": [], + "arguments": [], "return_type": "Service", "callee_signature": "service.Service.__init__", "is_constructor_call": true, @@ -383,8 +391,13 @@ "argument_types": [ "Name" ], - "return_type": "Service", - "callee_signature": "service.Service", + "arguments": [ + { + "ast_kind": "Name" + } + ], + "return_type": "str", + "callee_signature": "service.Service.announce", "is_constructor_call": false, "start_line": 17, "start_column": 11, @@ -459,8 +472,8 @@ "kind": "application", "call_graph": [ { - "src": "can://python/sample_proj/service.py/run(flag)", - "dst": "can://python/sample_proj/@external/service.Service/__init__", + "src": "can://python/sample_proj/service.py/Service/announce(self,flag)", + "dst": "can://python/sample_proj/service.py/build(x)", "weight": 1, "prov": [ "jedi" @@ -468,15 +481,15 @@ }, { "src": "can://python/sample_proj/service.py/run(flag)", - "dst": "can://python/sample_proj/service.py/Service", + "dst": "can://python/sample_proj/@external/service.Service/__init__", "weight": 1, "prov": [ "jedi" ] }, { - "src": "can://python/sample_proj/service.py/Service/announce(self,flag)", - "dst": "can://python/sample_proj/service.py/build(x)", + "src": "can://python/sample_proj/service.py/run(flag)", + "dst": "can://python/sample_proj/service.py/Service/announce(self,flag)", "weight": 1, "prov": [ "jedi" diff --git a/docs/handoff/analysis.l2.json b/docs/handoff/analysis.l2.json index 714aa89..765a400 100644 --- a/docs/handoff/analysis.l2.json +++ b/docs/handoff/analysis.l2.json @@ -4,7 +4,10 @@ "max_level": 2, "analyzer": { "name": "codeanalyzer-python", - "version": "1.0.0" + "version": "1.0.1", + "config": { + "analysis_level": 2 + } }, "application": { "symbol_table": { @@ -162,7 +165,11 @@ "argument_types": [ "Name" ], - "return_type": "build", + "arguments": [ + { + "ast_kind": "Name" + } + ], "callee_signature": "service.build", "is_constructor_call": false, "start_line": 6, @@ -369,6 +376,7 @@ { "method_name": "Service", "argument_types": [], + "arguments": [], "return_type": "Service", "callee_signature": "service.Service.__init__", "is_constructor_call": true, @@ -384,8 +392,13 @@ "argument_types": [ "Name" ], - "return_type": "Service", - "callee_signature": "service.Service", + "arguments": [ + { + "ast_kind": "Name" + } + ], + "return_type": "str", + "callee_signature": "service.Service.announce", "is_constructor_call": false, "start_line": 17, "start_column": 11, @@ -443,7 +456,7 @@ 296 ] }, - "callee": "can://python/sample_proj/service.py/Service" + "callee": "can://python/sample_proj/service.py/Service/announce(self,flag)" } }, "cfg": [], @@ -462,35 +475,28 @@ "kind": "application", "call_graph": [ { - "src": "can://python/sample_proj/service.py/run(flag)", - "dst": "can://python/sample_proj/@external/service.Service/__init__", - "weight": 1, + "src": "can://python/sample_proj/service.py/Service/announce(self,flag)", + "dst": "can://python/sample_proj/service.py/build(x)", + "weight": 2, "prov": [ - "jedi" + "jedi", + "pycg" ] }, { "src": "can://python/sample_proj/service.py/run(flag)", - "dst": "can://python/sample_proj/service.py/Service", + "dst": "can://python/sample_proj/@external/service.Service/__init__", "weight": 1, "prov": [ "jedi" ] }, - { - "src": "can://python/sample_proj/service.py/Service/announce(self,flag)", - "dst": "can://python/sample_proj/service.py/build(x)", - "weight": 2, - "prov": [ - "jedi", - "pycg" - ] - }, { "src": "can://python/sample_proj/service.py/run(flag)", "dst": "can://python/sample_proj/service.py/Service/announce(self,flag)", - "weight": 1, + "weight": 2, "prov": [ + "jedi", "pycg" ] } diff --git a/docs/handoff/analysis.l3.json b/docs/handoff/analysis.l3.json index 920d4cb..de29010 100644 --- a/docs/handoff/analysis.l3.json +++ b/docs/handoff/analysis.l3.json @@ -5,7 +5,10 @@ "k_limit": 3, "analyzer": { "name": "codeanalyzer-python", - "version": "1.0.0" + "version": "1.0.1", + "config": { + "analysis_level": 3 + } }, "application": { "symbol_table": { @@ -163,7 +166,11 @@ "argument_types": [ "Name" ], - "return_type": "build", + "arguments": [ + { + "ast_kind": "Name" + } + ], "callee_signature": "service.build", "is_constructor_call": false, "start_line": 6, @@ -637,6 +644,7 @@ { "method_name": "Service", "argument_types": [], + "arguments": [], "return_type": "Service", "callee_signature": "service.Service.__init__", "is_constructor_call": true, @@ -652,8 +660,13 @@ "argument_types": [ "Name" ], - "return_type": "Service", - "callee_signature": "service.Service", + "arguments": [ + { + "ast_kind": "Name" + } + ], + "return_type": "str", + "callee_signature": "service.Service.announce", "is_constructor_call": false, "start_line": 17, "start_column": 11, @@ -711,7 +724,7 @@ 296 ] }, - "callee": "can://python/sample_proj/service.py/Service" + "callee": "can://python/sample_proj/service.py/Service/announce(self,flag)" }, "@entry": { "kind": "entry" @@ -838,35 +851,28 @@ "kind": "application", "call_graph": [ { - "src": "can://python/sample_proj/service.py/run(flag)", - "dst": "can://python/sample_proj/@external/service.Service/__init__", - "weight": 1, + "src": "can://python/sample_proj/service.py/Service/announce(self,flag)", + "dst": "can://python/sample_proj/service.py/build(x)", + "weight": 2, "prov": [ - "jedi" + "jedi", + "pycg" ] }, { "src": "can://python/sample_proj/service.py/run(flag)", - "dst": "can://python/sample_proj/service.py/Service", + "dst": "can://python/sample_proj/@external/service.Service/__init__", "weight": 1, "prov": [ "jedi" ] }, - { - "src": "can://python/sample_proj/service.py/Service/announce(self,flag)", - "dst": "can://python/sample_proj/service.py/build(x)", - "weight": 2, - "prov": [ - "jedi", - "pycg" - ] - }, { "src": "can://python/sample_proj/service.py/run(flag)", "dst": "can://python/sample_proj/service.py/Service/announce(self,flag)", - "weight": 1, + "weight": 2, "prov": [ + "jedi", "pycg" ] } diff --git a/docs/handoff/analysis.l4.json b/docs/handoff/analysis.l4.json index ccd8f7b..3bcdd19 100644 --- a/docs/handoff/analysis.l4.json +++ b/docs/handoff/analysis.l4.json @@ -5,7 +5,10 @@ "k_limit": 3, "analyzer": { "name": "codeanalyzer-python", - "version": "1.0.0" + "version": "1.0.1", + "config": { + "analysis_level": 4 + } }, "application": { "symbol_table": { @@ -163,7 +166,11 @@ "argument_types": [ "Name" ], - "return_type": "build", + "arguments": [ + { + "ast_kind": "Name" + } + ], "callee_signature": "service.build", "is_constructor_call": false, "start_line": 6, @@ -680,6 +687,7 @@ { "method_name": "Service", "argument_types": [], + "arguments": [], "return_type": "Service", "callee_signature": "service.Service.__init__", "is_constructor_call": true, @@ -695,8 +703,13 @@ "argument_types": [ "Name" ], - "return_type": "Service", - "callee_signature": "service.Service", + "arguments": [ + { + "ast_kind": "Name" + } + ], + "return_type": "str", + "callee_signature": "service.Service.announce", "is_constructor_call": false, "start_line": 17, "start_column": 11, @@ -754,7 +767,7 @@ 296 ] }, - "callee": "can://python/sample_proj/service.py/Service" + "callee": "can://python/sample_proj/service.py/Service/announce(self,flag)" }, "@entry": { "kind": "entry" @@ -804,6 +817,10 @@ "kind": "formal_in", "of": ":service::Service" }, + "@formal_in:2": { + "kind": "formal_in", + "of": ":service::build" + }, "@formal_out:0": { "kind": "formal_out", "of": "" @@ -811,6 +828,31 @@ "@formal_out:1": { "kind": "formal_out", "of": "flag" + }, + "17:4/actual_in:0": { + "kind": "actual_in", + "of": "self", + "parent": "17:4" + }, + "17:4/actual_in:1": { + "kind": "actual_in", + "of": "flag", + "parent": "17:4" + }, + "17:4/actual_in:2": { + "kind": "actual_in", + "of": ":service::build", + "parent": "17:4" + }, + "17:4/actual_out:0": { + "kind": "actual_out", + "of": "", + "parent": "17:4" + }, + "17:4/actual_out:1": { + "kind": "actual_out", + "of": "flag", + "parent": "17:4" } }, "cfg": [ @@ -892,7 +934,24 @@ ] } ], - "summary": [] + "summary": [ + { + "src": "17:4/actual_in:1", + "dst": "17:4/actual_out:0" + }, + { + "src": "17:4/actual_in:1", + "dst": "17:4/actual_out:1" + }, + { + "src": "17:4/actual_in:2", + "dst": "17:4/actual_out:0" + }, + { + "src": "17:4/actual_in:2", + "dst": "17:4/actual_out:1" + } + ] } }, "variables": [], @@ -905,35 +964,28 @@ "kind": "application", "call_graph": [ { - "src": "can://python/sample_proj/service.py/run(flag)", - "dst": "can://python/sample_proj/@external/service.Service/__init__", - "weight": 1, + "src": "can://python/sample_proj/service.py/Service/announce(self,flag)", + "dst": "can://python/sample_proj/service.py/build(x)", + "weight": 2, "prov": [ - "jedi" + "jedi", + "pycg" ] }, { "src": "can://python/sample_proj/service.py/run(flag)", - "dst": "can://python/sample_proj/service.py/Service", + "dst": "can://python/sample_proj/@external/service.Service/__init__", "weight": 1, "prov": [ "jedi" ] }, - { - "src": "can://python/sample_proj/service.py/Service/announce(self,flag)", - "dst": "can://python/sample_proj/service.py/build(x)", - "weight": 2, - "prov": [ - "jedi", - "pycg" - ] - }, { "src": "can://python/sample_proj/service.py/run(flag)", "dst": "can://python/sample_proj/service.py/Service/announce(self,flag)", - "weight": 1, + "weight": 2, "prov": [ + "jedi", "pycg" ] } @@ -950,9 +1002,29 @@ { "src": "can://python/sample_proj/service.py/Service/announce(self,flag)@6:8/actual_in:0", "dst": "can://python/sample_proj/service.py/build(x)@formal_in:0" + }, + { + "src": "can://python/sample_proj/service.py/run(flag)@17:4/actual_in:0", + "dst": "can://python/sample_proj/service.py/Service/announce(self,flag)@formal_in:0" + }, + { + "src": "can://python/sample_proj/service.py/run(flag)@17:4/actual_in:1", + "dst": "can://python/sample_proj/service.py/Service/announce(self,flag)@formal_in:1" + }, + { + "src": "can://python/sample_proj/service.py/run(flag)@17:4/actual_in:2", + "dst": "can://python/sample_proj/service.py/Service/announce(self,flag)@formal_in:2" } ], "param_out": [ + { + "src": "can://python/sample_proj/service.py/Service/announce(self,flag)@formal_out:0", + "dst": "can://python/sample_proj/service.py/run(flag)@17:4/actual_out:0" + }, + { + "src": "can://python/sample_proj/service.py/Service/announce(self,flag)@formal_out:1", + "dst": "can://python/sample_proj/service.py/run(flag)@17:4/actual_out:1" + }, { "src": "can://python/sample_proj/service.py/build(x)@formal_out", "dst": "can://python/sample_proj/service.py/Service/announce(self,flag)@6:8/actual_out" diff --git a/docs/handoff/graph.cypher b/docs/handoff/graph.cypher index 05cd4be..b8757f4 100644 --- a/docs/handoff/graph.cypher +++ b/docs/handoff/graph.cypher @@ -20,7 +20,7 @@ DETACH DELETE x, m, a; // ── nodes ── UNWIND [ - {k: 'sample_proj', p: {schema_version: '2.0.0'}} + {k: 'sample_proj', p: {schema_version: '2.0.0', analyzer_name: 'codeanalyzer-python', analyzer_version: '1.0.1'}} ] AS row MERGE (n:PyApplication {name: row.k}) SET n += row.p; @@ -33,8 +33,8 @@ MERGE (n:PyCFGNode {id: row.k}) SET n += row.p; UNWIND [ {k: 'service.py#16:10-16:19', p: {id: 'service.py#16:10-16:19', method_name: 'Service', argument_types: [], return_type: 'Service', callee_signature: 'service.Service.__init__', is_constructor_call: true, start_line: 16, start_column: 10, end_line: 16, end_column: 19, _module: 'service.py'}}, - {k: 'service.py#17:11-17:29', p: {id: 'service.py#17:11-17:29', method_name: 'announce', receiver_expr: 'svc', receiver_type: 'Service', argument_types: ['Name'], return_type: 'Service', callee_signature: 'service.Service', is_constructor_call: false, start_line: 17, start_column: 11, end_line: 17, end_column: 29, _module: 'service.py'}}, - {k: 'service.py#6:18-6:29', p: {id: 'service.py#6:18-6:29', method_name: 'build', argument_types: ['Name'], return_type: 'build', callee_signature: 'service.build', is_constructor_call: false, start_line: 6, start_column: 18, end_line: 6, end_column: 29, _module: 'service.py'}} + {k: 'service.py#17:11-17:29', p: {id: 'service.py#17:11-17:29', method_name: 'announce', receiver_expr: 'svc', receiver_type: 'Service', argument_types: ['Name'], arguments_json: '[{"ast_kind": "Name", "inferred_type": null}]', return_type: 'str', callee_signature: 'service.Service.announce', is_constructor_call: false, start_line: 17, start_column: 11, end_line: 17, end_column: 29, _module: 'service.py'}}, + {k: 'service.py#6:18-6:29', p: {id: 'service.py#6:18-6:29', method_name: 'build', argument_types: ['Name'], arguments_json: '[{"ast_kind": "Name", "inferred_type": null}]', callee_signature: 'service.build', is_constructor_call: false, start_line: 6, start_column: 18, end_line: 6, end_column: 29, _module: 'service.py'}} ] AS row MERGE (n:PyCallSite {id: row.k}) SET n += row.p; @@ -74,7 +74,7 @@ SET n += row.p; UNWIND [ {f: 'can://python/sample_proj/service.py/Service/announce(self,flag)', t: 'can://python/sample_proj/service.py/build(x)', p: {weight: 1, prov: ['jedi']}}, {f: 'can://python/sample_proj/service.py/run(flag)', t: 'can://python/sample_proj/@external/service.Service/__init__', p: {weight: 1, prov: ['jedi']}}, - {f: 'can://python/sample_proj/service.py/run(flag)', t: 'can://python/sample_proj/service.py/Service', p: {weight: 1, prov: ['jedi']}} + {f: 'can://python/sample_proj/service.py/run(flag)', t: 'can://python/sample_proj/service.py/Service/announce(self,flag)', p: {weight: 1, prov: ['jedi']}} ] AS row MATCH (a:PySymbol {id: row.f}) MATCH (b:PySymbol {id: row.t}) @@ -133,7 +133,7 @@ MATCH (b:PyModule {id: row.t}) MERGE (a)-[r:PY_HAS_MODULE]->(b) SET r += row.p; UNWIND [ - {f: 'service.py#17:11-17:29', t: 'can://python/sample_proj/service.py/Service', p: {}}, + {f: 'service.py#17:11-17:29', t: 'can://python/sample_proj/service.py/Service/announce(self,flag)', p: {}}, {f: 'service.py#6:18-6:29', t: 'can://python/sample_proj/service.py/build(x)', p: {}} ] AS row MATCH (a:PyCallSite {id: row.f}) diff --git a/pyproject.toml b/pyproject.toml index 9a3cee4..5965942 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codeanalyzer-python" -version = "1.0.0" +version = "1.0.1" description = "Static analysis for Python — canonical schema v2 (symbol table, call graph, and native CFG/PDG/SDG dataflow) as analysis.json or a Neo4j property graph." readme = "README.md" authors = [ From c58be9ba35a43547b362c573a3d35a3764e3d4b5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 15 Jul 2026 21:30:24 +0000 Subject: [PATCH 04/22] docs: sync README --help and Neo4j schema for v1.0.1 --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 8c8acf9..0a4e8e1 100644 --- a/README.md +++ b/README.md @@ -164,19 +164,19 @@ $ canpy --help │ --version Show the canpy │ │ version and │ │ exit. │ -│ --input -i PATH Path to the │ +│ --input -i Path to the │ │ project root │ │ directory (not │ │ required for │ │ --emit schema). │ -│ --output -o PATH Output directory │ +│ --output -o Output directory │ │ for artifacts. │ -│ --format -f [json|msgpack] Output format │ +│ --format -f Output format │ │ for --emit json: │ │ json or msgpack. │ │ [default: json] │ -│ --emit [json|neo4j|sche Output target: │ -│ ma] json │ +│ --emit json │ │ (analysis.json, │ │ default) | neo4j │ │ (graph.cypher or │ @@ -186,13 +186,13 @@ $ canpy --help │ schema.json │ │ contract). │ │ [default: json] │ -│ --app-name TEXT Logical │ +│ --app-name Logical │ │ application name │ │ for the graph │ │ :PyApplication │ │ anchor (default: │ │ input dir name). │ -│ --neo4j-uri TEXT Push the graph │ +│ --neo4j-uri Push the graph │ │ to a live Neo4j │ │ over Bolt │ │ (incremental); │ @@ -200,11 +200,11 @@ $ canpy --help │ graph.cypher. │ │ [env var: │ │ NEO4J_URI] │ -│ --neo4j-user TEXT Neo4j username. │ +│ --neo4j-user Neo4j username. │ │ [env var: │ │ NEO4J_USERNAME] │ │ [default: neo4j] │ -│ --neo4j-password TEXT Neo4j password. │ +│ --neo4j-password Neo4j password. │ │ Prefer the env │ │ var over the │ │ flag (the flag │ @@ -214,12 +214,12 @@ $ canpy --help │ [env var: │ │ NEO4J_PASSWORD] │ │ [default: neo4j] │ -│ --neo4j-database TEXT Neo4j database │ +│ --neo4j-database Neo4j database │ │ name (default: │ │ server default). │ │ [env var: │ │ NEO4J_DATABASE] │ -│ --analysis-level -a INTEGER RANGE Analysis depth: │ +│ --analysis-level -a Analysis depth: │ │ [1<=x<=4] 1=symbol │ │ table+Jedi call │ │ graph, 2=+PyCG │ @@ -235,7 +235,7 @@ $ canpy --help │ alias-aware │ │ DDG). │ │ [default: 1] │ -│ --graphs TEXT Level 3+ only: │ +│ --graphs Level 3+ only: │ │ comma-separated │ │ program-graph │ │ sections to emit │ @@ -248,7 +248,7 @@ $ canpy --help │ requires -a 4. │ │ [default: │ │ cfg,dfg,pdg] │ -│ --graph-field-de… INTEGER RANGE Level 3 only: │ +│ --graph-field-de… Level 3 only: │ │ [x>=1] k-limit on │ │ access-path │ │ depth (x.f.g.h │ @@ -285,12 +285,12 @@ $ canpy --help │ environment │ │ instead. │ │ [default: venv] │ -│ --file-name PATH Analyze only the │ +│ --file-name Analyze only the │ │ specified file │ │ (relative to │ │ input │ │ directory). │ -│ --cache-dir -c PATH Directory to │ +│ --cache-dir -c Directory to │ │ store analysis │ │ cache. Defaults │ │ to │ @@ -304,7 +304,7 @@ $ canpy --help │ retained. │ │ [default: │ │ keep-cache] │ -│ -v INTEGER Increase │ +│ -v Increase │ │ verbosity: -v, │ │ -vv, -vvv │ │ [default: 0] │ @@ -331,7 +331,7 @@ $ canpy --help │ Jedi-only edges. │ │ [default: │ │ no-pycg-shard] │ -│ --pycg-shard-cei… INTEGER RANGE Maximum files │ +│ --pycg-shard-cei… Maximum files │ │ [x>=1] per shard when │ │ --pycg-shard is │ │ active (default │ @@ -353,7 +353,7 @@ $ canpy --help │ heavy import │ │ graphs. │ │ [default: 100] │ -│ --pycg-shard-tim… INTEGER RANGE Per-shard │ +│ --pycg-shard-tim… Per-shard │ │ [x>=0] wall-clock │ │ timeout in │ │ seconds when │ @@ -381,7 +381,7 @@ $ canpy --help │ ignored on │ │ Windows. │ │ [default: 120] │ -│ --pycg-shard-str… [jedi|package] How --pycg-shard │ +│ --pycg-shard-str… How --pycg-shard │ │ groups files │ │ (level 2 only). │ │ 'jedi' (default) │ @@ -403,7 +403,7 @@ $ canpy --help │ one-shard-per-p… │ │ grouping. │ │ [default: jedi] │ -│ --pycg-max-iter INTEGER RANGE Cap on PyCG's │ +│ --pycg-max-iter Cap on PyCG's │ │ [x>=-1] fixpoint passes │ │ per │ │ shard/project │ From 03edbd8f17690d02d2977b38228a99e86e90e082 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Thu, 16 Jul 2026 10:43:39 -0400 Subject: [PATCH 05/22] chore: symlink GEMINI.md -> CLAUDE.md (#65) AGENTS.md already points at CLAUDE.md as the single source of agent guidance; add the matching GEMINI.md symlink issue #65 asked for so Gemini reads the same file. --- GEMINI.md | 1 + 1 file changed, 1 insertion(+) create mode 120000 GEMINI.md diff --git a/GEMINI.md b/GEMINI.md new file mode 120000 index 0000000..681311e --- /dev/null +++ b/GEMINI.md @@ -0,0 +1 @@ +CLAUDE.md \ No newline at end of file From 41cb70a795ad81f784902013f24f76a319541523 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Thu, 16 Jul 2026 22:23:23 -0400 Subject: [PATCH 06/22] fix(neo4j): derive the code property from module source spans (#104) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Schema v2 removed the per-node code field (module source is stored once, sliced by spans), but the Neo4j projection still read it via getattr(..., 'code', None) — so every :PyClass and :PyCallable node was written without code, deadening the py_code_fts fulltext index and the python-sdk's RETURN c.code queries. Thread the owning module's source through the declaration walk and slice it by each node's utf-8 byte span at projection time, restoring the declared graph contract without touching analysis.json. --- CHANGELOG.md | 9 +++++ codeanalyzer/neo4j/project.py | 43 ++++++++++++++++-------- test/test_v2_two_projection_agreement.py | 38 +++++++++++++++++++++ 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c2ab57..a80e3f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- **Neo4j `code` property was silently null** (#104): schema v2 removed the per-node `code` + field (source lives once on `PyModule.source`, sliced by spans), but the Neo4j projection + still read the old field — every `:PyClass`/`:PyCallable` node was written without `code`, + deadening the `py_code_fts` fulltext index and the SDK's `RETURN c.code` queries. The + projection now derives `code` at projection time by slicing the owning module's source + with the node's utf-8 byte span. Regression gate: + `test_projected_code_property_is_the_module_source_span_slice`. + ## [1.0.1] - 2026-07-15 ### Fixed diff --git a/codeanalyzer/neo4j/project.py b/codeanalyzer/neo4j/project.py index e44f948..672ee80 100644 --- a/codeanalyzer/neo4j/project.py +++ b/codeanalyzer/neo4j/project.py @@ -290,9 +290,11 @@ def _project_module_body( externals: dict, sig_to_id: dict, module_id_by_key: dict, ) -> None: for fn in (mod.functions or {}).values(): - _project_callable(b, file_key, mod_ref, "PY_DECLARES", fn, externals, sig_to_id) + _project_callable(b, file_key, mod_ref, "PY_DECLARES", fn, externals, sig_to_id, + mod.source) for cl in (mod.types or {}).values(): - _project_class(b, file_key, mod_ref, "PY_DECLARES", cl, externals, sig_to_id) + _project_class(b, file_key, mod_ref, "PY_DECLARES", cl, externals, sig_to_id, + mod.source) for v in mod.variables or []: _project_variable(b, file_key, mod_ref, file_key, v) _project_imports(b, mod_ref, mod, module_id_by_key) @@ -360,10 +362,10 @@ def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule, def _project_class( b: RowBuilder, file_key: str, parent: NodeRef, parent_rel: str, cl: PyClass, - externals: dict, sig_to_id: dict, + externals: dict, sig_to_id: dict, source: str, ) -> None: ref = b.node( - ["PySymbol", "PyClass"], "id", cl.id, _class_props(cl, file_key) + ["PySymbol", "PyClass"], "id", cl.id, _class_props(cl, file_key, source) ) b.edge(parent_rel, parent, ref) @@ -372,22 +374,23 @@ def _project_class( b.edge_to_symbol("PY_EXTENDS", ref, _symbol_ref(base, externals, sig_to_id)) for m in (cl.callables or {}).values(): - _project_callable(b, file_key, ref, "PY_HAS_METHOD", m, externals, sig_to_id) + _project_callable(b, file_key, ref, "PY_HAS_METHOD", m, externals, sig_to_id, + source) for a in (cl.attributes or {}).values(): _project_attribute(b, file_key, ref, cl.signature, a) for ic in (cl.types or {}).values(): - _project_class(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id) + _project_class(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id, source) def _project_callable( b: RowBuilder, file_key: str, owner: NodeRef, owner_rel: str, c: PyCallable, - externals: dict, sig_to_id: dict, + externals: dict, sig_to_id: dict, source: str, ) -> None: ref = b.node( ["PySymbol", "PyCallable"], "id", c.id, - _callable_props(c, file_key), + _callable_props(c, file_key, source), ) b.edge(owner_rel, owner, ref) @@ -410,9 +413,10 @@ def _project_callable( for v in c.local_variables or []: _project_variable(b, file_key, ref, c.signature, v) for ic in (c.callables or {}).values(): - _project_callable(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id) + _project_callable(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id, + source) for cl in (c.types or {}).values(): - _project_class(b, file_key, ref, "PY_DECLARES", cl, externals, sig_to_id) + _project_class(b, file_key, ref, "PY_DECLARES", cl, externals, sig_to_id, source) def _project_attribute( @@ -459,13 +463,24 @@ def _module_props(mod: PyModule, file_key: str) -> Props: ) -def _class_props(cl: PyClass, file_key: str) -> Props: +def _span_code(source: str, span) -> str | None: + """A declaration's text: the owning module's ``source`` sliced by the node's + utf-8 byte span. Schema v2 stores source once per module, so the graph's + ``code`` property (declared on :PyClass/:PyCallable and indexed by + ``py_code_fts``) is derived here at projection time (#104).""" + if span is None or not source: + return None + lo, hi = span.bytes + return source.encode("utf-8")[lo:hi].decode("utf-8") + + +def _class_props(cl: PyClass, file_key: str, source: str) -> Props: return prune( { "id": cl.id, "signature": cl.signature, "name": cl.name, - "code": getattr(cl, "code", None), + "code": _span_code(source, cl.span), "base_classes": list(cl.base_classes or []), "docstring": _docstring_of(cl.comments), "start_line": cl.start_line, @@ -475,7 +490,7 @@ def _class_props(cl: PyClass, file_key: str) -> Props: ) -def _callable_props(c: PyCallable, file_key: str) -> Props: +def _callable_props(c: PyCallable, file_key: str, source: str) -> Props: return prune( { "id": c.id, @@ -484,7 +499,7 @@ def _callable_props(c: PyCallable, file_key: str) -> Props: "path": c.path, "return_type": c.return_type, "cyclomatic_complexity": c.cyclomatic_complexity, - "code": getattr(c, "code", None), + "code": _span_code(source, c.span), "code_start_line": c.code_start_line, "start_line": c.start_line, "end_line": c.end_line, diff --git a/test/test_v2_two_projection_agreement.py b/test/test_v2_two_projection_agreement.py index 7d8ffd5..332e35c 100644 --- a/test/test_v2_two_projection_agreement.py +++ b/test/test_v2_two_projection_agreement.py @@ -207,3 +207,41 @@ def test_l4_param_summary_overlay_projects_onto_pycfgnode(tmp_path): assert any( e.type == "PY_DDG" and "prov" in e.props for e in rows.edges ), "expected a PY_DDG edge carrying a prov prop at -a 4" + + +# ---------------------------------------------------------------------------------------------- +# Regression for #104: schema v2 dropped the per-node `code` field (source lives once on +# PyModule.source, sliced by spans), but the graph schema still declares `code` on +# :PyClass / :PyCallable and indexes it (py_code_fts). The projection must therefore +# derive `code` at projection time — module source sliced by the node's byte span — +# or code search in the graph and the SDK's `RETURN c.code` go silently null. +# ---------------------------------------------------------------------------------------------- + + +def test_projected_code_property_is_the_module_source_span_slice(): + from sample_graph_app import make_sample_app + + app, sig_to_id = make_sample_app() + rows = project(app, "sample-app", sig_to_id) + by_id = {n.value: n for n in rows.nodes} + + checked = 0 + for mod in app.symbol_table.values(): + src = mod.source.encode("utf-8") + stack = list((mod.functions or {}).values()) + list((mod.types or {}).values()) + while stack: + decl = stack.pop() + stack += list((decl.callables or {}).values()) + stack += list((decl.types or {}).values()) + assert decl.span is not None, f"{decl.signature}: span missing at L1+" + lo, hi = decl.span.bytes + expected = src[lo:hi].decode("utf-8") + got = by_id[decl.id].props.get("code") + assert got == expected, ( + f"{decl.signature}: projected code must be the span slice of " + f"module.source, got {got!r}" + ) + checked += 1 + # the sample app has functions, methods, an inner class and a subclass — + # if we checked fewer than that, the walk itself is broken. + assert checked >= 6 From 9716632cbfbed0d300b16825db54e1594fa11f36 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Thu, 16 Jul 2026 22:42:30 -0400 Subject: [PATCH 07/22] chore(release): 1.0.2 --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a80e3f4..0f31363 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.0.2] - 2026-07-16 + ### Fixed - **Neo4j `code` property was silently null** (#104): schema v2 removed the per-node `code` field (source lives once on `PyModule.source`, sliced by spans), but the Neo4j projection diff --git a/pyproject.toml b/pyproject.toml index 5965942..5f8aa1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codeanalyzer-python" -version = "1.0.1" +version = "1.0.2" description = "Static analysis for Python — canonical schema v2 (symbol table, call graph, and native CFG/PDG/SDG dataflow) as analysis.json or a Neo4j property graph." readme = "README.md" authors = [ From 57c4043ab44b6ab40fa0617ab72d5fd7be127886 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Tue, 21 Jul 2026 18:30:40 -0400 Subject: [PATCH 08/22] fix(env): parso-version-aware interpreter selection for the analysis venv (#107) jedi parses the analysis environment's Python with parso, which ships one hardcoded grammar per minor version. On hosts whose default python3 is newer than the newest shipped grammar (e.g. 3.14 with parso <= 0.8.4), every file failed with 'Python version 3.14 is currently not supported' and the run still exited 0 with an empty symbol table. Provisioning now derives parso's ceiling at runtime from its shipped grammar files and swaps a too-new default for the newest supported interpreter on the host (versioned PATH names, then pyenv installs), falling back loudly only when none exists. An explicit SYSTEM_PYTHON is honored with a warning when unsupported. A run in which every discovered file fails now logs a prominent error instead of staying silent, and parso>=0.8.5 (first release with the 3.14 grammar) is a direct dependency. --- CHANGELOG.md | 11 +++ codeanalyzer/core.py | 169 +++++++++++++++++++++++++++++++++-- pyproject.toml | 3 + test/test_env_interpreter.py | 156 ++++++++++++++++++++++++++++++++ 4 files changed, 331 insertions(+), 8 deletions(-) create mode 100644 test/test_env_interpreter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f31363..6e85c34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- **Analysis env provisioning is parso-version-aware** (#107): on hosts whose default + `python3` is newer than the newest grammar the installed parso ships (e.g. Python 3.14 + with parso ≤ 0.8.4), every file failed jedi parsing and the run completed "successfully" + with an empty symbol table. Provisioning now derives parso's supported ceiling at runtime + from its shipped grammar files and prefers the newest supported interpreter on the host + (versioned PATH names, then pyenv installs), falling back loudly only when none exists; + an explicit `SYSTEM_PYTHON` is still honored, with a warning when unsupported. A run in + which every discovered file fails now logs a prominent error instead of staying silent, + and `parso>=0.8.5` (the first release with the 3.14 grammar) is a direct dependency. + ## [1.0.2] - 2026-07-16 ### Fixed diff --git a/codeanalyzer/core.py b/codeanalyzer/core.py index a533d0f..05b464d 100644 --- a/codeanalyzer/core.py +++ b/codeanalyzer/core.py @@ -163,8 +163,159 @@ def _cmd_exec_helper( stderr=None, ) + @classmethod + def _get_base_interpreter(cls) -> Path: + """The interpreter used to provision the analysis virtualenv. + + jedi parses the *analysis environment's* Python version with parso, + which ships one hardcoded grammar file per minor version — an + environment newer than the newest shipped grammar makes every file + fail with "Python version X.Y is currently not supported" while the + run still exits 0 (#107). So the default choice is gated on the + installed parso's ceiling: a too-new default is swapped for the + newest supported interpreter found on the host, falling back to the + default (loudly) only when none exists. An explicit ``SYSTEM_PYTHON`` + always wins, with a warning when parso cannot parse its version. + """ + # An explicit SYSTEM_PYTHON override wins (consulted only when running + # inside a virtualenv, matching the historical behavior). + if sys.prefix != sys.base_prefix: + system_python = os.getenv("SYSTEM_PYTHON") + if system_python: + system_python_path = Path(system_python) + if system_python_path.exists() and system_python_path.is_file(): + ceiling = cls._parso_supported_ceiling() + version = cls._interpreter_version(system_python_path) + if ceiling is not None and version is not None and version > ceiling: + logger.warning( + f"SYSTEM_PYTHON={system_python} is Python " + f"{version[0]}.{version[1]}, newer than the newest grammar " + f"the installed parso ships ({ceiling[0]}.{ceiling[1]}). " + "jedi will likely reject every file in the analysis " + "environment (#107); honoring the explicit override anyway." + ) + return system_python_path + + candidate = cls._default_base_interpreter() + ceiling = cls._parso_supported_ceiling() + if ceiling is None: + return candidate + version = cls._interpreter_version(candidate) + if version is None or version <= ceiling: + return candidate + logger.warning( + f"Default interpreter {candidate} is Python {version[0]}.{version[1]}, " + f"newer than the newest grammar the installed parso ships " + f"({ceiling[0]}.{ceiling[1]}) — looking for a supported interpreter " + "for the analysis environment (#107)." + ) + supported = cls._find_supported_interpreter(ceiling) + if supported is not None: + logger.info(f"Provisioning the analysis environment with {supported}.") + return supported + logger.warning( + f"No interpreter <= {ceiling[0]}.{ceiling[1]} found on this host; " + f"falling back to {candidate}. jedi/parso will likely reject every " + "file — install a supported Python or upgrade parso." + ) + return candidate + + @staticmethod + def _versions_from_grammar_stems(stems: List[str]) -> List[tuple]: + """``grammar313`` → ``(3, 13)``, sorted ascending; malformed stems dropped.""" + versions = [] + for stem in stems: + digits = stem[len("grammar"):] + if len(digits) >= 2 and digits.isdigit(): + versions.append((int(digits[0]), int(digits[1:]))) + return sorted(versions) + + @classmethod + def _parso_supported_ceiling(cls) -> Optional[tuple]: + """Newest ``(major, minor)`` the installed parso ships a grammar for, + derived from its ``python/grammar*.txt`` files so the ceiling moves + automatically when parso adds a version. ``None`` if undeterminable.""" + try: + import parso + + stems = [ + p.stem + for p in (Path(parso.__file__).parent / "python").glob("grammar*.txt") + ] + versions = cls._versions_from_grammar_stems(stems) + return versions[-1] if versions else None + except Exception: + return None + + @staticmethod + def _interpreter_version(interpreter: Path) -> Optional[tuple]: + """``(major, minor)`` of an interpreter, or ``None`` if it can't run.""" + try: + result = subprocess.run( + [ + str(interpreter), + "-c", + "import sys; print('%d.%d' % sys.version_info[:2])", + ], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + major, minor = result.stdout.strip().split(".") + return (int(major), int(minor)) + except (subprocess.TimeoutExpired, FileNotFoundError, PermissionError, ValueError): + pass + return None + + @staticmethod + def _pick_supported_interpreter( + candidates: List[tuple], ceiling: tuple + ) -> Optional[Path]: + """Newest candidate whose version is within the ceiling. + + ``candidates`` is ``[(path, (major, minor) | None), ...]``.""" + supported = [ + (version, path) + for path, version in candidates + if version is not None and version <= ceiling + ] + return max(supported)[1] if supported else None + + @classmethod + def _find_supported_interpreter(cls, ceiling: tuple) -> Optional[Path]: + """Search the host for the newest interpreter within the parso ceiling: + versioned names on PATH (``python3.13``, ``python3.12``, ...) first, + then pyenv installs.""" + paths: List[Path] = [] + for minor in range(ceiling[1], 7, -1): + which = shutil.which(f"python{ceiling[0]}.{minor}") + # Skip the current virtualenv's own interpreter (same rule as + # _default_base_interpreter): the analysis env must come from a + # base installation. + if which and not which.startswith(sys.prefix): + paths.append(Path(which)) + for pyenv_root in (os.getenv("PYENV_ROOT"), str(Path.home() / ".pyenv")): + if not pyenv_root: + continue + versions_dir = Path(pyenv_root) / "versions" + if versions_dir.is_dir(): + for install in sorted(versions_dir.iterdir(), reverse=True): + exe = install / "bin" / "python3" + if exe.exists(): + paths.append(exe) + seen = set() + candidates = [] + for path in paths: + key = str(path) + if key in seen: + continue + seen.add(key) + candidates.append((path, cls._interpreter_version(path))) + return cls._pick_supported_interpreter(candidates, ceiling) + @staticmethod - def _get_base_interpreter() -> Path: + def _default_base_interpreter() -> Path: """Get the base Python interpreter path. This method finds a suitable base Python interpreter that can be used @@ -183,13 +334,6 @@ def _get_base_interpreter() -> Path: # We're inside a virtual environment; need to find the base interpreter - # First, check if user explicitly set SYSTEM_PYTHON - system_python = os.getenv("SYSTEM_PYTHON") - if system_python: - system_python_path = Path(system_python) - if system_python_path.exists() and system_python_path.is_file(): - return system_python_path - # Try to get the base interpreter from sys.base_executable (Python 3.3+) if hasattr(sys, "base_executable") and sys.base_executable: base_exec = Path(sys.base_executable) @@ -778,6 +922,15 @@ def _build_symbol_table(self, cached_symbol_table: Optional[Dict[str, PyModule]] if files_from_cache > 0: logger.info(f"Reused {files_from_cache} files from cache, processed {files_processed} new/changed files") + if py_files and not symbol_table: + logger.error( + "Every one of the %d discovered Python files failed to process — " + "the symbol table is empty. This usually means the analysis " + "environment's interpreter is newer than the installed jedi/parso " + "stack supports (#107); check the per-file errors above.", + len(py_files), + ) + logger.info( "✅ Symbol table: %d modules in %.1fs", len(symbol_table), time.perf_counter() - t0_st, diff --git a/pyproject.toml b/pyproject.toml index 5f8aa1a..cb38cb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,9 @@ dependencies = [ # jedi "jedi>=0.18.0,<0.20.0; python_version < '3.11'", "jedi<=0.19.2; python_version >= '3.11'", + # parso 0.8.5 is the first release shipping the Python 3.14 grammar; older + # resolutions make jedi reject every file in a 3.14 analysis env (#107) + "parso>=0.8.5", # msgpack "msgpack>=1.0.0,<1.0.7; python_version < '3.11'", "msgpack>=1.0.7,<2.0.0; python_version >= '3.11'", diff --git a/test/test_env_interpreter.py b/test/test_env_interpreter.py new file mode 100644 index 0000000..68b1989 --- /dev/null +++ b/test/test_env_interpreter.py @@ -0,0 +1,156 @@ +"""Regression tests for #107: environment provisioning must prefer an +interpreter the installed jedi/parso stack can actually parse, and a run +where every module fails must not stay silent. + +parso ships one hardcoded grammar file per Python minor (grammar313.txt, +grammar314.txt, ...). If the provisioned analysis venv is newer than the +newest shipped grammar, jedi rejects every file and the symbol table comes +back empty while the process still exits 0. +""" +import logging +from pathlib import Path + +import pytest + +from codeanalyzer.core import Codeanalyzer + + +# ---------------------------------------------------------------------------------------------- +# The parso ceiling: derived from the shipped grammar files at runtime, never hardcoded. +# ---------------------------------------------------------------------------------------------- + + +def test_grammar_stems_parse_to_versions(): + got = Codeanalyzer._versions_from_grammar_stems( + ["grammar36", "grammar39", "grammar310", "grammar313", "grammar314"] + ) + assert got == [(3, 6), (3, 9), (3, 10), (3, 13), (3, 14)] + + +def test_malformed_grammar_stems_are_ignored(): + got = Codeanalyzer._versions_from_grammar_stems( + ["grammar", "grammarXY", "grammar3", "grammar312"] + ) + assert got == [(3, 12)] + + +def test_parso_ceiling_reflects_installed_parso(): + """The ceiling must be the max of the grammars parso actually ships — + on any env with parso >= 0.8.5 that is at least (3, 13).""" + ceiling = Codeanalyzer._parso_supported_ceiling() + assert ceiling is not None + assert ceiling >= (3, 13) + + +# ---------------------------------------------------------------------------------------------- +# Interpreter choice honors the ceiling. +# ---------------------------------------------------------------------------------------------- + + +def test_pick_supported_interpreter_prefers_newest_within_ceiling(): + candidates = [ + (Path("/opt/py315"), (3, 15)), + (Path("/opt/py313"), (3, 13)), + (Path("/opt/py311"), (3, 11)), + ] + assert Codeanalyzer._pick_supported_interpreter(candidates, (3, 14)) == Path( + "/opt/py313" + ) + assert Codeanalyzer._pick_supported_interpreter(candidates, (3, 10)) is None + + +def test_base_interpreter_swaps_out_unsupported_default(monkeypatch): + """If the default interpreter is newer than parso's ceiling, provisioning + must pick a supported one instead.""" + fake_default = Path("/opt/py399/bin/python3") + fake_supported = Path("/opt/py313/bin/python3") + + monkeypatch.setattr( + Codeanalyzer, "_default_base_interpreter", staticmethod(lambda: fake_default) + ) + monkeypatch.setattr( + Codeanalyzer, "_parso_supported_ceiling", staticmethod(lambda: (3, 13)) + ) + monkeypatch.setattr( + Codeanalyzer, + "_interpreter_version", + staticmethod(lambda p: (3, 99) if p == fake_default else (3, 13)), + ) + monkeypatch.setattr( + Codeanalyzer, + "_find_supported_interpreter", + staticmethod(lambda ceiling: fake_supported), + ) + assert Codeanalyzer._get_base_interpreter() == fake_supported + + +def test_base_interpreter_keeps_supported_default(monkeypatch): + fake_default = Path("/opt/py312/bin/python3") + monkeypatch.setattr( + Codeanalyzer, "_default_base_interpreter", staticmethod(lambda: fake_default) + ) + monkeypatch.setattr( + Codeanalyzer, "_parso_supported_ceiling", staticmethod(lambda: (3, 13)) + ) + monkeypatch.setattr( + Codeanalyzer, "_interpreter_version", staticmethod(lambda p: (3, 12)) + ) + assert Codeanalyzer._get_base_interpreter() == fake_default + + +def test_base_interpreter_falls_back_loudly_when_nothing_supported(monkeypatch, caplog): + fake_default = Path("/opt/py399/bin/python3") + monkeypatch.setattr( + Codeanalyzer, "_default_base_interpreter", staticmethod(lambda: fake_default) + ) + monkeypatch.setattr( + Codeanalyzer, "_parso_supported_ceiling", staticmethod(lambda: (3, 13)) + ) + monkeypatch.setattr( + Codeanalyzer, "_interpreter_version", staticmethod(lambda p: (3, 99)) + ) + monkeypatch.setattr( + Codeanalyzer, "_find_supported_interpreter", staticmethod(lambda ceiling: None) + ) + # the codeanalyzer logger sets propagate=False (rich handler); caplog needs + # propagation to observe records + monkeypatch.setattr(logging.getLogger("codeanalyzer"), "propagate", True) + with caplog.at_level(logging.WARNING, logger="codeanalyzer"): + assert Codeanalyzer._get_base_interpreter() == fake_default + assert any("parso" in r.getMessage() for r in caplog.records) + + +# ---------------------------------------------------------------------------------------------- +# A run where every module failed must be loud, not silently empty. +# ---------------------------------------------------------------------------------------------- + + +def test_all_files_failing_emits_an_error(tmp_path, monkeypatch, caplog): + proj = tmp_path / "proj" + proj.mkdir() + (proj / "a.py").write_text("def f():\n return 1\n", encoding="utf-8") + (proj / "b.py").write_text("def g():\n return 2\n", encoding="utf-8") + + from codeanalyzer.options.options import AnalysisOptions + from codeanalyzer.config import OutputFormat + from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder + + def boom(self, py_file): + raise RuntimeError("Python version 3.99 is currently not supported.") + + monkeypatch.setattr(SymbolTableBuilder, "build_pymodule_from_file", boom) + + opts = AnalysisOptions( + input=proj, output=None, format=OutputFormat.JSON, + skip_tests=True, no_venv=True, cache_dir=tmp_path / "cache", + rebuild_analysis=True, + ) + analyzer = Codeanalyzer(opts) + monkeypatch.setattr(logging.getLogger("codeanalyzer"), "propagate", True) + with caplog.at_level(logging.ERROR, logger="codeanalyzer"): + table = analyzer._build_symbol_table(cached_symbol_table={}) + assert table == {} + assert any( + "every" in r.getMessage().lower() or "all " in r.getMessage().lower() + for r in caplog.records + ), "an empty symbol table from total per-file failure must be reported loudly" From 8fdd50d2e57a939a43ec7db84a82798e8d3fb0ef Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Tue, 21 Jul 2026 18:32:09 -0400 Subject: [PATCH 09/22] chore(release): 1.0.3 --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e85c34..82479b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.0.3] - 2026-07-21 + ### Fixed - **Analysis env provisioning is parso-version-aware** (#107): on hosts whose default `python3` is newer than the newest grammar the installed parso ships (e.g. Python 3.14 diff --git a/pyproject.toml b/pyproject.toml index cb38cb7..69820b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codeanalyzer-python" -version = "1.0.2" +version = "1.0.3" description = "Static analysis for Python — canonical schema v2 (symbol table, call graph, and native CFG/PDG/SDG dataflow) as analysis.json or a Neo4j property graph." readme = "README.md" authors = [ From 4405aa0bf4a4e784f0bc81eb1c0970421f97e8a7 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 22 Jul 2026 15:57:13 -0400 Subject: [PATCH 10/22] docs(spec): vendored typed_ast-free scalpel as the default L4 oracle Vendor the 9-module SSA/cfg/core slice of python-scalpel 1.0b0 (Apache-2.0, typed_ast-free) into codeanalyzer/dataflow/scalpel/, making ScalpelAliasOracle the shipping-default L4 oracle on Python 3.9-3.14+ with no external python-scalpel/typed_ast dependency. Type-based oracle demotes to the runtime safety net. Includes feasibility evidence, oracle rewiring, the L4 precision behavior change, and the testing plan. --- ...-vendored-scalpel-default-oracle-design.md | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-22-vendored-scalpel-default-oracle-design.md diff --git a/docs/superpowers/specs/2026-07-22-vendored-scalpel-default-oracle-design.md b/docs/superpowers/specs/2026-07-22-vendored-scalpel-default-oracle-design.md new file mode 100644 index 0000000..fb0ac43 --- /dev/null +++ b/docs/superpowers/specs/2026-07-22-vendored-scalpel-default-oracle-design.md @@ -0,0 +1,198 @@ +# Vendored, typed_ast-free Scalpel as the default L4 oracle + +- **Date:** 2026-07-22 +- **Status:** Design approved; ready for an implementation plan +- **Scope:** Make the Scalpel-backed points-to oracle (`ScalpelAliasOracle`) the + shipping default for L4 dataflow on **every** supported Python, by vendoring a + minimal, `typed_ast`-free slice of `python-scalpel` into the package. + +## Motivation + +`python-scalpel` is the primary L4 may-alias oracle, but it ships today as an +**optional** extra with a type-based fallback — because `python-scalpel 1.0b0` +hard-depends on **`typed_ast`**, an abandoned package whose last release +(`1.5.5`) has no wheel for Python 3.12+ and fails to compile from source on a +modern compiler. So on the interpreters most users (and this repo's own dev env) +now run — 3.12, 3.13, 3.14 — scalpel cannot be installed at all, and L4 silently +degrades to the coarser type-based oracle. + +We want scalpel to be the **default** L4 oracle everywhere. It cannot be a hard +PyPI dependency (the `typed_ast` wall), and a Python-version cap to reach it +(≤3.11) would drop the 3.12–3.14 support the analyzer just added. The way +through is to **vendor** the small slice of scalpel we use, minus the one module +that drags `typed_ast`. + +## Feasibility (verified, not assumed) + +- **`typed_ast` is spurious for our use.** It is imported in exactly one scalpel + module — `scalpel/typeinfer/analysers.py` — which codeanalyzer never touches. + The oracle uses only `scalpel.SSA.const` and `scalpel.cfg`. +- **The slice runs without `typed_ast`.** On Python 3.12 with `typed_ast` *not* + installed, `cfg.CFGBuilder` + `SSA.const.SSA` import and `compute_SSA` returns + the exact SSA/const facts the oracle consumes (verified: a copy chain yields + `('b',0), ('c',0), ('',0)`). +- **Exact closure = 9 modules**, provably free of `typeinfer`/`typed_ast` + (checked via `sys.modules` after importing the two entry points): + `scalpel/__init__.py`, `SSA/{__init__,const}.py`, + `cfg/{__init__,builder,model}.py`, `core/{__init__,func_call_visitor,vars_visitor}.py`. +- **Third-party imports touched:** `astor` (used at runtime — `SSA/const.py:191` + `astor.to_source`), `graphviz` (imported at module load in `cfg/model.py` but + only used by the unused `build_visual()`), `networkx` (already a dependency). +- **License = Apache-2.0** (confirmed on `SMAT-Lab/Scalpel` and in the wheel's + `LICENSE`) — vendorable with attribution, same family as the already-bundled + PyCG. +- **Upstream is abandoned** (`1.0b0`, frozen) — the usual "vendored copy drifts + from upstream" cost does not apply. + +## Goals + +- `ScalpelAliasOracle` is the **default** L4 oracle on Python 3.9–3.14+, with no + external `python-scalpel`/`typed_ast` dependency and no `pip install` breakage. +- The type-based oracle (`TypeBasedAliasOracle`) is retained purely as the + runtime safety net (per-callable build failure, per-query unresolved path). +- Behavior for users who already had the `[scalpel]` extra on ≤3.11 is unchanged + (same code). Behavior on 3.12+ *improves* (real points-to instead of the + type-based over-approximation). + +## Non-goals + +- No change to the L4 *schema* (`prov` values `ssa`/`points-to` unchanged; the + DDG shape is the same). +- No `requires-python` change — stays `>=3.9`. +- No vendoring of scalpel's `typeinfer`, `call_graph`, `pycg`, `import_graph`, + `scope_graph`, or `dataflow` packages — only the 9-module `SSA`/`cfg`/`core` + closure the oracle actually loads. +- No change to `defuse.py`'s two-rule DDG contract or the `k_limit` machinery. + +## Design + +### The vendored package: `codeanalyzer/dataflow/scalpel/` + +A normal package (no `_vendor/` layer), mirroring upstream's layout so scalpel's +relative imports (`from ..core.vars_visitor import get_vars`) resolve within +`codeanalyzer.dataflow.scalpel`: + +``` +codeanalyzer/dataflow/scalpel/ + __init__.py + SSA/__init__.py, const.py + cfg/__init__.py, builder.py, model.py + core/__init__.py, func_call_visitor.py, vars_visitor.py + LICENSE # upstream Apache-2.0, copied verbatim + README.md # provenance + the one patch (see below) +``` + +The `scalpel/` wrapper name is kept (rather than flattening `SSA`/`cfg`/`core` +into `dataflow/`) because `codeanalyzer/dataflow/cfg.py` already exists and +scalpel ships its own `cfg/` package — the wrapper namespaces them apart. + +The 9 files are copied **verbatim** from `python-scalpel 1.0b0`, with **exactly +one patch**: + +- `cfg/model.py`: the module-load `import graphviz as gv` (line 12) is moved to a + lazy import *inside* the unused `build_visual()` method. This removes `graphviz` + as a runtime dependency (we never render). No other line changes; the + SSA/CFG-computation code paths are byte-identical to upstream. + +### Attribution (Apache-2.0) + +- `codeanalyzer/dataflow/scalpel/LICENSE` — upstream's Apache-2.0 license, copied. +- `codeanalyzer/dataflow/scalpel/README.md` — records: source repo + (`SMAT-Lab/Scalpel`), the pinned version (`1.0b0`), the exact 9-file list, the + single `graphviz`-lazy patch, and that `typeinfer` (the sole `typed_ast` user) + was deliberately excluded. +- The repo's top-level `NOTICE` gains a one-line entry crediting vendored Scalpel + (Apache-2.0), alongside the existing attributions. + +### Dependencies (`pyproject.toml`) + +- **Add** `astor` to core `dependencies` (genuine runtime dep; pure-Python; + installs on every platform/Python). +- **Remove** the `[project.optional-dependencies].scalpel` extra (scalpel is now + built in). +- `networkx` unchanged (already core); `graphviz` and `typed_ast` are **not** + dependencies. +- `requires-python` stays `>=3.9`. + +### Oracle rewiring: `codeanalyzer/dataflow/scalpel_oracle.py` + +- `ScalpelAliasOracle.from_function` imports from the vendored path: + `from codeanalyzer.dataflow.scalpel.SSA.const import SSA` and + `from codeanalyzer.dataflow.scalpel.cfg import CFGBuilder`. +- `make_alias_oracle`: the `ImportError` ("python-scalpel not installed") branch + becomes dead — the vendored import cannot be absent — and is **removed**. The + per-callable **build-failure** `except` (returns the type-based fallback) and + the per-query unresolved-path fallback inside `may_alias` both **stay**, so the + total, never-raises contract is preserved. `ScalpelAliasOracle` is now the + default the selector returns. +- Docstrings/comments updated to drop the "optional / not installed" framing. + +### Behavior change + +On Python 3.12+/3.14 — and for anyone who never installed the old `[scalpel]` +extra — L4's `prov:["points-to"]` DDG edges were previously derived from the +*type-based* over-approximation; they now become **scalpel-precise** (a tighter +subset). The `prov:["ssa"]` edges (the L3 subset) are unchanged, so the +**monotonicity invariant `L3 ⊆ L4` still holds** (the ssa set is untouched; +points-to is an additive overlay). Users who already had the `[scalpel]` extra on +≤3.11 see no change. This L4 precision improvement is recorded in `CHANGELOG.md`. + +### Docs + +`CLAUDE.md` and `.claude/SCHEMA_DECISIONS.md` are updated to record: scalpel is +now **vendored** (`codeanalyzer/dataflow/scalpel/`, `typed_ast`-free) and the +**shipping default** L4 oracle on all supported Python — superseding the earlier +"the type-based oracle is the sanctioned fallback, not the shipping default" +statement (it is now the runtime safety net, not the default). The Stage-0 +`SCHEMA_DECISIONS` entry gains a follow-up noting the `typed_ast` wall on 3.12+ +that forced vendoring. + +## Testing plan + +1. **Import hygiene.** `import codeanalyzer.dataflow.scalpel` and building the + oracle succeed with no external `scalpel`/`typed_ast`/`graphviz` installed; + assert `typeinfer` is never imported (scan `sys.modules` after building the + oracle). +2. **`typed_ast`-free property gate.** Assert `typed_ast` is not importable in + the environment, yet `make_alias_oracle` builds a `ScalpelAliasOracle` and + `may_alias` returns verdicts on a copy-chain fixture — the whole point, + encoded as a regression gate. +3. **Vendored-fidelity check.** On Python ≤3.11 (where pip `python-scalpel` + installs), compare the vendored `compute_SSA`/CFG output against the + pip-installed scalpel on a set of fixtures — proves the copy is faithful. + `skipif` the pip package can't install (3.12+); the copy is verbatim, so this + is a belt-and-suspenders check. +4. **L4 regression (the live acceptance test).** `test_v2_l4.py`, + `test_v2_l4_summary.py`, `test_dataflow_sdg.py`, `test_dataflow_defuse.py` now + exercise the **scalpel** path by default — the first time scalpel-L4 runs on + the 3.14 dev env. They must stay green and now genuinely test scalpel-derived + points-to rather than the fallback. +5. **Determinism.** Run L4 twice on a fixture and assert identical output (the + scalpel SSA feeds `may_alias`; the `#99` `PYTHONHASHSEED=0` pin + the oracle's + access-path normalization must keep it stable). + +## Risks and rollback + +- **Incomplete closure** — a missed transitive import would surface as an + `ImportError` at oracle build, which the per-callable fallback swallows into a + silent type-based degrade. Mitigation: the import-hygiene test (1) builds the + oracle and asserts it is a `ScalpelAliasOracle`, so a broken closure fails + loudly rather than degrading silently. +- **Vendored-copy divergence from upstream behavior** — mitigated by the verbatim + copy (only the `graphviz` import patched) and the fidelity check (3). +- **L4 output change on 3.12+** is intended, documented in the CHANGELOG, and + bounded by the preserved monotonicity invariant. +- **Rollback** is a single revert: delete `codeanalyzer/dataflow/scalpel/`, + restore the `[scalpel]` extra + the `make_alias_oracle` ImportError branch, and + drop `astor`. No schema or id changes to unwind. + +## Out of scope + +- Replacing `astor.to_source` with stdlib `ast.unparse` to drop the `astor` dep + (a behavior-risking change to vendored SSA logic — deferred; `astor` is tiny + and universal). +- A scalpel-installed CI lane / capturing the L4 (`a4`) equivalence goldens from + the separate analysis-pipeline branch — that work belongs to that branch once + it merges; here scalpel simply becomes available for it. +- Vendoring or using scalpel's type-inference (`typeinfer`) for the type-guided + alias branch — the type-based oracle already covers that fallback. From d4fed11746a52c2d3bbf28a11f171559385ed7cf Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 22 Jul 2026 16:04:48 -0400 Subject: [PATCH 11/22] docs(plan): implementation plan for vendored scalpel default L4 oracle --- ...6-07-22-vendored-scalpel-default-oracle.md | 455 ++++++++++++++++++ 1 file changed, 455 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-22-vendored-scalpel-default-oracle.md diff --git a/docs/superpowers/plans/2026-07-22-vendored-scalpel-default-oracle.md b/docs/superpowers/plans/2026-07-22-vendored-scalpel-default-oracle.md new file mode 100644 index 0000000..18520bb --- /dev/null +++ b/docs/superpowers/plans/2026-07-22-vendored-scalpel-default-oracle.md @@ -0,0 +1,455 @@ +# Vendored typed_ast-free Scalpel as Default L4 Oracle — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Vendor a minimal, `typed_ast`-free slice of `python-scalpel 1.0b0` into `codeanalyzer/dataflow/scalpel/` and make `ScalpelAliasOracle` the shipping-default L4 oracle on every supported Python, with no external `python-scalpel`/`typed_ast` dependency. + +**Architecture:** Copy the 9-module `SSA`/`cfg`/`core` closure the oracle actually loads (verified free of `typeinfer`/`typed_ast`) into a `codeanalyzer.dataflow.scalpel` package, patched only to make `graphviz` a non-required import. Repoint `scalpel_oracle.py` at the vendored path and drop its "optional dependency absent" fallback branch, so the type-based oracle demotes to a pure runtime safety net. + +**Tech Stack:** Python 3.9–3.14, `astor` (new runtime dep), `networkx` (existing), pytest, Poetry (dev env, currently 3.14.5), uv (CI + fetching the vendored source). + +## Global Constraints + +- Conventional Commits (`type(scope): summary`). +- NEVER add AI/Claude authorship anywhere (no `Co-Authored-By`, "Generated with", 🤖) — commits, PRs, code, docs. Absolute. +- **Apache-2.0 attribution is mandatory** for the vendored code: copy upstream `LICENSE` into the vendored dir, add a provenance `README.md`, and add a `NOTICE` entry. +- **`requires-python` stays `>=3.9`** — do NOT cap it. +- **Behavior:** the monotonicity invariant `L3 ⊆ L4` must still hold (the `prov:["ssa"]` edge set is unchanged; scalpel only sharpens the additive `prov:["points-to"]` overlay). No schema change (`schema_version` stays `2.0.0`). +- The vendored copy is **verbatim** from `python-scalpel 1.0b0` except the single documented `graphviz` patch. +- Vendor **exactly** these 9 files — no other scalpel modules (`typeinfer`, `call_graph`, `pycg`, `import_graph`, `scope_graph`, `dataflow`, `rewriter.py`, and the unused `SSA/{alg,def_use,ssa}.py` + extra `core/*.py`): + `__init__.py`, `SSA/__init__.py`, `SSA/const.py`, `cfg/__init__.py`, `cfg/builder.py`, `cfg/model.py`, `core/__init__.py`, `core/func_call_visitor.py`, `core/vars_visitor.py`. +- Run tests with `poetry run python -m pytest` (managed env, py3.14). Do NOT use `uv run` for the dev env (it creates a stray in-project `.venv` that shadows Poetry). There must be no in-project `.venv`. + +## File Structure + +- Create `codeanalyzer/dataflow/scalpel/` — the 9 vendored files + `LICENSE` + `README.md`. +- Modify `codeanalyzer/dataflow/scalpel/cfg/model.py` — the one `graphviz`-lazy patch. +- Modify `codeanalyzer/dataflow/scalpel_oracle.py` — repoint imports, drop the dead ImportError branch. +- Modify `pyproject.toml` — add `astor`, remove the `[scalpel]` extra. +- Modify `NOTICE` — Scalpel Apache-2.0 attribution. +- Modify `CLAUDE.md`, `.claude/SCHEMA_DECISIONS.md`, `CHANGELOG.md` — record the reversal + behavior change. +- Create `test/test_vendored_scalpel.py` — import-hygiene, typed_ast-free, fidelity, determinism tests. + +--- + +### Task 1: Vendor the scalpel slice + dependency + attribution + +**Files:** +- Create: `codeanalyzer/dataflow/scalpel/**` (9 files + `LICENSE` + `README.md`) +- Modify: `codeanalyzer/dataflow/scalpel/cfg/model.py` (graphviz patch) +- Modify: `pyproject.toml` (add `astor`, remove `[scalpel]` extra) +- Modify: `NOTICE` +- Test: `test/test_vendored_scalpel.py` + +**Interfaces:** +- Produces: importable `codeanalyzer.dataflow.scalpel.SSA.const.SSA` and `codeanalyzer.dataflow.scalpel.cfg.CFGBuilder`, functional with no external `scalpel`/`typed_ast`/`graphviz`. + +- [ ] **Step 1: Write the failing import-hygiene test** + +Create `test/test_vendored_scalpel.py`: + +```python +"""The vendored, typed_ast-free scalpel slice: it must import and compute SSA +with no external scalpel / typed_ast / graphviz, and never pull in typeinfer.""" +import sys +import importlib + + +def test_vendored_scalpel_imports_and_computes_ssa_typed_ast_free(): + # No external scalpel shadowing the vendored copy. + assert "scalpel" not in sys.modules or not sys.modules["scalpel"].__file__.endswith( + "site-packages/scalpel/__init__.py" + ), "external python-scalpel is installed; the test must exercise the vendored copy" + + from codeanalyzer.dataflow.scalpel.cfg import CFGBuilder + from codeanalyzer.dataflow.scalpel.SSA.const import SSA + + src = "def f(a):\n b = a\n c = b\n return c\n" + module_cfg = CFGBuilder().build_from_src("m", src) + func_cfg = list(module_cfg.functioncfgs.values())[0] + ssa_results, const_dict = SSA().compute_SSA(func_cfg) + # The copy chain + return pseudo-name scalpel is known to produce. + assert ("b", 0) in const_dict and ("c", 0) in const_dict and ("", 0) in const_dict + + # The whole point: no typed_ast, and typeinfer was never vendored/loaded. + assert "typed_ast" not in sys.modules + loaded = [m for m in sys.modules if m.startswith("codeanalyzer.dataflow.scalpel")] + assert not any("typeinfer" in m for m in loaded), f"typeinfer leaked: {loaded}" +``` + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `poetry run python -m pytest test/test_vendored_scalpel.py -q` +Expected: FAIL with `ModuleNotFoundError: No module named 'codeanalyzer.dataflow.scalpel'`. + +- [ ] **Step 3: Vendor the 9 files + LICENSE (reproducible fetch)** + +Fetch the exact upstream source and copy only the 9 files + the license: + +```bash +cd /home/rkrsn/workspace/codellm-devkit/codeanalyzer-python +TMP=$(mktemp -d) +uv pip install --no-deps --target "$TMP" 'python-scalpel==1.0b0' +DST=codeanalyzer/dataflow/scalpel +mkdir -p "$DST/SSA" "$DST/cfg" "$DST/core" +cp "$TMP/scalpel/__init__.py" "$DST/__init__.py" +cp "$TMP/scalpel/SSA/__init__.py" "$DST/SSA/__init__.py" +cp "$TMP/scalpel/SSA/const.py" "$DST/SSA/const.py" +cp "$TMP/scalpel/cfg/__init__.py" "$DST/cfg/__init__.py" +cp "$TMP/scalpel/cfg/builder.py" "$DST/cfg/builder.py" +cp "$TMP/scalpel/cfg/model.py" "$DST/cfg/model.py" +cp "$TMP/scalpel/core/__init__.py" "$DST/core/__init__.py" +cp "$TMP/scalpel/core/func_call_visitor.py" "$DST/core/func_call_visitor.py" +cp "$TMP/scalpel/core/vars_visitor.py" "$DST/core/vars_visitor.py" +cp "$TMP"/python_scalpel-1.0b0.dist-info/LICENSE "$DST/LICENSE" +rm -rf "$TMP" +``` + +Verify exactly 9 `.py` files landed: + +```bash +find codeanalyzer/dataflow/scalpel -name '*.py' | sort +# expect the 9 listed in Global Constraints, nothing else +``` + +- [ ] **Step 4: Apply the single graphviz patch** + +In `codeanalyzer/dataflow/scalpel/cfg/model.py`, the top-level `import graphviz as gv` (line 12) makes the module require `graphviz` at load. `graphviz` is used only by the visualization methods (`_build_visual`/`build_visual`), which codeanalyzer never calls. Replace the bare import: + +```python +import graphviz as gv +``` + +with a guarded import so the module loads without `graphviz`: + +```python +try: # PATCH (codeanalyzer): graphviz is only used by build_visual(), which + import graphviz as gv # codeanalyzer never calls. Keep the module importable +except ImportError: # without the graphviz dependency. + gv = None +``` + +This is the ONLY change to any vendored file. + +- [ ] **Step 5: Add attribution — `README.md` + `NOTICE` entry** + +Create `codeanalyzer/dataflow/scalpel/README.md`: + +```markdown +# Vendored Scalpel (typed_ast-free slice) + +Vendored from **[SMAT-Lab/Scalpel](https://github.com/SMAT-Lab/Scalpel)**, +package `python-scalpel==1.0b0`, licensed **Apache-2.0** (see `LICENSE`). + +## Why vendored + +`python-scalpel` hard-depends on `typed_ast`, whose last release (1.5.5) has no +wheel for Python 3.12+ and fails to build from source on modern compilers — so +`pip install python-scalpel` fails on 3.12/3.13/3.14. `typed_ast` is imported by +exactly one scalpel module, `typeinfer/analysers.py`, which codeanalyzer does not +use. Vendoring the small slice the L4 may-alias oracle needs removes the +`typed_ast` dependency and makes scalpel the default oracle on every supported +Python. + +## What is vendored + +Exactly the 9-module closure that `scalpel.SSA.const` + `scalpel.cfg` load +(verified via `sys.modules`; provably free of `typeinfer`/`typed_ast`): + + __init__.py + SSA/__init__.py, SSA/const.py + cfg/__init__.py, cfg/builder.py, cfg/model.py + core/__init__.py, core/func_call_visitor.py, core/vars_visitor.py + +Copied verbatim except **one patch**: `cfg/model.py`'s top-level +`import graphviz as gv` is guarded (`try/except ImportError: gv = None`) so the +module imports without the `graphviz` package — the graphviz-using +`build_visual()` methods are unused here. + +Runtime deps of this slice: `astor`, `networkx` (both core dependencies of +codeanalyzer). `typed_ast` and `graphviz` are NOT required. + +To refresh: re-run the vendoring in `docs/superpowers/plans/2026-07-22-vendored-scalpel-default-oracle.md` Task 1. +``` + +Append to the top-level `NOTICE` (after the existing content): + +``` +--- Scalpel License Notice --- + +This project vendors a slice of Scalpel (python-scalpel), a product of SMAT-Lab, +under codeanalyzer/dataflow/scalpel/. Scalpel is licensed under the Apache +License 2.0. The full license text is included at +codeanalyzer/dataflow/scalpel/LICENSE. Source: https://github.com/SMAT-Lab/Scalpel +``` + +- [ ] **Step 6: Add `astor` dep, remove the `[scalpel]` extra** + +In `pyproject.toml`, add to the core `dependencies` array (after the `uv` entry), with a comment: + +```toml + # astor: runtime dependency of the vendored Scalpel SSA slice + # (scalpel/SSA/const.py uses astor.to_source). Pure-Python, installs + # everywhere; scalpel pins ~=0.8.1. + "astor>=0.8.1,<0.9.0", +``` + +Remove the entire `[project.optional-dependencies].scalpel` group (the +`scalpel = ["python-scalpel>=1.0b0"]` block and its comment). Leave the `neo4j` +extra intact. + +- [ ] **Step 7: Install the new dep into the dev env** + +Run: `poetry install --with test` +Then: `poetry run python -c "import astor; print('astor', astor.__version__)"` +Expected: prints the astor version (installs on 3.14 — pure Python). + +- [ ] **Step 8: Run the hygiene test — now green** + +Run: `poetry run python -m pytest test/test_vendored_scalpel.py -q` +Expected: PASS (the vendored slice imports + computes SSA on 3.14, typed_ast-free). + +- [ ] **Step 9: Commit** + +```bash +git add codeanalyzer/dataflow/scalpel pyproject.toml NOTICE test/test_vendored_scalpel.py +git commit -m "feat(dataflow): vendor typed_ast-free scalpel SSA/cfg slice" +``` + +--- + +### Task 2: Repoint the oracle at the vendored slice and make it the default + +**Files:** +- Modify: `codeanalyzer/dataflow/scalpel_oracle.py` +- Test: `test/test_vendored_scalpel.py` (append) + +**Interfaces:** +- Consumes: the vendored `codeanalyzer.dataflow.scalpel.SSA.const.SSA` / `...cfg.CFGBuilder` (Task 1). +- Produces: `make_alias_oracle(pycallable, func_ast, base_types)` returns a `ScalpelAliasOracle` by default (never the type-based oracle for *absence*, only for a per-callable build failure). + +- [ ] **Step 1: Write the failing "scalpel is the default" test** + +Append to `test/test_vendored_scalpel.py`: + +```python +import ast + + +def test_make_alias_oracle_defaults_to_scalpel_without_typed_ast(): + import sys + assert "typed_ast" not in sys.modules + from codeanalyzer.dataflow.scalpel_oracle import make_alias_oracle, ScalpelAliasOracle + + src = "def f(a):\n b = a\n c = b\n return c\n" + func_ast = ast.parse(src).body[0] + oracle = make_alias_oracle(pycallable=None, func_ast=func_ast, base_types={}) + # The whole point: scalpel is the default, not the type-based fallback. + assert isinstance(oracle, ScalpelAliasOracle), type(oracle).__name__ + # It answers queries (copies alias; unrelated locals do not). + assert oracle.may_alias("b", "c") is True + + +def test_make_alias_oracle_is_deterministic(): + import ast as _ast + from codeanalyzer.dataflow.scalpel_oracle import make_alias_oracle + src = "def f(a):\n b = a\n c = b\n return c\n" + fa = _ast.parse(src).body[0] + o1 = make_alias_oracle(None, _ast.parse(src).body[0], {}) + o2 = make_alias_oracle(None, _ast.parse(src).body[0], {}) + pairs = [("a", "b"), ("b", "c"), ("a", "c"), ("b", "b")] + assert [o1.may_alias(x, y) for x, y in pairs] == [o2.may_alias(x, y) for x, y in pairs] +``` + +- [ ] **Step 2: Run it to confirm it fails** + +Run: `poetry run python -m pytest test/test_vendored_scalpel.py -k make_alias_oracle -q` +Expected: FAIL — `make_alias_oracle` still imports `from scalpel.SSA.const import SSA` (external, absent on 3.14) and returns the type-based fallback, so `isinstance(..., ScalpelAliasOracle)` is False. + +- [ ] **Step 3: Repoint the imports in `from_function`** + +In `codeanalyzer/dataflow/scalpel_oracle.py`, in `ScalpelAliasOracle.from_function`, change: + +```python + from scalpel.SSA.const import SSA + from scalpel.cfg import CFGBuilder +``` + +to: + +```python + from codeanalyzer.dataflow.scalpel.SSA.const import SSA + from codeanalyzer.dataflow.scalpel.cfg import CFGBuilder +``` + +Update that method's docstring line "Imports Scalpel lazily (``ImportError`` if the optional dependency is absent)…" to "Imports the vendored Scalpel slice (``codeanalyzer.dataflow.scalpel``) and reuses the *same source*…". Also update the module docstring reference at the top of the file (`from scalpel.SSA.const import SSA`) to the vendored path. + +- [ ] **Step 4: Drop the dead ImportError branch in `make_alias_oracle`** + +Replace the current `make_alias_oracle` body: + +```python +def make_alias_oracle(pycallable, func_ast, base_types) -> object: + """Total selector for the L4 may-alias oracle. + + Returns a :class:`ScalpelAliasOracle` built on the vendored, typed_ast-free + Scalpel slice (``codeanalyzer.dataflow.scalpel``) — the default L4 oracle. + Falls back to :class:`TypeBasedAliasOracle` only when the per-callable + Scalpel build fails on this AST. Never raises. + """ + fallback = TypeBasedAliasOracle(base_types) + try: + return ScalpelAliasOracle.from_function( + func_ast, base_types=base_types, fallback=fallback + ) + except Exception: + _note_fallback("scalpel alias build failed") + logger.debug("scalpel alias oracle build error", exc_info=True) + return fallback +``` + +(The `except ImportError: _note_fallback("python-scalpel not installed"); return fallback` branch is removed — the vendored import cannot be absent. Keep the per-query `self._fallback.may_alias(...)` inside `may_alias` unchanged.) + +- [ ] **Step 5: Run the oracle tests — now green** + +Run: `poetry run python -m pytest test/test_vendored_scalpel.py -q` +Expected: PASS (scalpel is now the default oracle; determinism holds). + +- [ ] **Step 6: L4 regression — the live acceptance test (scalpel-L4 on 3.14)** + +Run: `poetry run python -m pytest test/test_v2_l4.py test/test_v2_l4_summary.py test/test_dataflow_sdg.py test/test_dataflow_defuse.py -q` +Expected: PASS. These now exercise the vendored **scalpel** path (previously the type-based fallback on 3.14). If a case fails, investigate: a genuine scalpel-vs-type-based points-to difference is expected and the test's expectation may need updating to the (correct, tighter) scalpel result — but confirm it's a precision tightening, not a regression, before changing any assertion. If unsure, report rather than edit the assertion. + +- [ ] **Step 7: Commit** + +```bash +git add codeanalyzer/dataflow/scalpel_oracle.py test/test_vendored_scalpel.py +git commit -m "feat(dataflow): make vendored scalpel the default L4 alias oracle" +``` + +--- + +### Task 3: Fidelity check + documentation + +**Files:** +- Test: `test/test_vendored_scalpel.py` (append the fidelity check) +- Modify: `CLAUDE.md`, `.claude/SCHEMA_DECISIONS.md`, `CHANGELOG.md` + +**Interfaces:** +- Consumes: the vendored slice (Task 1) and the rewired oracle (Task 2). Produces no new code interface. + +- [ ] **Step 1: Add the vendored-vs-upstream fidelity test (skipif no pip scalpel)** + +Append to `test/test_vendored_scalpel.py`: + +```python +import pytest + + +def _pip_scalpel_available(): + try: + import importlib.util + # the EXTERNAL package, not our vendored copy + return importlib.util.find_spec("scalpel") is not None and \ + not importlib.util.find_spec("scalpel").origin.endswith( + "codeanalyzer/dataflow/scalpel/__init__.py") + except Exception: + return False + + +@pytest.mark.skipif(not _pip_scalpel_available(), + reason="upstream python-scalpel not installed (3.12+ can't build typed_ast)") +def test_vendored_ssa_matches_upstream(): + """On a Python where pip python-scalpel installs (<=3.11), the vendored copy + must produce identical SSA const_dict keys as upstream on the same source.""" + import scalpel.SSA.const as up_ssa + import scalpel.cfg as up_cfg + from codeanalyzer.dataflow.scalpel.SSA.const import SSA as VSSA + from codeanalyzer.dataflow.scalpel.cfg import CFGBuilder as VCFG + + src = "def f(a):\n b = a\n if a:\n b = 2\n return b\n" + up_c = up_cfg.CFGBuilder().build_from_src("m", src) + v_c = VCFG().build_from_src("m", src) + up_fn = list(up_c.functioncfgs.values())[0] + v_fn = list(v_c.functioncfgs.values())[0] + _, up_const = up_ssa.SSA().compute_SSA(up_fn) + _, v_const = VSSA().compute_SSA(v_fn) + assert sorted(map(str, up_const)) == sorted(map(str, v_const)) +``` + +- [ ] **Step 2: Run the fidelity test (skips on 3.14; that's expected)** + +Run: `poetry run python -m pytest test/test_vendored_scalpel.py -q` +Expected: PASS with the fidelity case SKIPPED on the 3.14 dev env (pip scalpel not installable). + +- [ ] **Step 3: Update `CLAUDE.md`** + +In `CLAUDE.md`, find the "**L4 points-to oracle = Scalpel.**" bullet and replace its "optional dependency … sanctioned fallback, not the shipping default" framing. Change the sentence: + +> `python-scalpel` is an **optional** dependency: if it is absent or a build/query fails, the analyzer falls back to the total `TypeBasedAliasOracle` (`alias.py`) and degrades, never raising. The type-based oracle is the sanctioned fallback, not the shipping default. + +to: + +> Scalpel is **vendored** (`codeanalyzer/dataflow/scalpel/`, a `typed_ast`-free 9-module slice of `python-scalpel 1.0b0`, Apache-2.0) so it is the **shipping default** L4 oracle on every supported Python — there is no external `python-scalpel`/`typed_ast` dependency. `TypeBasedAliasOracle` (`alias.py`) is retained only as the runtime safety net: on a per-callable Scalpel build failure or a per-query unresolved access path, `may_alias` degrades to it, never raising. + +- [ ] **Step 4: Update `.claude/SCHEMA_DECISIONS.md`** + +Append a follow-up note to the "## Stage 0 — Scalpel oracle spike" section: + +```markdown +**Follow-up (2026-07-22): Scalpel vendored, now the default.** The Stage-0 +"dependency hygiene" concern proved fatal for a hard dependency: `python-scalpel` +drags `typed_ast`, which has no wheel for Python 3.12+ and does not build from +source there, so `pip install python-scalpel` fails on 3.12/3.13/3.14. Since +`typed_ast` is imported only by `scalpel/typeinfer` (unused here), the 9-module +`SSA`/`cfg`/`core` slice the oracle loads was **vendored** into +`codeanalyzer/dataflow/scalpel/` (Apache-2.0, verbatim but for a `graphviz`-lazy +patch). `ScalpelAliasOracle` is now the shipping default on all supported Python; +`TypeBasedAliasOracle` is the runtime safety net only. See +`docs/superpowers/specs/2026-07-22-vendored-scalpel-default-oracle-design.md`. +``` + +- [ ] **Step 5: Update `CHANGELOG.md`** + +Under the `## [Unreleased]` heading, add: + +```markdown +### Changed +- The Scalpel-backed L4 points-to oracle is now **vendored** (`typed_ast`-free) + and the default on all supported Python (3.9–3.14); `python-scalpel` is no + longer an optional dependency and the `[scalpel]` extra is removed. On Python + 3.12+ (and anywhere the `[scalpel]` extra was not installed), L4 + `prov:["points-to"]` data-dependence edges are now Scalpel-precise rather than + the coarser type-based over-approximation; the `prov:["ssa"]` set and the + `L3 ⊆ L4` monotonicity invariant are unchanged. Adds `astor` as a runtime + dependency. +``` + +- [ ] **Step 6: Run the full behavior-preservation gate** + +Run: `poetry run python -m pytest test/test_vendored_scalpel.py test/test_v2_l4.py test/test_v2_l4_summary.py test/test_dataflow_sdg.py test/test_dataflow_defuse.py test/test_v2_superset.py -q` +Expected: PASS (incl. the `test_l3_subset_of_l4` monotonicity gate in `test_v2_superset.py`). Note: `test_v2_superset.py` runs the CLI with `--no-venv`; it exercises the vendored scalpel on the L4 fixture. + +- [ ] **Step 7: Commit** + +```bash +git add test/test_vendored_scalpel.py CLAUDE.md .claude/SCHEMA_DECISIONS.md CHANGELOG.md +git commit -m "docs(dataflow): record vendored scalpel as the default L4 oracle" +``` + +--- + +## Self-Review + +**1. Spec coverage:** +- Vendored `codeanalyzer/dataflow/scalpel/` 9-file slice, verbatim + graphviz patch → Task 1 Steps 3–4. ✓ +- Attribution (LICENSE, README, NOTICE) → Task 1 Steps 3, 5. ✓ +- `astor` added, `[scalpel]` extra removed, `requires-python` unchanged → Task 1 Step 6. ✓ +- Oracle repointed + default (ImportError branch dropped, fallback kept) → Task 2 Steps 3–4. ✓ +- Behavior change (L4 points-to precision), monotonicity preserved → Task 2 Step 6, Task 3 Steps 5–6. ✓ +- Docs: CLAUDE.md, SCHEMA_DECISIONS, CHANGELOG → Task 3 Steps 3–5. ✓ +- Testing: import hygiene, typed_ast-free gate, default-scalpel, determinism, fidelity, L4 regression, monotonicity → Tasks 1–3. ✓ + +**2. Placeholder scan:** No TBD/TODO. The vendoring uses exact `cp` commands and an explicit file list; the graphviz patch shows the exact before/after; every doc edit gives the exact old→new text. + +**3. Type consistency:** `make_alias_oracle(pycallable, func_ast, base_types)` signature matches the existing call site in `core`/the pipeline. Import paths (`codeanalyzer.dataflow.scalpel.SSA.const`, `...cfg`) are consistent across Task 1 (creation), Task 2 (oracle import), and the tests. `ScalpelAliasOracle`/`TypeBasedAliasOracle` names match `scalpel_oracle.py`/`alias.py`. From ab088a1b1ec4ce0896b361bff14a1ebcf4fbdccb Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 22 Jul 2026 16:52:01 -0400 Subject: [PATCH 12/22] feat(dataflow): vendor typed_ast-free scalpel SSA/cfg slice --- NOTICE | 7 + codeanalyzer/dataflow/scalpel/LICENSE | 201 +++++ codeanalyzer/dataflow/scalpel/README.md | 34 + codeanalyzer/dataflow/scalpel/SSA/__init__.py | 8 + codeanalyzer/dataflow/scalpel/SSA/const.py | 398 ++++++++++ codeanalyzer/dataflow/scalpel/__init__.py | 11 + codeanalyzer/dataflow/scalpel/cfg/__init__.py | 10 + codeanalyzer/dataflow/scalpel/cfg/builder.py | 700 ++++++++++++++++++ codeanalyzer/dataflow/scalpel/cfg/model.py | 331 +++++++++ .../dataflow/scalpel/core/__init__.py | 0 .../scalpel/core/func_call_visitor.py | 234 ++++++ .../dataflow/scalpel/core/vars_visitor.py | 205 +++++ pyproject.toml | 10 +- test/test_vendored_scalpel.py | 26 + 14 files changed, 2169 insertions(+), 6 deletions(-) create mode 100644 codeanalyzer/dataflow/scalpel/LICENSE create mode 100644 codeanalyzer/dataflow/scalpel/README.md create mode 100644 codeanalyzer/dataflow/scalpel/SSA/__init__.py create mode 100644 codeanalyzer/dataflow/scalpel/SSA/const.py create mode 100644 codeanalyzer/dataflow/scalpel/__init__.py create mode 100644 codeanalyzer/dataflow/scalpel/cfg/__init__.py create mode 100644 codeanalyzer/dataflow/scalpel/cfg/builder.py create mode 100644 codeanalyzer/dataflow/scalpel/cfg/model.py create mode 100644 codeanalyzer/dataflow/scalpel/core/__init__.py create mode 100644 codeanalyzer/dataflow/scalpel/core/func_call_visitor.py create mode 100644 codeanalyzer/dataflow/scalpel/core/vars_visitor.py create mode 100644 test/test_vendored_scalpel.py diff --git a/NOTICE b/NOTICE index bd0af50..4ec640c 100644 --- a/NOTICE +++ b/NOTICE @@ -16,3 +16,10 @@ Any other use of CodeQL, including its use on proprietary code or in closed-sour You can view the full CodeQL license at: https://github.com/github/codeql/blob/main/LICENSE + +--- Scalpel License Notice --- + +This project vendors a slice of Scalpel (python-scalpel), a product of SMAT-Lab, +under codeanalyzer/dataflow/scalpel/. Scalpel is licensed under the Apache +License 2.0. The full license text is included at +codeanalyzer/dataflow/scalpel/LICENSE. Source: https://github.com/SMAT-Lab/Scalpel diff --git a/codeanalyzer/dataflow/scalpel/LICENSE b/codeanalyzer/dataflow/scalpel/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/codeanalyzer/dataflow/scalpel/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/codeanalyzer/dataflow/scalpel/README.md b/codeanalyzer/dataflow/scalpel/README.md new file mode 100644 index 0000000..d2c630f --- /dev/null +++ b/codeanalyzer/dataflow/scalpel/README.md @@ -0,0 +1,34 @@ +# Vendored Scalpel (typed_ast-free slice) + +Vendored from **[SMAT-Lab/Scalpel](https://github.com/SMAT-Lab/Scalpel)**, +package `python-scalpel==1.0b0`, licensed **Apache-2.0** (see `LICENSE`). + +## Why vendored + +`python-scalpel` hard-depends on `typed_ast`, whose last release (1.5.5) has no +wheel for Python 3.12+ and fails to build from source on modern compilers — so +`pip install python-scalpel` fails on 3.12/3.13/3.14. `typed_ast` is imported by +exactly one scalpel module, `typeinfer/analysers.py`, which codeanalyzer does not +use. Vendoring the small slice the L4 may-alias oracle needs removes the +`typed_ast` dependency and makes scalpel the default oracle on every supported +Python. + +## What is vendored + +Exactly the 9-module closure that `scalpel.SSA.const` + `scalpel.cfg` load +(verified via `sys.modules`; provably free of `typeinfer`/`typed_ast`): + + __init__.py + SSA/__init__.py, SSA/const.py + cfg/__init__.py, cfg/builder.py, cfg/model.py + core/__init__.py, core/func_call_visitor.py, core/vars_visitor.py + +Copied verbatim except **one patch**: `cfg/model.py`'s top-level +`import graphviz as gv` is guarded (`try/except ImportError: gv = None`) so the +module imports without the `graphviz` package — the graphviz-using +`build_visual()` methods are unused here. + +Runtime deps of this slice: `astor`, `networkx` (both core dependencies of +codeanalyzer). `typed_ast` and `graphviz` are NOT required. + +To refresh: re-run the vendoring in `docs/superpowers/plans/2026-07-22-vendored-scalpel-default-oracle.md` Task 1. diff --git a/codeanalyzer/dataflow/scalpel/SSA/__init__.py b/codeanalyzer/dataflow/scalpel/SSA/__init__.py new file mode 100644 index 0000000..7cf7a86 --- /dev/null +++ b/codeanalyzer/dataflow/scalpel/SSA/__init__.py @@ -0,0 +1,8 @@ +""" +Static Single Assignment (SSA) is a technique of IR in the compiling thoery, it also shows great benefits to static anaysis tasks such as constant propagation, dead code elimination and etc. +Constant propagation is also a matured technique in static anaysis. +It is the process of evaluating or recognizing the actual constant values or expressions at a particular program point. This is realized by utilizing control flow and data flow information. Determining the possible values for variables before runtime gives great benefits to software anaysis. +For instance, with constant value propagation, we can detect and remove dead code or perfrom type checking. +In scalpel, we implement constant propagation along with the SSA for execution efficiency. +""" +__slots__ = ["const"] \ No newline at end of file diff --git a/codeanalyzer/dataflow/scalpel/SSA/const.py b/codeanalyzer/dataflow/scalpel/SSA/const.py new file mode 100644 index 0000000..2b97895 --- /dev/null +++ b/codeanalyzer/dataflow/scalpel/SSA/const.py @@ -0,0 +1,398 @@ +""" +In this module, the single static assignment forms are implemented to allow +further analysis. The module contain a single class named SSA. +""" +import ast +import astor +from functools import reduce +from collections import OrderedDict +import networkx as nx +from ..core.vars_visitor import get_vars + +def parse_val(node): + # does not return anything + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Str): + if hasattr(node, "value"): + return node.value + else: + return node.s + return "other" + + +class SSA: + """ + Build SSA graph from a given AST node based on the CFG. + """ + def __init__ (self): + """ + Args: + src: the source code as input. + """ + # the class SSA takes a module as the input + self.numbering = {} # numbering variables + self.var_values = {} # numbering variables + self.global_live_idents = [] + self.ssa_blocks = [] + self.error_paths = {} + self.dom = {} + + self.block_ident_gen = {} + self.block_ident_use = {} + self.reachable_table = {} + id2block = {} + self.unreachable_names = {} + self.undefined_names_from = {} + self.global_names = [] + + def get_attribute_stmts(self, stmts): + call_stmts = [] + for stmt in stmts: + if isinstance(stmt,ast.Call) and isinstance(stmt.func, ast.Attribute): + call_stmts += [stmt] + + def get_identifiers(self, ast_node): + """ + Extract all identifiers from the given AST node. + Args: + ast_node: AST node. + """ + if ast_node is None: + return [] + res = get_vars(ast_node) + idents = [r['name'] for r in res if r['name'] is not None and "." not in r['name']] + return idents + + def compute_SSA(self, cfg): + """ + Compute single static assignment form representations for a given CFG. + During the computing, constant value and alias pairs are generated. The following steps are used to compute SSA representations: + step 1a: compute the dominance frontier + step 1b: use dominance frontier to place phi node + if node X contains assignment to a, put phi node for an in dominance frontier of X + adding phi function may require introducing additional phi function + start from the entry node + step2: rename variables so only one definition per name + + Args: + cfg: a control flow graph. + """ + # to count how many times a var is defined + ident_name_counter = {} + # constant assignment dict + ident_const_dict = {} + # step 1a: compute the dominance frontier + all_blocks = cfg.get_all_blocks() + id2blocks = {block.id:block for block in all_blocks} + + block_loaded_idents = {block.id:[] for block in all_blocks} + block_stored_idents = {block.id:[] for block in all_blocks} + + block_const_dict = {block.id:[] for block in all_blocks} + + block_renamed_stored = {block.id:[] for block in all_blocks} + block_renamed_loaded = {block.id:[] for block in all_blocks} + + DF = self.compute_DF(all_blocks) + + for block in all_blocks: + df_nodes = DF[block.id] + tmp_const_dict = {} + + + for idx, stmt in enumerate(block.statements): + stmt_const_dict = {} + stored_idents, loaded_idents, func_names = self.get_stmt_idents_ctx(stmt, const_dict=stmt_const_dict) + tmp_const_dict[idx] = stmt_const_dict + block_loaded_idents[block.id] += [loaded_idents] + block_stored_idents[block.id] += [stored_idents] + block_renamed_loaded[block.id] += [{ident:set() for ident in loaded_idents}] + + block_const_dict[block.id] = tmp_const_dict + + for block in all_blocks: + stored_idents = block_stored_idents[block.id] + loaded_idents = block_loaded_idents[block.id] + n_stmts = len(stored_idents) + assert (n_stmts == len(loaded_idents)) + affected_idents = [] + tmp_const_dict = block_const_dict[block.id] + for i in range(n_stmts): + stmt_stored_idents = stored_idents[i] + stmt_loaded_idents = loaded_idents[i] + stmt_renamed_stored = {} + + for ident in stmt_stored_idents: + affected_idents.append(ident) + if ident in ident_name_counter: + ident_name_counter[ident] += 1 + else: + ident_name_counter[ident] = 0 + # rename the var name as the number of assignments + stmt_const_dict = tmp_const_dict[i] + if ident in stmt_const_dict: + ident_const_dict[(ident, ident_name_counter[ident])] = stmt_const_dict[ident] + + stmt_renamed_stored[ident] = ident_name_counter[ident] + block_renamed_stored[block.id] += [stmt_renamed_stored] + + + #same block, number used identifiers + for ident in stmt_loaded_idents: + # a list of dictions for each of idents used in this statement + phi_loaded_idents = block_renamed_loaded[block.id][i] + if ident in ident_name_counter: + phi_loaded_idents[ident].add(ident_name_counter[ident]) + + df_block_ids = DF[block.id] + for df_block_id in df_block_ids: + df_block = id2blocks[df_block_id] + block_ident_gen_produced = [] + df_block_stored_idents = block_stored_idents[df_block_id] + for af_ident in affected_idents: + # this for-loop process every statement in the block + for idx, phi_loaded_idents in enumerate(block_renamed_loaded[df_block_id]): + block_ident_gen_produced.extend(df_block_stored_idents[idx]) + if af_ident in block_ident_gen_produced: + continue + # place phi function here this var used + # if af_ident has been assigned in this block beforclee this statement, then discard it + # so theck af_ident has been generated in this block + if af_ident in phi_loaded_idents: + phi_loaded_idents[af_ident].add(ident_name_counter[af_ident]) + + return block_renamed_loaded, ident_const_dict + + def get_stmt_idents_ctx(self, stmt, del_set=[], const_dict = {}): + """ + Extract the contextual information of each of identifiers. + For assignment statements, the assigned values for each of variables will be stored. + In addition, the del_set will store all deleted variables. + Args: + stmt: statement from AST trees. + del_set: deleted identifiers + const_dict: a mapping relationship between variables and their assigned values in this statement + """ + # if this is a definition of class/function, ignore + stored_idents = [] + loaded_idents = [] + func_names = [] + # assignment with only one target + + if isinstance(stmt, ast.Assign): + targets = stmt.targets + value = stmt.value + if len(targets) == 1: + if hasattr(targets[0], "id"): + left_name = stmt.targets[0].id + const_dict[left_name] = stmt.value + elif isinstance(targets[0], ast.Attribute): + left_name = astor.to_source(stmt.targets[0]).strip() + const_dict[left_name] = value + # multiple targets are represented as tuple + elif isinstance(targets[0], ast.Tuple): + # value is also represented as tuple + if isinstance(value, ast.Tuple): + for elt, val in zip(targets[0].elts, value.elts): + if hasattr(elt, "id"): + left_name = elt.id + const_dict[left_name] = val + elif isinstance(targets[0], ast.Attribute): + #TODO: resolve attributes + pass + # value is represented as call + if isinstance(value, ast.Call): + for elt in targets[0].elts: + if hasattr(elt, "id"): + left_name = elt.id + const_dict[left_name] = value + elif isinstance(targets[0], ast.Attribute): + #TODO: resolve attributes + pass + else: + # Note in some python versions, there are more than one target for an assignment + # while in some other python versions, multiple targets are deemed as ast.Tuple type in assignment statement + for target in stmt.targets: + # this is an assignment to tuple such as a,b = fun() + # then no valid constant value can be recorded for this statement + if hasattr(target, "id"): + left_name = target.id + const_dict[left_name] = None # TODO: design a type for these kind of values + elif isinstance(stmt.targets[0], ast.Attribute): + #TODO: resolve attributes + pass + + + + # one target assignment with type annotations + if isinstance(stmt, ast.AnnAssign): + if hasattr(stmt.target, "id"): + left_name = stmt.target.id + const_dict[left_name] = stmt.value + elif isinstance(stmt.target, ast.Attribute): + #TODO: resolve attributes + pass + if isinstance(stmt, ast.AugAssign): + # note here , we need to rewrite this value to its extended form + # if the statement is "a += 1", then the assigned value should be a+1 + if hasattr(stmt.target, "id"): + left_name = stmt.target.id + extended_right = ast.BinOp(ast.Name(left_name, ast.Load()), stmt.op, stmt.value) + const_dict[left_name] = extended_right + elif isinstance(stmt.target, ast.Attribute): + #TODO: resolve attributes + pass + if isinstance(stmt, ast.For): + # there is a variation of assignment in for loop + # in the case of : for i in [1,2,3] + # the element of stmt.iter is the value of this assignment + if hasattr(stmt.target, "id"): + left_name = stmt.target.id + iter_value = stmt.iter + # make a iter call + #iter_node = ast.Call(ast.Name("iter", ast.Load()), [stmt.iter], []) + # make a next call + #next_call_node = ast.Call(ast.Name("next", ast.Load()), [iter_node], []) + const_dict[left_name] = iter_value + + elif isinstance(stmt.target, ast.Tuple): + # to handle for-loop uch as: + # for x, y in fun(): + for elt in stmt.target.elts: + if hasattr(elt, "id"): + const_dict[elt.id] = stmt.iter + elif isinstance(stmt.target, ast.Attribute): + #TODO: resolve attributes + pass + + + + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)): + stored_idents.append(stmt.name) + const_dict[stmt.name] = stmt + func_names.append(stmt.name) + new_stmt = stmt + new_stmt.body = [] + ident_info = get_vars(new_stmt) + for r in ident_info: + if r['name'] is None: + continue + if r['usage'] == "load": + loaded_idents.append(r['name']) + return stored_idents, loaded_idents, func_names + + if isinstance(stmt, ast.ClassDef): + stored_idents.append(stmt.name) + const_dict[stmt.name] = None + func_names.append(stmt.name) + return stored_idents, loaded_idents, func_names + + # if this is control flow statements, we should not visit its body to avoid duplicates + # as they are already in the next blocks + if isinstance(stmt, (ast.Import, ast.ImportFrom)): + for alias in stmt.names: + if alias.asname is None: + stored_idents += [alias.name.split('.')[0]] + else: + stored_idents += [alias.asname.split('.')[0]] + return stored_idents, loaded_idents, [] + + if isinstance(stmt, (ast.Try)): + for handler in stmt.handlers: + if handler.name is not None: + stored_idents.append(handler.name) + + if isinstance(handler.type, ast.Name): + loaded_idents.append(handler.type.id) + elif isinstance(handler.type, ast.Attribute) and isinstance(handler.type.value, ast.Name): + loaded_idents.append(handler.type.value.id) + return stored_idents, loaded_idents, [] + if isinstance(stmt, ast.Global): + for name in stmt.names: + self.global_names.append(name) + return stored_idents, loaded_idents, [] + + visit_node = stmt + + if isinstance(visit_node,(ast.If, ast.IfExp)): + # visit_node.body = [] + # visit_node.orlse=[] + visit_node = stmt.test + + elif isinstance(visit_node, (ast.With)): + visit_node.body = [] + visit_node.orlse=[] + + elif isinstance(visit_node, (ast.While)): + visit_node.body = [] + + elif isinstance(visit_node, (ast.For)): + visit_node.body = [] + + elif isinstance(visit_node, ast.Return): + # imaginary variable + stored_idents.append("") + const_dict[""] = visit_node.value + elif isinstance(visit_node, ast.Yield): + # imaginary variable + stored_idents.append("") + const_dict[""] = visit_node.value + + ident_info = get_vars(visit_node) + for r in ident_info: + if r['name'] is None or "_hidden_" in r['name']: + continue + if r['usage'] == 'store': + stored_idents.append(r['name']) + else: + loaded_idents.append(r['name']) + if r['usage'] == 'del': + del_set.append(r['name']) + return stored_idents, loaded_idents, [] + + def to_json(self): + pass + + def print_block(self, block): + return block.get_source() + + # compute the dominators + def compute_idom(self, ssa_blocks): + """ + Compute immediate dominators for each of blocks + Args: + ssa_blocks: blocks from a control flow graph. + """ + # construct the Graph + entry_block = ssa_blocks[0] + G = nx.DiGraph() + for block in ssa_blocks: + G.add_node(block.id) + exits = block.exits + preds = block.predecessors + for link in preds+exits: + G.add_edge(link.source.id, link.target.id) + # DF = nx.dominance_frontiers(G, entry_block.id) + idom = nx.immediate_dominators(G, entry_block.id) + return idom + + # compute dominance frontiers + def compute_DF(self, ssa_blocks): + """ + Compute dominating frontiers for each of blocks + Args: + ssa_blocks: blocks from a control flow graph. + """ + # construct the Graph + entry_block = ssa_blocks[0] + G = nx.DiGraph() + for block in ssa_blocks: + G.add_node(block.id) + exits = block.exits + preds = block.predecessors + for link in preds+exits: + G.add_edge(link.source.id, link.target.id) + DF = nx.dominance_frontiers(G, entry_block.id) + #idom = nx.immediate_dominators(G, entry_block.id) + return DF diff --git a/codeanalyzer/dataflow/scalpel/__init__.py b/codeanalyzer/dataflow/scalpel/__init__.py new file mode 100644 index 0000000..961ac40 --- /dev/null +++ b/codeanalyzer/dataflow/scalpel/__init__.py @@ -0,0 +1,11 @@ +""" +Static Anaysis for Python Programs +================================== +Scalpel is a Python library integrating classical program anaysis algorithms +with tailored features for Python language. It aims to provide simple and +efficient solutions to software engineering researchers that are accessible to +everybody and reusable in various contexts. +""" + +__all__ = ["cfg", "call_graph", "SSA", "core", "typeinfer", "import_graph", "rewriter"] +__version__ = '1.0dev' diff --git a/codeanalyzer/dataflow/scalpel/cfg/__init__.py b/codeanalyzer/dataflow/scalpel/cfg/__init__.py new file mode 100644 index 0000000..ddb2fd5 --- /dev/null +++ b/codeanalyzer/dataflow/scalpel/cfg/__init__.py @@ -0,0 +1,10 @@ +""" +The control-flow graph(CFG) is an essential component in static flow analysis with applications such as program +optimization and taint analysis. +scalpel.cfg module is used to construct the control flow graph for given python programs. The basic unit in the CFG, +Block, contains a list of sequential statements that can be executed in a program without any control jumps. The Blocks +are linked by Link objects, which represent control flow jumps between two blocks and contain the jump conditions in +the form of an expression. Please see the example diagram a control flow graph ![Fibonacci CFG](https://raw.githubusercontent.com/SMAT-Lab/Scalpel/main/docs/_static/resources/cfg_example.png) +""" +from .builder import CFGBuilder +from .model import Block, Link, CFG diff --git a/codeanalyzer/dataflow/scalpel/cfg/builder.py b/codeanalyzer/dataflow/scalpel/cfg/builder.py new file mode 100644 index 0000000..05cb2e5 --- /dev/null +++ b/codeanalyzer/dataflow/scalpel/cfg/builder.py @@ -0,0 +1,700 @@ +""" +This implementation is partly adapted from the static cfg project +https://github.com/coetaur0/staticfg +""" + +import ast +from .model import Block, Link, CFG +import sys +from ..core.func_call_visitor import get_func_calls + + +def is_py38_or_higher(): + if sys.version_info.major == 3 and sys.version_info.minor >= 8: + return True + return False + + +NAMECONSTANT_TYPE = ast.Constant if is_py38_or_higher() else ast.NameConstant + + +def invert(node): + """ + Invert the operation in an ast node object (get its negation). + Args: + node: An ast node object. + Returns: + An ast node object containing the inverse (negation) of the input node. + """ + inverse = {ast.Eq: ast.NotEq, + ast.NotEq: ast.Eq, + ast.Lt: ast.GtE, + ast.LtE: ast.Gt, + ast.Gt: ast.LtE, + ast.GtE: ast.Lt, + ast.Is: ast.IsNot, + ast.IsNot: ast.Is, + ast.In: ast.NotIn, + ast.NotIn: ast.In} + + if type(node) == ast.Compare: + op = type(node.ops[0]) + inverse_node = ast.Compare(left=node.left, ops=[inverse[op]()], + comparators=node.comparators) + elif isinstance(node, ast.BinOp) and type(node.op) in inverse: + op = type(node.op) + inverse_node = ast.BinOp(node.left, inverse[op](), node.right) + elif type(node) == NAMECONSTANT_TYPE and node.value in [True, False]: + inverse_node = NAMECONSTANT_TYPE(value=not node.value) + else: + inverse_node = ast.UnaryOp(op=ast.Not(), operand=node) + return inverse_node + + +def merge_exitcases(exit1, exit2): + """ + Merge the exitcases of two Links. + + Args: + exit1: The exitcase of a Link object. + exit2: Another exitcase to merge with exit1. + + Returns: + The merged exitcases. + """ + if exit1: + if exit2: + return ast.BoolOp(ast.And(), values=[exit1, exit2]) + return exit1 + return exit2 + + +class CFGBuilder(ast.NodeVisitor): + """ + Control flow graph builder. + + A control flow graph builder is an ast.NodeVisitor that can walk through + a program's AST and iteratively build the corresponding CFG. + """ + + __all__ = ["build", "build_from_src", "build_from_file"] + + def __init__(self, separate=False): + super().__init__() + self.after_loop_block_stack = [] + self.curr_loop_guard_stack = [] + self.current_block = None + self.separate_node_blocks = separate + self.enter_func_def = False + + # ---------- CFG building methods ---------- # + def build(self, name, tree, asynchr=False, entry_id=0, flattened=False): + """ + Build a CFG from an AST. + + Args: + name: The name of the CFG being built. + tree: The root of the AST from which the CFG must be built. + async: Boolean indicating whether the CFG being built represents an + asynchronous function or not. When the CFG of a Python + program is being built, it is considered like a synchronous + 'main' function. + entry_id: Value for the id of the entry block of the CFG. + flattened: if use k-v format for all CFGs while hiding its nested information. Key will be fully-qualified names. + + Returns: + The CFG produced from the AST. + """ + self.cfg = CFG(name, asynchr=asynchr) + # Tracking of the current block while building the CFG. + self.current_id = entry_id + self.current_block = self.new_block() + self.cfg.entryblock = self.current_block + # Actual building of the CFG is done here. + self.visit(tree) + visited = [] + self.clean_cfg(self.cfg.entryblock,visited) + + if flattened: + self.cfg = self._flatten_cfg(self.cfg) + pass + return self.cfg + + def _flatten_cfg(self, mod_cfg): + flattend_cfg = {} + + def process_cfg(cfg, dotted_name=["mod"], name_type="mod"): + fully_qualified_name = ".".join(dotted_name) + flattend_cfg[fully_qualified_name] = cfg + for fun_name_tup, fun_cfg in cfg.functioncfgs.items(): + process_cfg(fun_cfg, dotted_name = dotted_name +[fun_name_tup[1]], name_type= "func") + + for cls_name, cls_cfg in cfg.class_cfgs.items(): + process_cfg(cls_cfg, dotted_name = dotted_name +[cls_name], name_type = "cls") + + process_cfg(mod_cfg) + + return flattend_cfg + def build_from_src(self, name, src, flattened=False): + """ + Build a CFG from some Python source code. + + Args: + name: The name of the CFG being built. + src: A string containing the source code to build the CFG from. + flattened: if use k-v format for all CFGs while hiding its nested information. Key will be fully-qualified names. + + + Returns: + The CFG produced from the source code. + """ + tree = ast.parse(src, mode='exec') + return self.build(name, tree, flattened=flattened) + + def build_from_file(self, name, filepath, flattened=False): + """ + Build a CFG from some Python source file. + + Args: + name: The name of the CFG being built. + filepath: The path to the file containing the Python source code to build the CFG from. + flattened: if use k-v format for all CFGs while hiding its nested information. Key will be fully-qualified names. + + + Returns: + The CFG produced from the source file. + """ + with open(filepath, 'r',encoding="utf8") as src_file: + src = src_file.read() + return self.build_from_src(name, src, flattened=flattened) + + # ---------- Graph management methods ---------- # + def new_block(self): + """ + Create a new block with a new id. + Returns: + A Block object with a new unique id. + """ + self.current_id += 1 + return Block(self.current_id) + + def add_statement(self, block, statement): + """ + Add a statement to a block. + Args: + block: A Block object to which a statement must be added. + statement: An AST node representing the statement that must be + added to the current block. + """ + # remove function def nodes + block.statements.append(statement) + + def add_exit(self, block, nextblock, exitcase=None): + """ + Add a new exit to a block. + Args: + block: A block to which an exit must be added. + nextblock: The block to which control jumps from the new exit. + exitcase: An AST node representing the 'case' (or condition) + leading to the exit from the block in the program. + """ + newlink = Link(block, nextblock, exitcase) + block.exits.append(newlink) + nextblock.predecessors.append(newlink) + + def new_loopguard(self): + """ + Create a new block for a loop's guard if the current block is not + empty. Links the current block to the new loop guard. + + Returns: + The block to be used as new loop guard. + """ + if (self.current_block.is_empty() and + len(self.current_block.exits) == 0): + # If the current block is empty and has no exits, it is used as + # entry block (condition test) for the loop. + loopguard = self.current_block + else: + # Jump to a new block for the loop's guard if the current block + # isn't empty or has exits. + loopguard = self.new_block() + self.add_exit(self.current_block, loopguard) + return loopguard + + def new_functionCFG(self, node, asynchr=False, enclosing_block_id=-1): + """ + Create a new sub-CFG for a function definition and add it to the + function CFGs of the CFG being built. + + Args: + node: The AST node containing the function definition. + async: Boolean indicating whether the function for which the CFG is + being built is asynchronous or not. + """ + self.current_id += 1 + # A new sub-CFG is created for the body of the function definition and + # added to the function CFGs of the current CFG. + func_body = ast.Module(body=node.body) + func_builder = CFGBuilder() + self.cfg.functioncfgs[(enclosing_block_id,node.name)] = func_builder.build(node.name, + func_body, + asynchr, + self.current_id) + + def get_arg_names(argument_node): + arg_names = [] + for node in ast.walk(argument_node): + if isinstance(node, ast.arg): + arg_names.append( node.arg) + return arg_names + + self.cfg.function_args[(enclosing_block_id, node.name)] = get_arg_names(node.args) + self.current_id = func_builder.current_id + 1 + + def new_ClassCFG(self, node, asynchr=False): + """ + Create a new sub-CFG for a class definition and add it to the + function CFGs of the CFG being built. + + Args: + node: The AST node containing the function definition. + asynchr: Boolean indicating whether the function for which the CFG is + being built is asynchronous or not. + """ + self.current_id += 1 + # A new sub-CFG is created for the body of the function definition and + # added to the function CFGs of the current CFG. + func_body = ast.Module(body=node.body) + func_builder = CFGBuilder() + base_names = [] + for base in node.bases: + if not isinstance(base, ast.Name): + continue + base_names.append(base.id) + if node.name in self.cfg.class_cfgs and node.name in base_names: + existing_class_cfg = self.cfg.class_cfgs[node.name] + new_class_cfg = func_builder.build(node.name, + func_body, + asynchr, + self.current_id) + new_class_cfg.entryblock.statements = new_class_cfg.entryblock.statements+existing_class_cfg.entryblock.statements + new_class_cfg.functioncfgs.update(existing_class_cfg.functioncfgs) + new_class_cfg.function_args.update(existing_class_cfg.function_args) + self.cfg.class_cfgs[node.name]=new_class_cfg + else: + self.cfg.class_cfgs[node.name] = func_builder.build(node.name, + func_body, + asynchr, + self.current_id) + + self.current_id = func_builder.current_id + 1 + + def clean_cfg(self, block, visited): + """ + Remove the useless (empty) blocks from a CFG. + + Args: + block: The block from which to start traversing the CFG to clean + it. + visited: A list of blocks that already have been visited by + clean_cfg (recursive function). + """ + # Don't visit blocks twice. + if block.id in visited: + return + visited.append(block.id) + + # Empty blocks are removed from the CFG. + if block.is_empty(): + for pred in block.predecessors: + for exit in block.exits: + self.add_exit(pred.source, exit.target, + merge_exitcases(pred.exitcase, + exit.exitcase)) + # Check if the exit hasn't yet been removed from + # the predecessors of the target block. + if exit in exit.target.predecessors: + exit.target.predecessors.remove(exit) + # Check if the predecessor hasn't yet been removed from + # the exits of the source block. + if pred in pred.source.exits: + pred.source.exits.remove(pred) + + block.predecessors = [] + # as the exits may be modified during the recursive call, it is unsafe to iterate on block.exits + # Created a copy of block.exits before calling clean cfg , and iterate over it instead. + for exit in block.exits[:]: + self.clean_cfg(exit.target, visited) + block.exits = [] + else: + for exit in block.exits[:]: + self.clean_cfg(exit.target, visited) + + def goto_new_block(self, node): + if self.separate_node_blocks: + newblock = self.new_block() + self.add_exit(self.current_block, newblock) + self.current_block = newblock + self.generic_visit(node) + + # start visting all statements in AST tree + def visit_Expr(self, node): + self.add_statement(self.current_block, node) + self.goto_new_block(node) + + def visit_Call(self, node): + def visit_func(node): + if type(node) == ast.Name: + return node.id + elif type(node) == ast.Attribute: + # Recursion on series of calls to attributes. + func_name = visit_func(node.value) + func_name += "." + node.attr + return func_name + elif type(node) == ast.Str: + return node.s + elif type(node) == ast.Subscript: + return node.value.id + + #func = node.func + #func_name = visit_func(func) + func_name = get_func_calls(node)[0] + self.current_block.func_calls.append(func_name) + + def visit_Assign(self, node): + self.add_statement(self.current_block, node) + self.goto_new_block(node) + + def visit_AnnAssign(self, node): + self.add_statement(self.current_block, node) + self.goto_new_block(node) + + def visit_AugAssign(self, node): + self.add_statement(self.current_block, node) + self.goto_new_block(node) + + def visit_Global(self, node): + self.add_statement(self.current_block, node) + self.goto_new_block(node) + + def visit_Nonlocal(self, node): + self.add_statement(self.current_block, node) + self.goto_new_block(node) + + def visit_Pass(self, node): + self.add_statement(self.current_block, node) + self.goto_new_block(node) + + def visit_Delete(self, node): + self.add_statement(self.current_block, node) + self.goto_new_block(node) + + def visit_Raise(self, node): + self.add_statement(self.current_block, node) + self.cfg.finalblocks.append(self.current_block) + self.current_block = self.new_block() + + def visit_Assert(self, node): + self.add_statement(self.current_block, node) + # New block for the case in which the assertion 'fails'. + failblock = self.new_block() + self.add_exit(self.current_block, failblock, invert(node.test)) + # If the assertion fails, the current flow ends, so the fail block is a + # final block of the CFG. + self.cfg.finalblocks.append(failblock) + # If the assertion is True, continue the flow of the program. + successblock = self.new_block() + self.add_exit(self.current_block, successblock, node.test) + self.current_block = successblock + self.goto_new_block(node) + + def visit_Try(self, node): + # Add the try statement at the end of the current block. + self.add_statement(self.current_block, node) + + # Create a new block for the body of try. + try_block = self.new_block() + self.add_exit(self.current_block, try_block, ast.Constant(True)) + n_else_stmts = len(node.orelse) + #else_block = self.new_block() + #self.add_exit(self.current_block, try_block, ast.Constant(True)) + + # Create blocks for handlers + n_handlers = len(node.handlers) + handler_blocks = [] + for i in range(n_handlers): + h_block = self.new_block() + handler_blocks += [h_block] + after_try_block = self.new_block() + #self.add_exit(self.current_block, after_try_block, ast.Constant(False)) + # keep the original block + current_block = self.current_block + # + self.current_block = try_block + + for child in node.body: + self.visit(child) + + if n_else_stmts>0: + else_block = self.new_block() + self.add_exit(self.current_block, else_block) + self.current_block = else_block + # create else block + for child in node.orelse: + self.visit(child) + self.add_exit(self.current_block, after_try_block) + + + for i in range(n_handlers): + self.current_block = current_block + handler = node.handlers[i] + self.add_exit(self.current_block, handler_blocks[i], handler.type) + self.current_block = handler_blocks[i] + self.visit(handler) + # If encountered a break, exit will have already been added + if not self.current_block.exits: + self.add_exit(self.current_block, after_try_block) + #self.add_exit(self.current_block, after_try_block) + + #if not self.current_block.exits: + # self.add_exit(self.current_block, after_try_block) + # Continue building the CFG in the after-if block. + + self.current_block = after_try_block + + #populate the block in the try + #self.current_block = try_block + #for child in node.body: + # self.visit(child) + #if not self.current_block.exits: + # self.add_exit(self.current_block, after_try_block) + #self.current_block = after_try_block + + def visit_If(self, node): + # Add the If statement at the end of the current block. + self.add_statement(self.current_block, node) + + # Create a new block for the body of the if. + if_block = self.new_block() + self.add_exit(self.current_block, if_block, node.test) + + # Create a block for the code after the if-else. + afterif_block = self.new_block() + + # New block for the body of the else if there is an else clause. + if len(node.orelse) != 0: + else_block = self.new_block() + self.add_exit(self.current_block, else_block, invert(node.test)) + self.current_block = else_block + # Visit the children in the body of the else to populate the block. + for child in node.orelse: + self.visit(child) + # If encountered a break, exit will have already been added + if not self.current_block.exits: + self.add_exit(self.current_block, afterif_block) + else: + self.add_exit(self.current_block, afterif_block, invert(node.test)) + + # Visit children to populate the if block. + self.current_block = if_block + for child in node.body: + self.visit(child) + if not self.current_block.exits: + self.add_exit(self.current_block, afterif_block) + + # Continue building the CFG in the after-if block. + self.current_block = afterif_block + + + def visit_While(self, node): + loop_guard = self.new_loopguard() + self.current_block = loop_guard + self.add_statement(self.current_block, node) + self.curr_loop_guard_stack.append(loop_guard) + # New block for the case where the test in the while is True. + while_block = self.new_block() + self.add_exit(self.current_block, while_block, node.test) + + # New block for the case where the test in the while is False. + afterwhile_block = self.new_block() + self.after_loop_block_stack.append(afterwhile_block) + inverted_test = invert(node.test) + # Skip shortcut loop edge if while True: + if not (isinstance(inverted_test, NAMECONSTANT_TYPE) and + inverted_test.value is False): + self.add_exit(self.current_block, afterwhile_block, inverted_test) + # Populate the while block. + self.current_block = while_block + for child in node.body: + self.visit(child) + if not self.current_block.exits: + # Did not encounter a break statement, loop back + self.add_exit(self.current_block, loop_guard) + + # Continue building the CFG in the after-while block. + self.current_block = afterwhile_block + self.after_loop_block_stack.pop() + self.curr_loop_guard_stack.pop() + + def visit_For(self, node): + loop_guard = self.new_loopguard() + self.current_block = loop_guard + self.add_statement(self.current_block, node) + self.curr_loop_guard_stack.append(loop_guard) + # New block for the body of the for-loop. + for_block = self.new_block() + self.add_exit(self.current_block, for_block, node.iter) + + # Block of code after the for loop. + afterfor_block = self.new_block() + self.add_exit(self.current_block, afterfor_block) + self.after_loop_block_stack.append(afterfor_block) + self.current_block = for_block + + # Populate the body of the for loop. + for child in node.body: + self.visit(child) + if not self.current_block.exits: + # Did not encounter a break + self.add_exit(self.current_block, loop_guard) + + # Continue building the CFG in the after-for block. + self.current_block = afterfor_block + # Popping the current after loop stack,taking care of errors in case of nested for loops + self.after_loop_block_stack.pop() + self.curr_loop_guard_stack.pop() + + # Async for loops and async with context managers. + # They have the same fields as For and With, respectively. + # Only valid in the body of an AsyncFunctionDef. + # https://docs.python.org/3/library/ast.html + def visit_AsyncFor(self, node): + loop_guard = self.new_loopguard() + self.current_block = loop_guard + self.add_statement(self.current_block, node) + self.curr_loop_guard_stack.append(loop_guard) + # New block for the body of the for-loop. + for_block = self.new_block() + self.add_exit(self.current_block, for_block, node.iter) + + # Block of code after the for loop. + afterfor_block = self.new_block() + self.add_exit(self.current_block, afterfor_block) + self.after_loop_block_stack.append(afterfor_block) + self.current_block = for_block + + # Populate the body of the for loop. + for child in node.body: + self.visit(child) + if not self.current_block.exits: + # Did not encounter a break + self.add_exit(self.current_block, loop_guard) + + # Continue building the CFG in the after-for block. + self.current_block = afterfor_block + # Popping the current after loop stack,taking care of errors in case of nested for loops + self.after_loop_block_stack.pop() + self.curr_loop_guard_stack.pop() + def visit_Break(self, node): + assert len(self.after_loop_block_stack), "Found break not inside loop" + self.add_exit(self.current_block, self.after_loop_block_stack[-1]) + + def visit_Continue(self, node): + assert len(self.curr_loop_guard_stack), "Found continue outside loop" + self.add_exit(self.current_block, self.curr_loop_guard_stack[-1]) + + def visit_Import(self, node): + self.add_statement(self.current_block, node) + + def visit_ImportFrom(self, node): + self.add_statement(self.current_block, node) + + def visit_FunctionDef(self, node): + self.add_statement(self.current_block, node) + self.new_functionCFG(node, asynchr=False, enclosing_block_id=self.current_block.id) + + def visit_AsyncFunctionDef(self, node): + self.add_statement(self.current_block, node) + self.new_functionCFG(node, asynchr=True,enclosing_block_id=self.current_block.id) + + def visit_ClassDef(self, node): + self.add_statement(self.current_block, node) + self.new_ClassCFG(node, asynchr=True) + return node + + def visit_Await(self, node): + afterawait_block = self.new_block() + self.add_exit(self.current_block, afterawait_block) + self.goto_new_block(node) + self.current_block = afterawait_block + + def visit_Return(self, node): + self.add_statement(self.current_block, node) + self.cfg.finalblocks.append(self.current_block) + # Continue in a new block but without any jump to it -> all code after + # the return statement will not be included in the CFG. + self.current_block = self.new_block() + + def visit_Yield(self, node): + self.cfg.asynchr = True + afteryield_block = self.new_block() + self.add_exit(self.current_block, afteryield_block) + self.current_block = afteryield_block + + def visit_With(self, node): + # add with statement to the current block + self.add_statement(self.current_block, node) + # New block for the body of the with. + with_block = self.new_block() + # link current block to with block + self.add_exit(self.current_block, with_block) + + # Block of code after the with. + afterwith_block = self.new_block() + # no branch here + # link with block and body of with + #print(with_block, afterwith_block) + #self.add_exit(with_block, afterwith_block) + # go to with block and create more + self.current_block = with_block + + # Populate the body of the with loop. + for child in node.body: + self.visit(child) + + if not self.current_block.exits: + self.add_exit(self.current_block, afterwith_block) + # Continue building the CFG in the after-with block. + self.current_block = afterwith_block + + # Async for loops and async with context managers. + # They have the same fields as For and With, respectively. + # Only valid in the body of an AsyncFunctionDef. + # https://docs.python.org/3/library/ast.html + def visit_AsyncWith(self, node): + # add with statement to the current block + self.add_statement(self.current_block, node) + # New block for the body of the with. + with_block = self.new_block() + # link current block to with block + self.add_exit(self.current_block, with_block) + + # Block of code after the with. + afterwith_block = self.new_block() + # no branch here + # link with block and body of with + #print(with_block, afterwith_block) + #self.add_exit(with_block, afterwith_block) + # go to with block and create more + self.current_block = with_block + + # Populate the body of the with loop. + for child in node.body: + self.visit(child) + + if not self.current_block.exits: + self.add_exit(self.current_block, afterwith_block) + # Continue building the CFG in the after-with block. + self.current_block = afterwith_block + diff --git a/codeanalyzer/dataflow/scalpel/cfg/model.py b/codeanalyzer/dataflow/scalpel/cfg/model.py new file mode 100644 index 0000000..6191706 --- /dev/null +++ b/codeanalyzer/dataflow/scalpel/cfg/model.py @@ -0,0 +1,331 @@ +""" +Control flow graph for Python programs. +""" + +import ast +import sys +import re +import token +import tokenize +import astor + +try: # PATCH (codeanalyzer): graphviz is only used by build_visual(), which + import graphviz as gv # codeanalyzer never calls. Keep the module importable +except ImportError: # without the graphviz dependency. + gv = None + +__all__ = ["Block", "Link", "CFG"] + +class Block(object): + """ + Basic block in a control flow graph. + + Contains a list of statements executed in a program without any control + jumps. A block of statements is exited through one of its exits. Exits are + a list of Links that represent control flow jumps. + """ + + __slots__ = ["id", "statements", "func_calls", "predecessors", "exits"] + + def __init__(self, id): + # Id of the block. + self.id = id + # Statements in the block. + self.statements = [] + # Calls to functions inside the block (represents context switches to + # some functions' CFGs). + self.func_calls = [] + # Links to predecessors in a control flow graph. + self.predecessors = [] + # Links to the next blocks in a control flow graph. + self.exits = [] + + def __del__(self): + self.statements.clear() + self.func_calls.clear() + self.predecessors.clear() + self.exits.clear() + + def __str__(self): + if self.statements: + return "block:{}@{}".format(self.id, self.at()) + return "empty block:{}".format(self.id) + + def __repr__(self): + txt = "{} with {} exits".format(str(self), len(self.exits)) + if self.statements: + txt += ", body=[" + txt += ", ".join([ast.dump(node) for node in self.statements]) + txt += "]" + return txt + + def at(self): + """ + Get the line number of the first statement of the block in the program. + """ + if self.statements and self.statements[0].lineno >= 0: + return self.statements[0].lineno + return None + + def is_empty(self): + """ + Check if the block is empty. + Returns: + A boolean indicating if the block is empty (True) or not (False). + """ + return len(self.statements) == 0 + ''' + def strip_comment(self, src): + clean_src = "" + + prev_toktype = token.INDENT + first_line = None + last_lineno = -1 + last_col = 0 + + tokgen = tokenize.generate_tokens(src) + for toktype, ttext, (slineno, scol), (elineno, ecol), ltext in tokgen: + if 0: # Change to if 1 to see the tokens fly by. + print("%10s %-14s %-20r %r" % ( + tokenize.tok_name.get(toktype, toktype), + "%d.%d-%d.%d" % (slineno, scol, elineno, ecol), + ttext, ltext + )) + if slineno > last_lineno: + last_col = 0 + if scol > last_col: + mod.write(" " * (scol - last_col)) + if toktype == token.STRING and prev_toktype == token.INDENT: + # Docstring + mod.write("#--") + elif toktype == tokenize.COMMENT: + # Comment + mod.write("##\n") + else: + mod.write(ttext) + prev_toktype = toktype + last_col = ecol + last_lineno = elineno + ''' + def get_source(self): + """ + Get a string containing the Python source code corresponding to the + statements in the block. + Returns: + A string containing the source code of the statements. + """ + src = "#" + str(self.id)+'\n' + for statement in self.statements: + if type(statement) in [ast.If, ast.For, ast.While, ast.With]: + src += (astor.to_source(statement)).split('\n')[0] + "\n" + elif type(statement) == ast.Try: + src += (astor.to_source(statement)).split('\n')[0] + "\n" + #elif type(statement) == ast.If: + # src += (astor.to_source(statement)).split('\n')[0] + "\n" + elif type(statement) in [ast.FunctionDef,ast.AsyncFunctionDef, + ast.ClassDef]: + src += (astor.to_source(statement)).split('\n')[0] + "...\n" + elif type(statement) == ast.ClassDef: + src += (astor.to_source(statement)).split('\n')[0] + "...\n" + else: + src += astor.to_source(statement) + return src + + def get_calls(self): + """ + Get a string containing the calls to other functions inside the block. + + Returns: + A string containing the names of the functions called inside the + block. + """ + txt = "" + for func_call_entry in self.func_calls: + txt += func_call_entry['name'] + '\n' + return txt + + +class Link(object): + """ + Link between blocks in a control flow graph. + + Represents a control flow jump between two blocks. Contains an exitcase in + the form of an expression, representing the case in which the associated + control jump is made. + """ + + __slots__ = ["source", "target", "exitcase"] + + def __init__(self, source, target, exitcase=None): + assert type(source) == Block, "Source of a link must be a block" + assert type(target) == Block, "Target of a link must be a block" + # Block from which the control flow jump was made. + self.source = source + # Target block of the control flow jump. + self.target = target + # 'Case' leading to a control flow jump through this link. + self.exitcase = exitcase + + def __str__(self): + return "link from {} to {}".format(str(self.source), str(self.target)) + + def __repr__(self): + if self.exitcase is not None: + return "{}, with exitcase {}".format(str(self), + ast.dump(self.exitcase)) + return str(self) + + def get_exitcase(self): + """ + Get a string containing the Python source code corresponding to the + exitcase of the Link. + + Returns: + A string containing the source code. + """ + if self.exitcase: + return astor.to_source(self.exitcase) + return "" + def __del__(self): + self.source = None + # Target block of the control flow jump. + self.target = None + # 'Case' leading to a control flow jump through this link. + self.exitcase = None + + + +class CFG(object): + """ + Control flow graph (CFG). + + A control flow graph is composed of basic blocks and links between them + representing control flow jumps. It has a unique entry block and several + possible 'final' blocks (blocks with no exits representing the end of the + CFG). + """ + def __init__(self, name, asynchr=False): + """ + The constructor of CFG class. Only name of this graph is required. + """ + assert type(name) == str, "Name of a CFG must be a string" + assert type(asynchr) == bool, "Async must be a boolean value" + # Name of the function or module being represented. + self.name = name + # Type of function represented by the CFG (sync or async). A Python + # program is considered as a synchronous function (main). + self.asynchr = asynchr + # Entry block of the CFG. + self.entryblock = None + # Final blocks of the CFG. + self.finalblocks = [] + # Sub-CFGs for functions defined inside the current CFG. + self.functioncfgs = {} + self.class_cfgs = {} + + self.function_args = {} + self.class_args = {} + + def __del__(self): + pass + + def __str__(self): + return "CFG for {}".format(self.name) + + def remove_comments(self, src): + pass + + + + def get_all_blocks(self): + """ + Get a list of code blocks in this CFG; This is generated by BFS order. + + Returns: + A list of code blocks. + """ + import queue + all_blocks = [] + + visited = set() + working_queue = queue.Queue() + working_queue.put(self.entryblock) + + while not working_queue.empty(): + block = working_queue.get() + # this block has been visited + if block.id in visited: + continue + all_blocks.append(block) + visited.add(block.id) + for suc_link in block.exits: + if suc_link.target.id not in visited: + working_queue.put(suc_link.target) + return all_blocks + #def dfs(start_block): + # # non-recurisve implementation of DFS search + + def __iter__(self): + """ + Generator that yields all the blocks in the current graph, then + recursively yields from any sub graphs + """ + visited = set() + to_visit = [self.entryblock] + + while to_visit: + block = to_visit.pop(0) + visited.add(block) + for exit_ in block.exits: + if exit_.target in visited or exit_.target in to_visit: + continue + to_visit.append(exit_.target) + yield block + + for subcfg in self.functioncfgs.values(): + yield from subcfg + + def _visit_blocks(self, graph, block, visited=[], calls=True): + # Don't visit blocks twice. + if block.id in visited: + return + + nodelabel = block.get_source() + graph.node(str(block.id), label=nodelabel) + + visited.append(block.id) + # Show the block's function calls in a node. + if calls and block.func_calls: + calls_node = str(block.id)+"_calls" + calls_label = block.get_calls().strip() + graph.node(calls_node, label=calls_label, + _attributes={'shape': 'box'}) + graph.edge(str(block.id), calls_node, label="calls", + _attributes={'style': 'dashed'}) + # Recursively visit all the blocks of the CFG. + for exit in block.exits: + self._visit_blocks(graph, exit.target, visited, calls=calls) + edgelabel = exit.get_exitcase().strip() + graph.edge(str(block.id), str(exit.target.id), label=edgelabel) + + def _build_visual(self, format='pdf', calls=True): + graph = gv.Digraph(name='cluster'+self.name, format=format, + graph_attr={'label': self.name}) + self._visit_blocks(graph, self.entryblock, visited=[], calls=False) + return graph + + def build_visual(self, format, calls=True, show=True): + """ + Build a visualisation of the CFG with graphviz and output it in a DOT + file. + + Args: + filename: The name of the output file in which the visualisation + must be saved. + format: The format to use for the output file (PDF, ...). + show: A boolean indicating whether to automatically open the output + file after building the visualisation. + """ + graph = self._build_visual(format, calls) + return graph + diff --git a/codeanalyzer/dataflow/scalpel/core/__init__.py b/codeanalyzer/dataflow/scalpel/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/codeanalyzer/dataflow/scalpel/core/func_call_visitor.py b/codeanalyzer/dataflow/scalpel/core/func_call_visitor.py new file mode 100644 index 0000000..336fc13 --- /dev/null +++ b/codeanalyzer/dataflow/scalpel/core/func_call_visitor.py @@ -0,0 +1,234 @@ +import sys +import ast +from collections import deque +from ast import NodeVisitor +from copy import deepcopy + + +def is_py38_or_higher(): + if sys.version_info.major == 3 and sys.version_info.minor >= 8: + return True + return False + + +NAMECONSTANT_TYPE = ast.Constant if is_py38_or_higher() else ast.NameConstant + + +class CallTransformer(ast.NodeTransformer): + def __init__(self): + self.call_names = [] + + def visit_Attribute(self, node): + # self.generic_visit(node.value) + return node + + def param2str(self, param): + + def get_func(node): + if type(node) == ast.Name: + return node.id + elif type(node) == ast.Constant: + # ingore such as "this is a constant".join() + return "" + elif type(node) == ast.BinOp: + # ingore such as (a+b+c).fun() + return "" + elif type(node) == ast.Str: + # ingore such as "xxx".fun() + return "" + elif type(node) == ast.JoinedStr: + # ingore such as "xxx".fun() + return "" + elif type(node) == ast.Bytes: + # ingore such as "xxx".fun() + return "" + elif type(node) == ast.Compare: + # example "(x.matrix_exp() == torch.eye(20, 20, dtype=dtype, device=device)).all().item()" + # tests/test-cases/cfg-tests/pytorch-test-test_linalg.py + # ignore for now + return "" + elif type(node) == ast.Subscript: + # currently, we will ignore the slices because we cannot track the type of the value. + # for instance, a[something].fun() -> a.fun() + # this sacrifice + return get_func(node.value) + #elif type(node) == ast.JoinedStr: + # return "" + elif type(node) == ast.Attribute: + if type(node.value) in [ast.JoinedStr, ast.Constant]: + return node.attr + else: + return get_func(node.value) + "." + node.attr + elif type(node) == ast.Call: + return get_func(node.func) + elif type(node) == ast.IfExp: + return "" + elif type(node) == ast.Compare: + return "" + elif type(node) == ast.UnaryOp: + return "" + #ast.UnaryOp + else: + #import astor + #print(astor.to_source(node)) + raise Exception(str(type(node))) + + if isinstance(param, ast.Subscript): + return self.param2str(param.value) + if isinstance(param, ast.Call): + return get_func(param) + elif isinstance(param, ast.Name): + return param.id + elif isinstance(param, ast.Num): + # python 3.6 + return param.n + #return param.value + elif isinstance(param, ast.List): + return "List" + elif isinstance(param, ast.ListComp): + return "List" + elif isinstance(param, ast.Tuple): + return "Tuple" + elif isinstance(param, (ast.Dict, ast.DictComp)): + return "Dict" + elif isinstance(param, (ast.Set, ast.SetComp)): + return "Set" + elif isinstance(param, ast.Str): + return param.s + elif isinstance(param, ast.NameConstant): + return param.value + elif isinstance(param, ast.Constant): + return param.value + elif isinstance(param, ast.Expr): + return "Expr" + else: + return "unknown" + + def visit_Call(self, node): + + tmp_fun_node = deepcopy(node) + tmp_fun_node.args = [] + tmp_fun_node.keywords = [] + + callvisitor = FuncCallVisitor() + callvisitor.visit(tmp_fun_node) + + call_info = {"name": callvisitor.name, + "lineno": tmp_fun_node.lineno, + "col_offset": tmp_fun_node.col_offset, + "params": [] + } + self.call_names += [call_info] + for arg in node.args: + call_info["params"] += [self.param2str(arg)] + self.generic_visit(arg) + + for kw in node.keywords: + call_info["params"] += [self.param2str(kw.value)] + self.generic_visit(kw) + self.generic_visit(tmp_fun_node) + + return node + + +class FuncCallVisitor(ast.NodeVisitor): + def __init__(self): + self._name = deque() + self.call_names = [] + + def clear(self): + self._name = deque() + self.call_names = [] + + @property + def name(self): + return '.'.join(self._name) + + @name.deleter + def name(self): + self._name.clear() + + def visit_Name(self, node): + self._name.appendleft(node.id) + + def visit_Attribute(self, node): + + try: + self._name.appendleft(node.attr) + self._name.appendleft(node.value.id) + except AttributeError as e: + self.generic_visit(node) + + def visit_Call(self, node): + node.args = [] + node.keywords = [] + self.generic_visit(node) + return node + + + def visit_Subscript(self, node): + # ingore subscription slice + self.visit(node.value) + return node + +def get_args(node): + arg_type = [] + for arg in node.args: + if isinstance(arg, ast.Name): + arg_type.append(arg.id) + elif isinstance(arg, ast.Num): + arg_type.append("Num") + elif isinstance(arg, ast.List): + arg_type.append("List") + elif isinstance(arg, ast.ListComp): + arg_type.append("List") + elif isinstance(arg, ast.Tuple): + arg_type.append("Tuple") + elif isinstance(arg, ast.Dict): + arg_type.append("Dict") + elif isinstance(arg, ast.DictComp): + arg_type.append("Dict") + elif isinstance(arg, ast.Set): + arg_type.append("Set") + elif isinstance(arg, ast.SetComp): + arg_type.append("Set") + elif isinstance(arg, ast.Str): + arg_type.append("Str") + elif isinstance(arg, ast.NameConstant): + arg_type.append("NameConstant") + elif isinstance(arg, ast.Constant): + arg_type.append("Constant") + elif isinstance(arg, ast.Call): + arg_type.append(("Call", get_func_calls(arg)[0])) + else: + arg_type.append("Other") + return arg_type + + +def get_call_type(tree): + # how to remove + func_calls = [] + for node in ast.walk(tree): + if isinstance(node, ast.Call): + callvisitor = FuncCallVisitor() + callvisitor.visit(node.func) + func_calls += [(callvisitor.name, get_args(node))] + return func_calls + + +def get_call_type(tree): + # how to remove + func_calls = [] + for node in ast.walk(tree): + if isinstance(node, ast.Call): + callvisitor = FuncCallVisitor() + callvisitor.visit(node.func) + func_calls += [(callvisitor.name, get_args(node))] + return func_calls + + +def get_func_calls(tree): + node = deepcopy(tree) + transformer = CallTransformer() + transformer.visit(node) + return transformer.call_names diff --git a/codeanalyzer/dataflow/scalpel/core/vars_visitor.py b/codeanalyzer/dataflow/scalpel/core/vars_visitor.py new file mode 100644 index 0000000..f5e64f2 --- /dev/null +++ b/codeanalyzer/dataflow/scalpel/core/vars_visitor.py @@ -0,0 +1,205 @@ +import ast + +class VarsVisitor(ast.NodeVisitor): + def __init__(self): + self.result = list() + + def _ctx2str(self, ctx): + if isinstance(ctx, ast.Load): + return "load" + elif isinstance(ctx, ast.Store): + return "store" + elif isinstance(ctx, ast.Del): + return "del" + else: + raise Exception("unknown variable context") + + def visit_Name(self, node): + var_info = { + "name": node.id, + "lineno": node.lineno, + "col_offset": node.col_offset, + "usage" : self._ctx2str(node.ctx) + } + + self.result.append(var_info) + + + def visit_BoolOp(self, node): + for v in node.values: + self.visit(v) + + def visit_BinOp(self, node): + self.visit(node.left) + self.visit(node.right) + + def visit_UnaryOp(self, node): + self.visit(node.operand) + + def visit_Lambda(self, node): + return node + + def visit_IfExp(self, node): + self.visit(node.test) + self.visit(node.body) + self.visit(node.orelse) + + def visit_Dict(self, node): + for k in node.keys: + if k is not None: + self.visit(k) + for v in node.values: + self.visit(v) + + def visit_Set(self, node): + for e in node.elts: + self.visit(e) + + def comprehension(self, node): + self.visit(node.target) + self.visit(node.iter) + for c in node.ifs: + self.visit(c) + + def visit_ListComp(self, node): + for gen in node.generators: + self.comprehension(gen) + self.visit(node.elt) + + def visit_SetComp(self, node): + self.visit(node.elt) + for gen in node.generators: + self.comprehension(gen) + + def visit_DictComp(self, node): + for gen in node.generators: + self.comprehension(gen) + self.visit(node.key) + self.visit(node.value) + + + def visit_GeneratorComp(self, node): + self.visit(node.elt) + for gen in node.generators: + self.comprehension(gen) + + def visit_Yield(self, node): + if node.value: + self.visit(node.value) + + def visit_YieldFrom(self, node): + self.visit(node.value) + + def visit_Compare(self, node): + self.visit(node.left) + for c in node.comparators: + self.visit(c) + + def visit_Call(self, node): + self.visit(node.func) + for arg in node.args: + self.visit(arg) + for keyword in node.keywords: + self.visit(keyword) + + def visit_keyword(self, node): + self.visit(node.value) + + def visit_Attribute(self, node): + full_name = self.get_attr_name(node) + + var_info = { + "name": full_name, + "lineno": node.lineno, + "col_offset": node.col_offset, + "usage" : self._ctx2str(node.ctx) + } + self.result.append(var_info) + self.visit(node.value) + + def get_attr_name (self, node): + if isinstance(node, ast.Call): + # to be test + return self.get_attr_name(node.func) + if isinstance(node, ast.Name): + return node.id + elif isinstance(node, ast.Attribute): + attr_name = self.get_attr_name(node.value) + if attr_name is None: + return None + return attr_name +"."+node.attr + elif isinstance(node, ast.Subscript): + return self.get_attr_name(node.value) + else: + # such as (a**2).sum() + return None + + def slicev(self, node): + + if isinstance(node, ast.Constant): + return + if isinstance(node, ast.Slice): + if node.lower: + self.visit(node.lower) + if node.upper: + self.visit(node.upper) + if node.step: + self.visit(node.step) + elif isinstance(node, ast.ExtSlice): + if node.dims: + for d in node.dims: + self.visit(d) + elif isinstance(node,ast.Tuple): + for elt in node.elts: + self.visit(elt) + elif isinstance(node, ast.UnaryOp): + self.visit(node.operand) + elif isinstance(node, ast.Name): + self.visit(node) + # this is due to syntax change + elif hasattr(node, "value"): + self.visit(node.value) + + def visit_Subscript(self, node): + if isinstance(node.value, ast.Attribute): + pass + self.visit(node.value) + self.slicev(node.slice) + + def visit_Starred(self, node): + self.visit(node.value) + + def visit_List(self, node): + for el in node.elts: + self.visit(el) + + def visit_Tuple(self, node): + for el in node.elts: + self.visit(el) + + def visit_FunctionDef(self, node): + + for stmt in node.body: + self.visit(stmt) + #return node + + def visit_Assign(self, node): + for target in node.targets: + #if isinstance(target, ast.Subscript): + # target.value.ctx = ast.Store() + self.visit(target) + if not isinstance(node.value, ast.Lambda): + self.visit(node.value) + + #for target in node.targets: + # if isinstance(target, ast.Subscript): + # target.value.ctx = ast.Store() + # self.visit(target) + #else: + # self.visit(target) + + +def get_vars(node): + visitor = VarsVisitor() + visitor.visit(node) + return visitor.result diff --git a/pyproject.toml b/pyproject.toml index 69820b4..d36dece 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,10 @@ dependencies = [ # Shipped as a self-contained binary in its wheel, so it's available wherever # canpy is pip-installed (incl. Docker); core.py falls back to pip without it. "uv>=0.5.0", + # astor: runtime dependency of the vendored Scalpel SSA slice + # (scalpel/SSA/const.py uses astor.to_source). Pure-Python, installs + # everywhere; scalpel pins ~=0.8.1. + "astor>=0.8.1,<0.9.0", ] [project.optional-dependencies] @@ -61,12 +65,6 @@ dependencies = [ neo4j = [ "neo4j>=5.0.0,<6.0.0", ] -# The Scalpel-backed L4 may-alias oracle (ScalpelAliasOracle). Optional: when -# absent, `make_alias_oracle` degrades to the type-based fallback, so analysis -# still runs — installing it only sharpens alias precision. -scalpel = [ - "python-scalpel>=1.0b0", -] [dependency-groups] test = [ diff --git a/test/test_vendored_scalpel.py b/test/test_vendored_scalpel.py new file mode 100644 index 0000000..10ccef0 --- /dev/null +++ b/test/test_vendored_scalpel.py @@ -0,0 +1,26 @@ +"""The vendored, typed_ast-free scalpel slice: it must import and compute SSA +with no external scalpel / typed_ast / graphviz, and never pull in typeinfer.""" +import sys +import importlib + + +def test_vendored_scalpel_imports_and_computes_ssa_typed_ast_free(): + # No external scalpel shadowing the vendored copy. + assert "scalpel" not in sys.modules or not sys.modules["scalpel"].__file__.endswith( + "site-packages/scalpel/__init__.py" + ), "external python-scalpel is installed; the test must exercise the vendored copy" + + from codeanalyzer.dataflow.scalpel.cfg import CFGBuilder + from codeanalyzer.dataflow.scalpel.SSA.const import SSA + + src = "def f(a):\n b = a\n c = b\n return c\n" + module_cfg = CFGBuilder().build_from_src("m", src) + func_cfg = list(module_cfg.functioncfgs.values())[0] + ssa_results, const_dict = SSA().compute_SSA(func_cfg) + # The copy chain + return pseudo-name scalpel is known to produce. + assert ("b", 0) in const_dict and ("c", 0) in const_dict and ("", 0) in const_dict + + # The whole point: no typed_ast, and typeinfer was never vendored/loaded. + assert "typed_ast" not in sys.modules + loaded = [m for m in sys.modules if m.startswith("codeanalyzer.dataflow.scalpel")] + assert not any("typeinfer" in m for m in loaded), f"typeinfer leaked: {loaded}" From a68eb285a82c04b3400e762c1db2f181b254a8ed Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 22 Jul 2026 17:07:46 -0400 Subject: [PATCH 13/22] feat(dataflow): make vendored scalpel the default L4 alias oracle Repoint ScalpelAliasOracle.from_function at the vendored, typed_ast-free codeanalyzer.dataflow.scalpel slice instead of the external python-scalpel package, and drop the now-dead ImportError fallback branch in make_alias_oracle -- Scalpel can no longer be "absent" since it ships in the package. TypeBasedAliasOracle remains the runtime safety net for a per-callable Scalpel build failure only. Reframe the two L4 tests whose premise this repoint invalidates: test_make_alias_oracle_falls_back_when_scalpel_absent now forces a per-callable build failure instead of poisoning sys.modules["scalpel"] (which no longer affects the vendored import path), and test_scalpel_oracle_copy_chain drops its pytest.importorskip("scalpel") guard since the vendored oracle is always present. --- codeanalyzer/dataflow/scalpel_oracle.py | 21 ++++++++---------- test/test_v2_l4.py | 27 +++++++++++------------ test/test_vendored_scalpel.py | 29 ++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/codeanalyzer/dataflow/scalpel_oracle.py b/codeanalyzer/dataflow/scalpel_oracle.py index 1e25325..5b11e03 100644 --- a/codeanalyzer/dataflow/scalpel_oracle.py +++ b/codeanalyzer/dataflow/scalpel_oracle.py @@ -22,7 +22,7 @@ forks or re-runs Scalpel's solver — and turns the copy/const records into per-function copy-closure equivalence classes: - ``from scalpel.SSA.const import SSA`` + ``from codeanalyzer.dataflow.scalpel.SSA.const import SSA`` ``ssa_results, const_dict = SSA().compute_SSA(func_cfg)`` ``const_dict`` maps ``(name, version)`` to the ``ast`` value node that defined @@ -138,14 +138,14 @@ def from_function( ) -> "ScalpelAliasOracle": """Build from a function AST by consuming Scalpel's solved SSA state. - Imports Scalpel lazily (``ImportError`` if the optional dependency is - absent) and reuses the *same source* both graphs are built from — the + Imports the vendored Scalpel slice (``codeanalyzer.dataflow.scalpel``) + and reuses the *same source* both graphs are built from — the function's unparsed text — so the join is identity, not a fuzzy match. Raises on any build failure; :func:`make_alias_oracle` is the total, never-raising entry point callers should prefer. """ - from scalpel.SSA.const import SSA - from scalpel.cfg import CFGBuilder + from codeanalyzer.dataflow.scalpel.SSA.const import SSA + from codeanalyzer.dataflow.scalpel.cfg import CFGBuilder src = ast.unparse(func_ast) fname = name or getattr(func_ast, "name", None) @@ -250,19 +250,16 @@ def may_alias(self, path_a: str, path_b: str) -> bool: def make_alias_oracle(pycallable, func_ast, base_types) -> object: """Total selector for the L4 may-alias oracle. - Returns a :class:`ScalpelAliasOracle` when ``python-scalpel`` is importable - *and* builds successfully on ``func_ast``; otherwise logs once (INFO) and - returns a :class:`TypeBasedAliasOracle` over ``base_types``. Never raises — - mirrors how ``core._get_pycg_call_graph`` degrades on a missing/failed PyCG. + Returns a :class:`ScalpelAliasOracle` built on the vendored, typed_ast-free + Scalpel slice (``codeanalyzer.dataflow.scalpel``) — the default L4 oracle. + Falls back to :class:`TypeBasedAliasOracle` only when the per-callable + Scalpel build fails on this AST. Never raises. """ fallback = TypeBasedAliasOracle(base_types) try: return ScalpelAliasOracle.from_function( func_ast, base_types=base_types, fallback=fallback ) - except ImportError: - _note_fallback("python-scalpel not installed") - return fallback except Exception: _note_fallback("scalpel alias build failed") logger.debug("scalpel alias oracle build error", exc_info=True) diff --git a/test/test_v2_l4.py b/test/test_v2_l4.py index 016f12e..82bcb60 100644 --- a/test/test_v2_l4.py +++ b/test/test_v2_l4.py @@ -5,8 +5,6 @@ import sys from pathlib import Path -import pytest - from codeanalyzer.dataflow.alias import TypeBasedAliasOracle from codeanalyzer.dataflow.identity import IdentityMap from codeanalyzer.dataflow.sdg import ParamNode @@ -104,18 +102,19 @@ def emit(self, record): self.records.append(record) -def test_make_alias_oracle_falls_back_when_scalpel_absent(monkeypatch): +def test_make_alias_oracle_falls_back_on_build_failure(monkeypatch): """`make_alias_oracle` degrades to TypeBasedAliasOracle behavior and logs - once when Scalpel cannot be imported — regardless of whether the optional - dependency is actually installed in the runner.""" + once when the (now-vendored, always-present) Scalpel oracle fails to build + for a given callable — the only surviving fallback path now that Scalpel + can no longer be "absent" (it is vendored, not an optional import).""" import codeanalyzer.dataflow.scalpel_oracle as so - # Force `import scalpel...` to raise ImportError even if it is installed: - # a None entry in sys.modules makes the import machinery halt. Cover the - # top-level package and every submodule the oracle imports so a previously - # cached submodule cannot satisfy the import. - for mod in ("scalpel", "scalpel.cfg", "scalpel.SSA", "scalpel.SSA.const"): - monkeypatch.setitem(sys.modules, mod, None) + # Force the per-callable Scalpel build to fail, regardless of input. + monkeypatch.setattr( + so.ScalpelAliasOracle, + "from_function", + classmethod(lambda cls, *a, **k: (_ for _ in ()).throw(RuntimeError("boom"))), + ) # Reset the process-wide "logged once" guard and capture on the actual # (propagate=False) codeanalyzer logger. @@ -144,9 +143,9 @@ def test_make_alias_oracle_falls_back_when_scalpel_absent(monkeypatch): def test_scalpel_oracle_copy_chain(): - """When Scalpel is importable, the copy chain `b = a; c = b` places a, b, c - in one copy-closure class: a/b may-alias, an unrelated name does not.""" - pytest.importorskip("scalpel") + """Scalpel is vendored and always importable now, so the copy chain + `b = a; c = b` always places a, b, c in one copy-closure class: a/b + may-alias, an unrelated name does not.""" from codeanalyzer.dataflow.scalpel_oracle import ScalpelAliasOracle func_ast = ast.parse(COPY_CHAIN).body[0] diff --git a/test/test_vendored_scalpel.py b/test/test_vendored_scalpel.py index 10ccef0..c78e4d3 100644 --- a/test/test_vendored_scalpel.py +++ b/test/test_vendored_scalpel.py @@ -1,7 +1,6 @@ """The vendored, typed_ast-free scalpel slice: it must import and compute SSA with no external scalpel / typed_ast / graphviz, and never pull in typeinfer.""" import sys -import importlib def test_vendored_scalpel_imports_and_computes_ssa_typed_ast_free(): @@ -24,3 +23,31 @@ def test_vendored_scalpel_imports_and_computes_ssa_typed_ast_free(): assert "typed_ast" not in sys.modules loaded = [m for m in sys.modules if m.startswith("codeanalyzer.dataflow.scalpel")] assert not any("typeinfer" in m for m in loaded), f"typeinfer leaked: {loaded}" + + +import ast + + +def test_make_alias_oracle_defaults_to_scalpel_without_typed_ast(): + import sys + assert "typed_ast" not in sys.modules + from codeanalyzer.dataflow.scalpel_oracle import make_alias_oracle, ScalpelAliasOracle + + src = "def f(a):\n b = a\n c = b\n return c\n" + func_ast = ast.parse(src).body[0] + oracle = make_alias_oracle(pycallable=None, func_ast=func_ast, base_types={}) + # The whole point: scalpel is the default, not the type-based fallback. + assert isinstance(oracle, ScalpelAliasOracle), type(oracle).__name__ + # It answers queries (copies alias; unrelated locals do not). + assert oracle.may_alias("b", "c") is True + + +def test_make_alias_oracle_is_deterministic(): + import ast as _ast + from codeanalyzer.dataflow.scalpel_oracle import make_alias_oracle + src = "def f(a):\n b = a\n c = b\n return c\n" + fa = _ast.parse(src).body[0] + o1 = make_alias_oracle(None, _ast.parse(src).body[0], {}) + o2 = make_alias_oracle(None, _ast.parse(src).body[0], {}) + pairs = [("a", "b"), ("b", "c"), ("a", "c"), ("b", "b")] + assert [o1.may_alias(x, y) for x, y in pairs] == [o2.may_alias(x, y) for x, y in pairs] From 372db107de5bc39def8bd70efc1ae932569c1e21 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 22 Jul 2026 17:24:07 -0400 Subject: [PATCH 14/22] docs(dataflow): record vendored scalpel as the default L4 oracle --- .claude/SCHEMA_DECISIONS.md | 11 +++++++++++ CHANGELOG.md | 10 ++++++++++ CLAUDE.md | 11 +++++++---- test/test_vendored_scalpel.py | 37 ++++++++++++++++++++++++++++++++--- 4 files changed, 62 insertions(+), 7 deletions(-) diff --git a/.claude/SCHEMA_DECISIONS.md b/.claude/SCHEMA_DECISIONS.md index f222e9f..12dc408 100644 --- a/.claude/SCHEMA_DECISIONS.md +++ b/.claude/SCHEMA_DECISIONS.md @@ -152,6 +152,17 @@ supported 3.9–3.13+ range; `compute_SSA`'s return contract is discovered from source (not a documented public API) — wrap it behind `ScalpelAliasOracle` to contain upstream drift. +**Follow-up (2026-07-22): Scalpel vendored, now the default.** The Stage-0 +"dependency hygiene" concern proved fatal for a hard dependency: `python-scalpel` +drags `typed_ast`, which has no wheel for Python 3.12+ and does not build from +source there, so `pip install python-scalpel` fails on 3.12/3.13/3.14. Since +`typed_ast` is imported only by `scalpel/typeinfer` (unused here), the 9-module +`SSA`/`cfg`/`core` slice the oracle loads was **vendored** into +`codeanalyzer/dataflow/scalpel/` (Apache-2.0, verbatim but for a `graphviz`-lazy +patch). `ScalpelAliasOracle` is now the shipping default on all supported Python; +`TypeBasedAliasOracle` is the runtime safety net only. See +`docs/superpowers/specs/2026-07-22-vendored-scalpel-default-oracle-design.md`. + ## Stage 5 — keystone conformance sweep (issue #98, schema_version 2.0.0) The stage-5 pre-release conformance check against the canonical schema-v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 82479b0..e64c020 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- The Scalpel-backed L4 points-to oracle is now **vendored** (`typed_ast`-free) + and the default on all supported Python (3.9–3.14); `python-scalpel` is no + longer an optional dependency and the `[scalpel]` extra is removed. On Python + 3.12+ (and anywhere the `[scalpel]` extra was not installed), L4 + `prov:["points-to"]` data-dependence edges are now Scalpel-precise rather than + the coarser type-based over-approximation; the `prov:["ssa"]` set and the + `L3 ⊆ L4` monotonicity invariant are unchanged. Adds `astor` as a runtime + dependency. + ## [1.0.3] - 2026-07-21 ### Fixed diff --git a/CLAUDE.md b/CLAUDE.md index 37a9a05..6816472 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -122,10 +122,13 @@ declared callables by their `can://` tree id, imported/builtin targets by a (`codeanalyzer/dataflow/scalpel_oracle.py`) consumes `python-scalpel`'s **solved SSA + copy/const** state (it never forks the solver) as a copy-closure union-find over access paths, behind the frozen `may_alias(path_a, path_b) -> bool` - interface. `python-scalpel` is an **optional** dependency: if it is absent or a - build/query fails, the analyzer falls back to the total `TypeBasedAliasOracle` - (`alias.py`) and degrades, never raising. The type-based oracle is the sanctioned - fallback, not the shipping default. + interface. Scalpel is **vendored** (`codeanalyzer/dataflow/scalpel/`, a + `typed_ast`-free 9-module slice of `python-scalpel 1.0b0`, Apache-2.0) so it is + the **shipping default** L4 oracle on every supported Python — there is no + external `python-scalpel`/`typed_ast` dependency. `TypeBasedAliasOracle` + (`alias.py`) is retained only as the runtime safety net: on a per-callable + Scalpel build failure or a per-query unresolved access path, `may_alias` + degrades to it, never raising. - **CLI gating** (`codeanalyzer/__main__.py`): `-a` max is 4; `--graphs sdg` requires `-a 4` (a flag error below that); `cfg,dfg,pdg` require `-a 3`; `--graph-field-depth` (the `k_limit`) is valid at L3+. diff --git a/test/test_vendored_scalpel.py b/test/test_vendored_scalpel.py index c78e4d3..f789199 100644 --- a/test/test_vendored_scalpel.py +++ b/test/test_vendored_scalpel.py @@ -1,7 +1,10 @@ """The vendored, typed_ast-free scalpel slice: it must import and compute SSA with no external scalpel / typed_ast / graphviz, and never pull in typeinfer.""" +import ast import sys +import pytest + def test_vendored_scalpel_imports_and_computes_ssa_typed_ast_free(): # No external scalpel shadowing the vendored copy. @@ -25,9 +28,6 @@ def test_vendored_scalpel_imports_and_computes_ssa_typed_ast_free(): assert not any("typeinfer" in m for m in loaded), f"typeinfer leaked: {loaded}" -import ast - - def test_make_alias_oracle_defaults_to_scalpel_without_typed_ast(): import sys assert "typed_ast" not in sys.modules @@ -51,3 +51,34 @@ def test_make_alias_oracle_is_deterministic(): o2 = make_alias_oracle(None, _ast.parse(src).body[0], {}) pairs = [("a", "b"), ("b", "c"), ("a", "c"), ("b", "b")] assert [o1.may_alias(x, y) for x, y in pairs] == [o2.may_alias(x, y) for x, y in pairs] + + +def _pip_scalpel_available(): + try: + import importlib.util + # the EXTERNAL package, not our vendored copy + return importlib.util.find_spec("scalpel") is not None and \ + not importlib.util.find_spec("scalpel").origin.endswith( + "codeanalyzer/dataflow/scalpel/__init__.py") + except Exception: + return False + + +@pytest.mark.skipif(not _pip_scalpel_available(), + reason="upstream python-scalpel not installed (3.12+ can't build typed_ast)") +def test_vendored_ssa_matches_upstream(): + """On a Python where pip python-scalpel installs (<=3.11), the vendored copy + must produce identical SSA const_dict keys as upstream on the same source.""" + import scalpel.SSA.const as up_ssa + import scalpel.cfg as up_cfg + from codeanalyzer.dataflow.scalpel.SSA.const import SSA as VSSA + from codeanalyzer.dataflow.scalpel.cfg import CFGBuilder as VCFG + + src = "def f(a):\n b = a\n if a:\n b = 2\n return b\n" + up_c = up_cfg.CFGBuilder().build_from_src("m", src) + v_c = VCFG().build_from_src("m", src) + up_fn = list(up_c.functioncfgs.values())[0] + v_fn = list(v_c.functioncfgs.values())[0] + _, up_const = up_ssa.SSA().compute_SSA(up_fn) + _, v_const = VSSA().compute_SSA(v_fn) + assert sorted(map(str, up_const)) == sorted(map(str, v_const)) From 416cf0ca8b934535a247132522d80e8fd26a0168 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 22 Jul 2026 17:33:32 -0400 Subject: [PATCH 15/22] docs: README reflects vendored scalpel as built-in default (drop removed [scalpel] extra) --- README.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 0a4e8e1..78fccd5 100644 --- a/README.md +++ b/README.md @@ -104,12 +104,9 @@ For the optional **live Neo4j push** (`--emit neo4j --neo4j-uri …`), install t pip install 'codeanalyzer-python[neo4j]' ``` -For the **Scalpel-backed points-to oracle** at level 4, install the `scalpel` extra. It is optional: -when it is absent, level 4 automatically falls back to the built-in type-based oracle. - -```sh -pip install 'codeanalyzer-python[scalpel]' -``` +The **Scalpel-backed points-to oracle** at level 4 is vendored and built in — no extra install +required. If Scalpel cannot resolve a construct, level 4 automatically falls back to the built-in +type-based oracle. ### Install via shell script @@ -529,11 +526,12 @@ symbol-table signature by construction - **Points-to oracle (level 4):** the **Scalpel** may-alias oracle — `ScalpelAliasOracle` (`codeanalyzer/dataflow/scalpel_oracle.py`) — consumes Scalpel's SSA copy/const facts to answer `may_alias(path_a, path_b)`, adding the alias-aware DDG edges (`prov: ["points-to"]`) and the - interprocedural summaries. `python-scalpel` is an **optional dependency** - (`pip install 'codeanalyzer-python[scalpel]'`); when it is absent or cannot resolve a construct, - the analyzer automatically falls back to the built-in `TypeBasedAliasOracle` (Jedi-inferred types; - unknown types conservatively alias), keeping the `may_alias` interface total. Call dispatch comes - from the merged Jedi(+PyCG) call graph, treated as a frozen oracle. + interprocedural summaries. Scalpel is **vendored** — a `typed_ast`-free slice built into the + package under `codeanalyzer/dataflow/scalpel/` — so it is the **default** level-4 oracle with no + external dependency to install; the analyzer falls back to the built-in `TypeBasedAliasOracle` + (Jedi-inferred types; unknown types conservatively alias) only when Scalpel can't resolve a + construct or a per-callable build fails, keeping the `may_alias` interface total. Call dispatch + comes from the merged Jedi(+PyCG) call graph, treated as a frozen oracle. - **Summaries:** relational formal-in → formal-out flows composed bottom-up over the Tarjan SCC condensation of the call graph, a monotone fixpoint within SCCs; globals ride as extra formals, closure captures bind at definition sites. From 3df8ed77518bac94b551595932dd724a537b0b47 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Mon, 27 Jul 2026 11:30:49 -0400 Subject: [PATCH 16/22] chore: untrack the superpowers design doc (convention: specs live in issues) The design content now lives in issue #113; docs/superpowers/** is local drafting scratch and stays out of the repo. --- ...-vendored-scalpel-default-oracle-design.md | 198 ------------------ 1 file changed, 198 deletions(-) delete mode 100644 docs/superpowers/specs/2026-07-22-vendored-scalpel-default-oracle-design.md diff --git a/docs/superpowers/specs/2026-07-22-vendored-scalpel-default-oracle-design.md b/docs/superpowers/specs/2026-07-22-vendored-scalpel-default-oracle-design.md deleted file mode 100644 index fb0ac43..0000000 --- a/docs/superpowers/specs/2026-07-22-vendored-scalpel-default-oracle-design.md +++ /dev/null @@ -1,198 +0,0 @@ -# Vendored, typed_ast-free Scalpel as the default L4 oracle - -- **Date:** 2026-07-22 -- **Status:** Design approved; ready for an implementation plan -- **Scope:** Make the Scalpel-backed points-to oracle (`ScalpelAliasOracle`) the - shipping default for L4 dataflow on **every** supported Python, by vendoring a - minimal, `typed_ast`-free slice of `python-scalpel` into the package. - -## Motivation - -`python-scalpel` is the primary L4 may-alias oracle, but it ships today as an -**optional** extra with a type-based fallback — because `python-scalpel 1.0b0` -hard-depends on **`typed_ast`**, an abandoned package whose last release -(`1.5.5`) has no wheel for Python 3.12+ and fails to compile from source on a -modern compiler. So on the interpreters most users (and this repo's own dev env) -now run — 3.12, 3.13, 3.14 — scalpel cannot be installed at all, and L4 silently -degrades to the coarser type-based oracle. - -We want scalpel to be the **default** L4 oracle everywhere. It cannot be a hard -PyPI dependency (the `typed_ast` wall), and a Python-version cap to reach it -(≤3.11) would drop the 3.12–3.14 support the analyzer just added. The way -through is to **vendor** the small slice of scalpel we use, minus the one module -that drags `typed_ast`. - -## Feasibility (verified, not assumed) - -- **`typed_ast` is spurious for our use.** It is imported in exactly one scalpel - module — `scalpel/typeinfer/analysers.py` — which codeanalyzer never touches. - The oracle uses only `scalpel.SSA.const` and `scalpel.cfg`. -- **The slice runs without `typed_ast`.** On Python 3.12 with `typed_ast` *not* - installed, `cfg.CFGBuilder` + `SSA.const.SSA` import and `compute_SSA` returns - the exact SSA/const facts the oracle consumes (verified: a copy chain yields - `('b',0), ('c',0), ('',0)`). -- **Exact closure = 9 modules**, provably free of `typeinfer`/`typed_ast` - (checked via `sys.modules` after importing the two entry points): - `scalpel/__init__.py`, `SSA/{__init__,const}.py`, - `cfg/{__init__,builder,model}.py`, `core/{__init__,func_call_visitor,vars_visitor}.py`. -- **Third-party imports touched:** `astor` (used at runtime — `SSA/const.py:191` - `astor.to_source`), `graphviz` (imported at module load in `cfg/model.py` but - only used by the unused `build_visual()`), `networkx` (already a dependency). -- **License = Apache-2.0** (confirmed on `SMAT-Lab/Scalpel` and in the wheel's - `LICENSE`) — vendorable with attribution, same family as the already-bundled - PyCG. -- **Upstream is abandoned** (`1.0b0`, frozen) — the usual "vendored copy drifts - from upstream" cost does not apply. - -## Goals - -- `ScalpelAliasOracle` is the **default** L4 oracle on Python 3.9–3.14+, with no - external `python-scalpel`/`typed_ast` dependency and no `pip install` breakage. -- The type-based oracle (`TypeBasedAliasOracle`) is retained purely as the - runtime safety net (per-callable build failure, per-query unresolved path). -- Behavior for users who already had the `[scalpel]` extra on ≤3.11 is unchanged - (same code). Behavior on 3.12+ *improves* (real points-to instead of the - type-based over-approximation). - -## Non-goals - -- No change to the L4 *schema* (`prov` values `ssa`/`points-to` unchanged; the - DDG shape is the same). -- No `requires-python` change — stays `>=3.9`. -- No vendoring of scalpel's `typeinfer`, `call_graph`, `pycg`, `import_graph`, - `scope_graph`, or `dataflow` packages — only the 9-module `SSA`/`cfg`/`core` - closure the oracle actually loads. -- No change to `defuse.py`'s two-rule DDG contract or the `k_limit` machinery. - -## Design - -### The vendored package: `codeanalyzer/dataflow/scalpel/` - -A normal package (no `_vendor/` layer), mirroring upstream's layout so scalpel's -relative imports (`from ..core.vars_visitor import get_vars`) resolve within -`codeanalyzer.dataflow.scalpel`: - -``` -codeanalyzer/dataflow/scalpel/ - __init__.py - SSA/__init__.py, const.py - cfg/__init__.py, builder.py, model.py - core/__init__.py, func_call_visitor.py, vars_visitor.py - LICENSE # upstream Apache-2.0, copied verbatim - README.md # provenance + the one patch (see below) -``` - -The `scalpel/` wrapper name is kept (rather than flattening `SSA`/`cfg`/`core` -into `dataflow/`) because `codeanalyzer/dataflow/cfg.py` already exists and -scalpel ships its own `cfg/` package — the wrapper namespaces them apart. - -The 9 files are copied **verbatim** from `python-scalpel 1.0b0`, with **exactly -one patch**: - -- `cfg/model.py`: the module-load `import graphviz as gv` (line 12) is moved to a - lazy import *inside* the unused `build_visual()` method. This removes `graphviz` - as a runtime dependency (we never render). No other line changes; the - SSA/CFG-computation code paths are byte-identical to upstream. - -### Attribution (Apache-2.0) - -- `codeanalyzer/dataflow/scalpel/LICENSE` — upstream's Apache-2.0 license, copied. -- `codeanalyzer/dataflow/scalpel/README.md` — records: source repo - (`SMAT-Lab/Scalpel`), the pinned version (`1.0b0`), the exact 9-file list, the - single `graphviz`-lazy patch, and that `typeinfer` (the sole `typed_ast` user) - was deliberately excluded. -- The repo's top-level `NOTICE` gains a one-line entry crediting vendored Scalpel - (Apache-2.0), alongside the existing attributions. - -### Dependencies (`pyproject.toml`) - -- **Add** `astor` to core `dependencies` (genuine runtime dep; pure-Python; - installs on every platform/Python). -- **Remove** the `[project.optional-dependencies].scalpel` extra (scalpel is now - built in). -- `networkx` unchanged (already core); `graphviz` and `typed_ast` are **not** - dependencies. -- `requires-python` stays `>=3.9`. - -### Oracle rewiring: `codeanalyzer/dataflow/scalpel_oracle.py` - -- `ScalpelAliasOracle.from_function` imports from the vendored path: - `from codeanalyzer.dataflow.scalpel.SSA.const import SSA` and - `from codeanalyzer.dataflow.scalpel.cfg import CFGBuilder`. -- `make_alias_oracle`: the `ImportError` ("python-scalpel not installed") branch - becomes dead — the vendored import cannot be absent — and is **removed**. The - per-callable **build-failure** `except` (returns the type-based fallback) and - the per-query unresolved-path fallback inside `may_alias` both **stay**, so the - total, never-raises contract is preserved. `ScalpelAliasOracle` is now the - default the selector returns. -- Docstrings/comments updated to drop the "optional / not installed" framing. - -### Behavior change - -On Python 3.12+/3.14 — and for anyone who never installed the old `[scalpel]` -extra — L4's `prov:["points-to"]` DDG edges were previously derived from the -*type-based* over-approximation; they now become **scalpel-precise** (a tighter -subset). The `prov:["ssa"]` edges (the L3 subset) are unchanged, so the -**monotonicity invariant `L3 ⊆ L4` still holds** (the ssa set is untouched; -points-to is an additive overlay). Users who already had the `[scalpel]` extra on -≤3.11 see no change. This L4 precision improvement is recorded in `CHANGELOG.md`. - -### Docs - -`CLAUDE.md` and `.claude/SCHEMA_DECISIONS.md` are updated to record: scalpel is -now **vendored** (`codeanalyzer/dataflow/scalpel/`, `typed_ast`-free) and the -**shipping default** L4 oracle on all supported Python — superseding the earlier -"the type-based oracle is the sanctioned fallback, not the shipping default" -statement (it is now the runtime safety net, not the default). The Stage-0 -`SCHEMA_DECISIONS` entry gains a follow-up noting the `typed_ast` wall on 3.12+ -that forced vendoring. - -## Testing plan - -1. **Import hygiene.** `import codeanalyzer.dataflow.scalpel` and building the - oracle succeed with no external `scalpel`/`typed_ast`/`graphviz` installed; - assert `typeinfer` is never imported (scan `sys.modules` after building the - oracle). -2. **`typed_ast`-free property gate.** Assert `typed_ast` is not importable in - the environment, yet `make_alias_oracle` builds a `ScalpelAliasOracle` and - `may_alias` returns verdicts on a copy-chain fixture — the whole point, - encoded as a regression gate. -3. **Vendored-fidelity check.** On Python ≤3.11 (where pip `python-scalpel` - installs), compare the vendored `compute_SSA`/CFG output against the - pip-installed scalpel on a set of fixtures — proves the copy is faithful. - `skipif` the pip package can't install (3.12+); the copy is verbatim, so this - is a belt-and-suspenders check. -4. **L4 regression (the live acceptance test).** `test_v2_l4.py`, - `test_v2_l4_summary.py`, `test_dataflow_sdg.py`, `test_dataflow_defuse.py` now - exercise the **scalpel** path by default — the first time scalpel-L4 runs on - the 3.14 dev env. They must stay green and now genuinely test scalpel-derived - points-to rather than the fallback. -5. **Determinism.** Run L4 twice on a fixture and assert identical output (the - scalpel SSA feeds `may_alias`; the `#99` `PYTHONHASHSEED=0` pin + the oracle's - access-path normalization must keep it stable). - -## Risks and rollback - -- **Incomplete closure** — a missed transitive import would surface as an - `ImportError` at oracle build, which the per-callable fallback swallows into a - silent type-based degrade. Mitigation: the import-hygiene test (1) builds the - oracle and asserts it is a `ScalpelAliasOracle`, so a broken closure fails - loudly rather than degrading silently. -- **Vendored-copy divergence from upstream behavior** — mitigated by the verbatim - copy (only the `graphviz` import patched) and the fidelity check (3). -- **L4 output change on 3.12+** is intended, documented in the CHANGELOG, and - bounded by the preserved monotonicity invariant. -- **Rollback** is a single revert: delete `codeanalyzer/dataflow/scalpel/`, - restore the `[scalpel]` extra + the `make_alias_oracle` ImportError branch, and - drop `astor`. No schema or id changes to unwind. - -## Out of scope - -- Replacing `astor.to_source` with stdlib `ast.unparse` to drop the `astor` dep - (a behavior-risking change to vendored SSA logic — deferred; `astor` is tiny - and universal). -- A scalpel-installed CI lane / capturing the L4 (`a4`) equivalence goldens from - the separate analysis-pipeline branch — that work belongs to that branch once - it merges; here scalpel simply becomes available for it. -- Vendoring or using scalpel's type-inference (`typeinfer`) for the type-guided - alias branch — the type-based oracle already covers that fallback. From 55eb4a367d34350156e58ffb92fa074af130ad5f Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Mon, 27 Jul 2026 11:37:28 -0400 Subject: [PATCH 17/22] chore: untrack the superpowers plan doc (convention: plans live in issues) Content remains readable at the merge commit; the work is tracked in #113. --- ...6-07-22-vendored-scalpel-default-oracle.md | 455 ------------------ 1 file changed, 455 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-22-vendored-scalpel-default-oracle.md diff --git a/docs/superpowers/plans/2026-07-22-vendored-scalpel-default-oracle.md b/docs/superpowers/plans/2026-07-22-vendored-scalpel-default-oracle.md deleted file mode 100644 index 18520bb..0000000 --- a/docs/superpowers/plans/2026-07-22-vendored-scalpel-default-oracle.md +++ /dev/null @@ -1,455 +0,0 @@ -# Vendored typed_ast-free Scalpel as Default L4 Oracle — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Vendor a minimal, `typed_ast`-free slice of `python-scalpel 1.0b0` into `codeanalyzer/dataflow/scalpel/` and make `ScalpelAliasOracle` the shipping-default L4 oracle on every supported Python, with no external `python-scalpel`/`typed_ast` dependency. - -**Architecture:** Copy the 9-module `SSA`/`cfg`/`core` closure the oracle actually loads (verified free of `typeinfer`/`typed_ast`) into a `codeanalyzer.dataflow.scalpel` package, patched only to make `graphviz` a non-required import. Repoint `scalpel_oracle.py` at the vendored path and drop its "optional dependency absent" fallback branch, so the type-based oracle demotes to a pure runtime safety net. - -**Tech Stack:** Python 3.9–3.14, `astor` (new runtime dep), `networkx` (existing), pytest, Poetry (dev env, currently 3.14.5), uv (CI + fetching the vendored source). - -## Global Constraints - -- Conventional Commits (`type(scope): summary`). -- NEVER add AI/Claude authorship anywhere (no `Co-Authored-By`, "Generated with", 🤖) — commits, PRs, code, docs. Absolute. -- **Apache-2.0 attribution is mandatory** for the vendored code: copy upstream `LICENSE` into the vendored dir, add a provenance `README.md`, and add a `NOTICE` entry. -- **`requires-python` stays `>=3.9`** — do NOT cap it. -- **Behavior:** the monotonicity invariant `L3 ⊆ L4` must still hold (the `prov:["ssa"]` edge set is unchanged; scalpel only sharpens the additive `prov:["points-to"]` overlay). No schema change (`schema_version` stays `2.0.0`). -- The vendored copy is **verbatim** from `python-scalpel 1.0b0` except the single documented `graphviz` patch. -- Vendor **exactly** these 9 files — no other scalpel modules (`typeinfer`, `call_graph`, `pycg`, `import_graph`, `scope_graph`, `dataflow`, `rewriter.py`, and the unused `SSA/{alg,def_use,ssa}.py` + extra `core/*.py`): - `__init__.py`, `SSA/__init__.py`, `SSA/const.py`, `cfg/__init__.py`, `cfg/builder.py`, `cfg/model.py`, `core/__init__.py`, `core/func_call_visitor.py`, `core/vars_visitor.py`. -- Run tests with `poetry run python -m pytest` (managed env, py3.14). Do NOT use `uv run` for the dev env (it creates a stray in-project `.venv` that shadows Poetry). There must be no in-project `.venv`. - -## File Structure - -- Create `codeanalyzer/dataflow/scalpel/` — the 9 vendored files + `LICENSE` + `README.md`. -- Modify `codeanalyzer/dataflow/scalpel/cfg/model.py` — the one `graphviz`-lazy patch. -- Modify `codeanalyzer/dataflow/scalpel_oracle.py` — repoint imports, drop the dead ImportError branch. -- Modify `pyproject.toml` — add `astor`, remove the `[scalpel]` extra. -- Modify `NOTICE` — Scalpel Apache-2.0 attribution. -- Modify `CLAUDE.md`, `.claude/SCHEMA_DECISIONS.md`, `CHANGELOG.md` — record the reversal + behavior change. -- Create `test/test_vendored_scalpel.py` — import-hygiene, typed_ast-free, fidelity, determinism tests. - ---- - -### Task 1: Vendor the scalpel slice + dependency + attribution - -**Files:** -- Create: `codeanalyzer/dataflow/scalpel/**` (9 files + `LICENSE` + `README.md`) -- Modify: `codeanalyzer/dataflow/scalpel/cfg/model.py` (graphviz patch) -- Modify: `pyproject.toml` (add `astor`, remove `[scalpel]` extra) -- Modify: `NOTICE` -- Test: `test/test_vendored_scalpel.py` - -**Interfaces:** -- Produces: importable `codeanalyzer.dataflow.scalpel.SSA.const.SSA` and `codeanalyzer.dataflow.scalpel.cfg.CFGBuilder`, functional with no external `scalpel`/`typed_ast`/`graphviz`. - -- [ ] **Step 1: Write the failing import-hygiene test** - -Create `test/test_vendored_scalpel.py`: - -```python -"""The vendored, typed_ast-free scalpel slice: it must import and compute SSA -with no external scalpel / typed_ast / graphviz, and never pull in typeinfer.""" -import sys -import importlib - - -def test_vendored_scalpel_imports_and_computes_ssa_typed_ast_free(): - # No external scalpel shadowing the vendored copy. - assert "scalpel" not in sys.modules or not sys.modules["scalpel"].__file__.endswith( - "site-packages/scalpel/__init__.py" - ), "external python-scalpel is installed; the test must exercise the vendored copy" - - from codeanalyzer.dataflow.scalpel.cfg import CFGBuilder - from codeanalyzer.dataflow.scalpel.SSA.const import SSA - - src = "def f(a):\n b = a\n c = b\n return c\n" - module_cfg = CFGBuilder().build_from_src("m", src) - func_cfg = list(module_cfg.functioncfgs.values())[0] - ssa_results, const_dict = SSA().compute_SSA(func_cfg) - # The copy chain + return pseudo-name scalpel is known to produce. - assert ("b", 0) in const_dict and ("c", 0) in const_dict and ("", 0) in const_dict - - # The whole point: no typed_ast, and typeinfer was never vendored/loaded. - assert "typed_ast" not in sys.modules - loaded = [m for m in sys.modules if m.startswith("codeanalyzer.dataflow.scalpel")] - assert not any("typeinfer" in m for m in loaded), f"typeinfer leaked: {loaded}" -``` - -- [ ] **Step 2: Run it to confirm it fails** - -Run: `poetry run python -m pytest test/test_vendored_scalpel.py -q` -Expected: FAIL with `ModuleNotFoundError: No module named 'codeanalyzer.dataflow.scalpel'`. - -- [ ] **Step 3: Vendor the 9 files + LICENSE (reproducible fetch)** - -Fetch the exact upstream source and copy only the 9 files + the license: - -```bash -cd /home/rkrsn/workspace/codellm-devkit/codeanalyzer-python -TMP=$(mktemp -d) -uv pip install --no-deps --target "$TMP" 'python-scalpel==1.0b0' -DST=codeanalyzer/dataflow/scalpel -mkdir -p "$DST/SSA" "$DST/cfg" "$DST/core" -cp "$TMP/scalpel/__init__.py" "$DST/__init__.py" -cp "$TMP/scalpel/SSA/__init__.py" "$DST/SSA/__init__.py" -cp "$TMP/scalpel/SSA/const.py" "$DST/SSA/const.py" -cp "$TMP/scalpel/cfg/__init__.py" "$DST/cfg/__init__.py" -cp "$TMP/scalpel/cfg/builder.py" "$DST/cfg/builder.py" -cp "$TMP/scalpel/cfg/model.py" "$DST/cfg/model.py" -cp "$TMP/scalpel/core/__init__.py" "$DST/core/__init__.py" -cp "$TMP/scalpel/core/func_call_visitor.py" "$DST/core/func_call_visitor.py" -cp "$TMP/scalpel/core/vars_visitor.py" "$DST/core/vars_visitor.py" -cp "$TMP"/python_scalpel-1.0b0.dist-info/LICENSE "$DST/LICENSE" -rm -rf "$TMP" -``` - -Verify exactly 9 `.py` files landed: - -```bash -find codeanalyzer/dataflow/scalpel -name '*.py' | sort -# expect the 9 listed in Global Constraints, nothing else -``` - -- [ ] **Step 4: Apply the single graphviz patch** - -In `codeanalyzer/dataflow/scalpel/cfg/model.py`, the top-level `import graphviz as gv` (line 12) makes the module require `graphviz` at load. `graphviz` is used only by the visualization methods (`_build_visual`/`build_visual`), which codeanalyzer never calls. Replace the bare import: - -```python -import graphviz as gv -``` - -with a guarded import so the module loads without `graphviz`: - -```python -try: # PATCH (codeanalyzer): graphviz is only used by build_visual(), which - import graphviz as gv # codeanalyzer never calls. Keep the module importable -except ImportError: # without the graphviz dependency. - gv = None -``` - -This is the ONLY change to any vendored file. - -- [ ] **Step 5: Add attribution — `README.md` + `NOTICE` entry** - -Create `codeanalyzer/dataflow/scalpel/README.md`: - -```markdown -# Vendored Scalpel (typed_ast-free slice) - -Vendored from **[SMAT-Lab/Scalpel](https://github.com/SMAT-Lab/Scalpel)**, -package `python-scalpel==1.0b0`, licensed **Apache-2.0** (see `LICENSE`). - -## Why vendored - -`python-scalpel` hard-depends on `typed_ast`, whose last release (1.5.5) has no -wheel for Python 3.12+ and fails to build from source on modern compilers — so -`pip install python-scalpel` fails on 3.12/3.13/3.14. `typed_ast` is imported by -exactly one scalpel module, `typeinfer/analysers.py`, which codeanalyzer does not -use. Vendoring the small slice the L4 may-alias oracle needs removes the -`typed_ast` dependency and makes scalpel the default oracle on every supported -Python. - -## What is vendored - -Exactly the 9-module closure that `scalpel.SSA.const` + `scalpel.cfg` load -(verified via `sys.modules`; provably free of `typeinfer`/`typed_ast`): - - __init__.py - SSA/__init__.py, SSA/const.py - cfg/__init__.py, cfg/builder.py, cfg/model.py - core/__init__.py, core/func_call_visitor.py, core/vars_visitor.py - -Copied verbatim except **one patch**: `cfg/model.py`'s top-level -`import graphviz as gv` is guarded (`try/except ImportError: gv = None`) so the -module imports without the `graphviz` package — the graphviz-using -`build_visual()` methods are unused here. - -Runtime deps of this slice: `astor`, `networkx` (both core dependencies of -codeanalyzer). `typed_ast` and `graphviz` are NOT required. - -To refresh: re-run the vendoring in `docs/superpowers/plans/2026-07-22-vendored-scalpel-default-oracle.md` Task 1. -``` - -Append to the top-level `NOTICE` (after the existing content): - -``` ---- Scalpel License Notice --- - -This project vendors a slice of Scalpel (python-scalpel), a product of SMAT-Lab, -under codeanalyzer/dataflow/scalpel/. Scalpel is licensed under the Apache -License 2.0. The full license text is included at -codeanalyzer/dataflow/scalpel/LICENSE. Source: https://github.com/SMAT-Lab/Scalpel -``` - -- [ ] **Step 6: Add `astor` dep, remove the `[scalpel]` extra** - -In `pyproject.toml`, add to the core `dependencies` array (after the `uv` entry), with a comment: - -```toml - # astor: runtime dependency of the vendored Scalpel SSA slice - # (scalpel/SSA/const.py uses astor.to_source). Pure-Python, installs - # everywhere; scalpel pins ~=0.8.1. - "astor>=0.8.1,<0.9.0", -``` - -Remove the entire `[project.optional-dependencies].scalpel` group (the -`scalpel = ["python-scalpel>=1.0b0"]` block and its comment). Leave the `neo4j` -extra intact. - -- [ ] **Step 7: Install the new dep into the dev env** - -Run: `poetry install --with test` -Then: `poetry run python -c "import astor; print('astor', astor.__version__)"` -Expected: prints the astor version (installs on 3.14 — pure Python). - -- [ ] **Step 8: Run the hygiene test — now green** - -Run: `poetry run python -m pytest test/test_vendored_scalpel.py -q` -Expected: PASS (the vendored slice imports + computes SSA on 3.14, typed_ast-free). - -- [ ] **Step 9: Commit** - -```bash -git add codeanalyzer/dataflow/scalpel pyproject.toml NOTICE test/test_vendored_scalpel.py -git commit -m "feat(dataflow): vendor typed_ast-free scalpel SSA/cfg slice" -``` - ---- - -### Task 2: Repoint the oracle at the vendored slice and make it the default - -**Files:** -- Modify: `codeanalyzer/dataflow/scalpel_oracle.py` -- Test: `test/test_vendored_scalpel.py` (append) - -**Interfaces:** -- Consumes: the vendored `codeanalyzer.dataflow.scalpel.SSA.const.SSA` / `...cfg.CFGBuilder` (Task 1). -- Produces: `make_alias_oracle(pycallable, func_ast, base_types)` returns a `ScalpelAliasOracle` by default (never the type-based oracle for *absence*, only for a per-callable build failure). - -- [ ] **Step 1: Write the failing "scalpel is the default" test** - -Append to `test/test_vendored_scalpel.py`: - -```python -import ast - - -def test_make_alias_oracle_defaults_to_scalpel_without_typed_ast(): - import sys - assert "typed_ast" not in sys.modules - from codeanalyzer.dataflow.scalpel_oracle import make_alias_oracle, ScalpelAliasOracle - - src = "def f(a):\n b = a\n c = b\n return c\n" - func_ast = ast.parse(src).body[0] - oracle = make_alias_oracle(pycallable=None, func_ast=func_ast, base_types={}) - # The whole point: scalpel is the default, not the type-based fallback. - assert isinstance(oracle, ScalpelAliasOracle), type(oracle).__name__ - # It answers queries (copies alias; unrelated locals do not). - assert oracle.may_alias("b", "c") is True - - -def test_make_alias_oracle_is_deterministic(): - import ast as _ast - from codeanalyzer.dataflow.scalpel_oracle import make_alias_oracle - src = "def f(a):\n b = a\n c = b\n return c\n" - fa = _ast.parse(src).body[0] - o1 = make_alias_oracle(None, _ast.parse(src).body[0], {}) - o2 = make_alias_oracle(None, _ast.parse(src).body[0], {}) - pairs = [("a", "b"), ("b", "c"), ("a", "c"), ("b", "b")] - assert [o1.may_alias(x, y) for x, y in pairs] == [o2.may_alias(x, y) for x, y in pairs] -``` - -- [ ] **Step 2: Run it to confirm it fails** - -Run: `poetry run python -m pytest test/test_vendored_scalpel.py -k make_alias_oracle -q` -Expected: FAIL — `make_alias_oracle` still imports `from scalpel.SSA.const import SSA` (external, absent on 3.14) and returns the type-based fallback, so `isinstance(..., ScalpelAliasOracle)` is False. - -- [ ] **Step 3: Repoint the imports in `from_function`** - -In `codeanalyzer/dataflow/scalpel_oracle.py`, in `ScalpelAliasOracle.from_function`, change: - -```python - from scalpel.SSA.const import SSA - from scalpel.cfg import CFGBuilder -``` - -to: - -```python - from codeanalyzer.dataflow.scalpel.SSA.const import SSA - from codeanalyzer.dataflow.scalpel.cfg import CFGBuilder -``` - -Update that method's docstring line "Imports Scalpel lazily (``ImportError`` if the optional dependency is absent)…" to "Imports the vendored Scalpel slice (``codeanalyzer.dataflow.scalpel``) and reuses the *same source*…". Also update the module docstring reference at the top of the file (`from scalpel.SSA.const import SSA`) to the vendored path. - -- [ ] **Step 4: Drop the dead ImportError branch in `make_alias_oracle`** - -Replace the current `make_alias_oracle` body: - -```python -def make_alias_oracle(pycallable, func_ast, base_types) -> object: - """Total selector for the L4 may-alias oracle. - - Returns a :class:`ScalpelAliasOracle` built on the vendored, typed_ast-free - Scalpel slice (``codeanalyzer.dataflow.scalpel``) — the default L4 oracle. - Falls back to :class:`TypeBasedAliasOracle` only when the per-callable - Scalpel build fails on this AST. Never raises. - """ - fallback = TypeBasedAliasOracle(base_types) - try: - return ScalpelAliasOracle.from_function( - func_ast, base_types=base_types, fallback=fallback - ) - except Exception: - _note_fallback("scalpel alias build failed") - logger.debug("scalpel alias oracle build error", exc_info=True) - return fallback -``` - -(The `except ImportError: _note_fallback("python-scalpel not installed"); return fallback` branch is removed — the vendored import cannot be absent. Keep the per-query `self._fallback.may_alias(...)` inside `may_alias` unchanged.) - -- [ ] **Step 5: Run the oracle tests — now green** - -Run: `poetry run python -m pytest test/test_vendored_scalpel.py -q` -Expected: PASS (scalpel is now the default oracle; determinism holds). - -- [ ] **Step 6: L4 regression — the live acceptance test (scalpel-L4 on 3.14)** - -Run: `poetry run python -m pytest test/test_v2_l4.py test/test_v2_l4_summary.py test/test_dataflow_sdg.py test/test_dataflow_defuse.py -q` -Expected: PASS. These now exercise the vendored **scalpel** path (previously the type-based fallback on 3.14). If a case fails, investigate: a genuine scalpel-vs-type-based points-to difference is expected and the test's expectation may need updating to the (correct, tighter) scalpel result — but confirm it's a precision tightening, not a regression, before changing any assertion. If unsure, report rather than edit the assertion. - -- [ ] **Step 7: Commit** - -```bash -git add codeanalyzer/dataflow/scalpel_oracle.py test/test_vendored_scalpel.py -git commit -m "feat(dataflow): make vendored scalpel the default L4 alias oracle" -``` - ---- - -### Task 3: Fidelity check + documentation - -**Files:** -- Test: `test/test_vendored_scalpel.py` (append the fidelity check) -- Modify: `CLAUDE.md`, `.claude/SCHEMA_DECISIONS.md`, `CHANGELOG.md` - -**Interfaces:** -- Consumes: the vendored slice (Task 1) and the rewired oracle (Task 2). Produces no new code interface. - -- [ ] **Step 1: Add the vendored-vs-upstream fidelity test (skipif no pip scalpel)** - -Append to `test/test_vendored_scalpel.py`: - -```python -import pytest - - -def _pip_scalpel_available(): - try: - import importlib.util - # the EXTERNAL package, not our vendored copy - return importlib.util.find_spec("scalpel") is not None and \ - not importlib.util.find_spec("scalpel").origin.endswith( - "codeanalyzer/dataflow/scalpel/__init__.py") - except Exception: - return False - - -@pytest.mark.skipif(not _pip_scalpel_available(), - reason="upstream python-scalpel not installed (3.12+ can't build typed_ast)") -def test_vendored_ssa_matches_upstream(): - """On a Python where pip python-scalpel installs (<=3.11), the vendored copy - must produce identical SSA const_dict keys as upstream on the same source.""" - import scalpel.SSA.const as up_ssa - import scalpel.cfg as up_cfg - from codeanalyzer.dataflow.scalpel.SSA.const import SSA as VSSA - from codeanalyzer.dataflow.scalpel.cfg import CFGBuilder as VCFG - - src = "def f(a):\n b = a\n if a:\n b = 2\n return b\n" - up_c = up_cfg.CFGBuilder().build_from_src("m", src) - v_c = VCFG().build_from_src("m", src) - up_fn = list(up_c.functioncfgs.values())[0] - v_fn = list(v_c.functioncfgs.values())[0] - _, up_const = up_ssa.SSA().compute_SSA(up_fn) - _, v_const = VSSA().compute_SSA(v_fn) - assert sorted(map(str, up_const)) == sorted(map(str, v_const)) -``` - -- [ ] **Step 2: Run the fidelity test (skips on 3.14; that's expected)** - -Run: `poetry run python -m pytest test/test_vendored_scalpel.py -q` -Expected: PASS with the fidelity case SKIPPED on the 3.14 dev env (pip scalpel not installable). - -- [ ] **Step 3: Update `CLAUDE.md`** - -In `CLAUDE.md`, find the "**L4 points-to oracle = Scalpel.**" bullet and replace its "optional dependency … sanctioned fallback, not the shipping default" framing. Change the sentence: - -> `python-scalpel` is an **optional** dependency: if it is absent or a build/query fails, the analyzer falls back to the total `TypeBasedAliasOracle` (`alias.py`) and degrades, never raising. The type-based oracle is the sanctioned fallback, not the shipping default. - -to: - -> Scalpel is **vendored** (`codeanalyzer/dataflow/scalpel/`, a `typed_ast`-free 9-module slice of `python-scalpel 1.0b0`, Apache-2.0) so it is the **shipping default** L4 oracle on every supported Python — there is no external `python-scalpel`/`typed_ast` dependency. `TypeBasedAliasOracle` (`alias.py`) is retained only as the runtime safety net: on a per-callable Scalpel build failure or a per-query unresolved access path, `may_alias` degrades to it, never raising. - -- [ ] **Step 4: Update `.claude/SCHEMA_DECISIONS.md`** - -Append a follow-up note to the "## Stage 0 — Scalpel oracle spike" section: - -```markdown -**Follow-up (2026-07-22): Scalpel vendored, now the default.** The Stage-0 -"dependency hygiene" concern proved fatal for a hard dependency: `python-scalpel` -drags `typed_ast`, which has no wheel for Python 3.12+ and does not build from -source there, so `pip install python-scalpel` fails on 3.12/3.13/3.14. Since -`typed_ast` is imported only by `scalpel/typeinfer` (unused here), the 9-module -`SSA`/`cfg`/`core` slice the oracle loads was **vendored** into -`codeanalyzer/dataflow/scalpel/` (Apache-2.0, verbatim but for a `graphviz`-lazy -patch). `ScalpelAliasOracle` is now the shipping default on all supported Python; -`TypeBasedAliasOracle` is the runtime safety net only. See -`docs/superpowers/specs/2026-07-22-vendored-scalpel-default-oracle-design.md`. -``` - -- [ ] **Step 5: Update `CHANGELOG.md`** - -Under the `## [Unreleased]` heading, add: - -```markdown -### Changed -- The Scalpel-backed L4 points-to oracle is now **vendored** (`typed_ast`-free) - and the default on all supported Python (3.9–3.14); `python-scalpel` is no - longer an optional dependency and the `[scalpel]` extra is removed. On Python - 3.12+ (and anywhere the `[scalpel]` extra was not installed), L4 - `prov:["points-to"]` data-dependence edges are now Scalpel-precise rather than - the coarser type-based over-approximation; the `prov:["ssa"]` set and the - `L3 ⊆ L4` monotonicity invariant are unchanged. Adds `astor` as a runtime - dependency. -``` - -- [ ] **Step 6: Run the full behavior-preservation gate** - -Run: `poetry run python -m pytest test/test_vendored_scalpel.py test/test_v2_l4.py test/test_v2_l4_summary.py test/test_dataflow_sdg.py test/test_dataflow_defuse.py test/test_v2_superset.py -q` -Expected: PASS (incl. the `test_l3_subset_of_l4` monotonicity gate in `test_v2_superset.py`). Note: `test_v2_superset.py` runs the CLI with `--no-venv`; it exercises the vendored scalpel on the L4 fixture. - -- [ ] **Step 7: Commit** - -```bash -git add test/test_vendored_scalpel.py CLAUDE.md .claude/SCHEMA_DECISIONS.md CHANGELOG.md -git commit -m "docs(dataflow): record vendored scalpel as the default L4 oracle" -``` - ---- - -## Self-Review - -**1. Spec coverage:** -- Vendored `codeanalyzer/dataflow/scalpel/` 9-file slice, verbatim + graphviz patch → Task 1 Steps 3–4. ✓ -- Attribution (LICENSE, README, NOTICE) → Task 1 Steps 3, 5. ✓ -- `astor` added, `[scalpel]` extra removed, `requires-python` unchanged → Task 1 Step 6. ✓ -- Oracle repointed + default (ImportError branch dropped, fallback kept) → Task 2 Steps 3–4. ✓ -- Behavior change (L4 points-to precision), monotonicity preserved → Task 2 Step 6, Task 3 Steps 5–6. ✓ -- Docs: CLAUDE.md, SCHEMA_DECISIONS, CHANGELOG → Task 3 Steps 3–5. ✓ -- Testing: import hygiene, typed_ast-free gate, default-scalpel, determinism, fidelity, L4 regression, monotonicity → Tasks 1–3. ✓ - -**2. Placeholder scan:** No TBD/TODO. The vendoring uses exact `cp` commands and an explicit file list; the graphviz patch shows the exact before/after; every doc edit gives the exact old→new text. - -**3. Type consistency:** `make_alias_oracle(pycallable, func_ast, base_types)` signature matches the existing call site in `core`/the pipeline. Import paths (`codeanalyzer.dataflow.scalpel.SSA.const`, `...cfg`) are consistent across Task 1 (creation), Task 2 (oracle import), and the tests. `ScalpelAliasOracle`/`TypeBasedAliasOracle` names match `scalpel_oracle.py`/`alias.py`. From 08ab9715cbc0fc975de7f90e16b228f8050435af Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Mon, 27 Jul 2026 11:37:40 -0400 Subject: [PATCH 18/22] chore(release): 1.1.0 --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e64c020..f15c77f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.0] - 2026-07-27 + ### Changed - The Scalpel-backed L4 points-to oracle is now **vendored** (`typed_ast`-free) and the default on all supported Python (3.9–3.14); `python-scalpel` is no diff --git a/pyproject.toml b/pyproject.toml index d36dece..eada73d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codeanalyzer-python" -version = "1.0.3" +version = "1.1.0" description = "Static analysis for Python — canonical schema v2 (symbol table, call graph, and native CFG/PDG/SDG dataflow) as analysis.json or a Neo4j property graph." readme = "README.md" authors = [ From 1eaa2bc78761bb25bdadf657e6653b9972359cd5 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Mon, 27 Jul 2026 15:08:52 -0400 Subject: [PATCH 19/22] fix(dataflow): connect the L4 SDG port layer to the statement ddg (#115) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDG assembler wires the binding edges — def stmt → actual_in, actual_out → callsite, formal_in → first use, return → formal_out — into the IR's extra_edges, and the old v1 program_graphs projection emitted them; the v2 emission dropped them, leaving the port lattice an island no end-to-end flows_to walk could cross. emit_l4 now emits the DDG-typed extra edges onto each callable's ddg tagged prov=['reaching-defs'] (the label codeanalyzer-typescript ships for its port-routing edges, keeping the prov vocabulary keystone-shared), deduplicated, deterministically ordered, endpoint-guarded, and idempotent under cache reuse. CDG-typed extras stay unemitted — actual vertices already carry that containment in parent. Nested call vertices (y = f(x)) are likewise anchored: from L3 they carry parent = the enclosing statement's local id, a sanctioned null → value refinement at L2→L3 mirroring callee null → id at L1→L2. Bare-call statements share their key with the CFG node and are untouched. Conformance now admits the third L4 prov value; both decisions are recorded in .claude/SCHEMA_DECISIONS.md. --- .claude/SCHEMA_DECISIONS.md | 28 +++++ CHANGELOG.md | 16 +++ codeanalyzer/dataflow/builder.py | 59 ++++++++- test/conftest_v2.py | 14 ++- test/test_v2_l4_ports.py | 200 +++++++++++++++++++++++++++++++ 5 files changed, 310 insertions(+), 7 deletions(-) create mode 100644 test/test_v2_l4_ports.py diff --git a/.claude/SCHEMA_DECISIONS.md b/.claude/SCHEMA_DECISIONS.md index 12dc408..c6a45d0 100644 --- a/.claude/SCHEMA_DECISIONS.md +++ b/.claude/SCHEMA_DECISIONS.md @@ -202,3 +202,31 @@ MERGE collapses legitimately-distinct edges (per-variable dependences; a conditional's true/false pair) and a live Bolt push then materializes fewer relationships than the projection produced (caught by the opt-in `test_neo4j_bolt.py` count gates). + +## L4 graph completeness — port wiring + call anchoring (#115, 1.1.1) + +Two connectivity gaps closed in the L4 emission (no vocabulary invented; both +decisions use surface the keystone already ships): + +1. **Statement ↔ port ddg wiring, `prov:["reaching-defs"]`.** The SDG's + binding edges — `def stmt → actual_in`, `actual_out → callsite`, + `formal_in → first use`, `def/return → formal_out` — existed in the IR + (`fg.extra_edges`, wired by `assemble_sdg`) and were emitted by the old v1 + `program_graphs` projection, but the v2 emission dropped them, leaving the + port lattice an island (no end-to-end `flows_to` witness could cross a + call). `emit_l4` now emits them onto each callable's `ddg` tagged + `prov:["reaching-defs"]` — the label codeanalyzer-typescript already ships + for its port-routing edges, so the prov vocabulary stays keystone-shared: + `ssa` (L3 syntactic) ⊂ + `points-to` (L4 alias delta) + `reaching-defs` + (L4 port bindings). Monotonicity: both L4 families are additive over the + untouched ssa set, and every port endpoint exists only at L4. +2. **Call vertices anchor via `parent`, not the CFG spine.** A call nested in + a larger statement (`y = f(x)`) keeps its own `"line:col"` body key and + deliberately stays OFF the cfg spine — calls are dataflow satellites of + their statement, not control-flow steps. From L3 (when statements exist) + such a call carries `parent` = its enclosing statement's local id — the + same anchoring `actual_in`/`actual_out` vertices already use. A bare-call + statement shares its key with the CFG node (no self-parent). This is a + sanctioned `null → value` refinement of `BodyNode.parent` at the L2→L3 + boundary, mirroring the `callee: null → id` refinement at L1→L2; the + superset gates compare body keys, so no gate exception was needed. diff --git a/CHANGELOG.md b/CHANGELOG.md index f15c77f..06a2c2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- **L4 SDG port layer is connected to the statement ddg** (#115): the binding + edges `def stmt → actual_in`, `actual_out → callsite`, `formal_in → first use` + and `return → formal_out` were built by the SDG assembler but dropped by the + v2 emission, leaving the interprocedural port lattice an island — no + end-to-end `flows_to(def, callee_formal)` path could cross a call. They are + now emitted on each callable's `ddg` with `prov:["reaching-defs"]` (the same + label codeanalyzer-typescript ships for its port-routing edges). Strictly + additive over the L3 `ssa` set, so `L3 ⊆ L4` monotonicity holds. +- **Nested call vertices are anchored to their statement** (#115): a call inside + a larger statement (`y = f(x)`) sits off the CFG spine by design (a dataflow + satellite); from L3 it now carries `parent` = its enclosing statement's local + id — the same anchoring `actual_in`/`actual_out` vertices use. Sanctioned + `null → value` refinement at L2→L3, mirroring `callee: null → id` at L1→L2; + recorded in `.claude/SCHEMA_DECISIONS.md`. + ## [1.1.0] - 2026-07-27 ### Changed diff --git a/codeanalyzer/dataflow/builder.py b/codeanalyzer/dataflow/builder.py index 6e58bd8..844910a 100644 --- a/codeanalyzer/dataflow/builder.py +++ b/codeanalyzer/dataflow/builder.py @@ -283,6 +283,23 @@ def _span_of(source: str, node) -> Optional["Span"]: continue pycallable.body[local] = BodyNode(kind=node.kind, span=span) + # #115: anchor nested call vertices to their statement. A bare-call + # statement shares its key with its CFG node (handled above); a call + # nested inside a larger statement (`y = f(x)`) has its own key and + # no cfg contact, so it carries `parent` = the enclosing statement's + # local id — the same anchoring actual_in/actual_out vertices use. + for node in pdg.cfg.nodes: + if node.ast_node is None: + continue + stmt_local = im.local(node.id) + for call in _calls_in(node.ast_node): + call_key = f"{call.lineno}:{call.col_offset}" + child = pycallable.body.get(call_key) + if child is None or call_key == stmt_local: + continue + if child.kind == "call": + child.parent = stmt_local + if want_cfg: pycallable.cfg = [ CfgEdge( @@ -437,7 +454,12 @@ def emit_l4( points-to provenance and taint are *not* emitted here (later tasks). """ from codeanalyzer.dataflow.identity import IdentityMap - from codeanalyzer.schema.py_schema import BodyNode, ParamEdge, SummaryEdge + from codeanalyzer.schema.py_schema import ( + BodyNode, + DdgEdge, + ParamEdge, + SummaryEdge, + ) # L4 emission is additive (it *appends* summary/param edges), so it must # first clear any L4 state a reused cache left on these live objects — @@ -517,6 +539,41 @@ def emit_l4( ) (app.param_in if e.type == "PARAM_IN" else app.param_out).append(edge) + # (e) statement ↔ port ddg wiring (#115). The IR's ``extra_edges`` — the + # def→actual_in, actual_out→callsite, formal_in→use and def→formal_out + # bindings ``assemble_sdg`` wires — are what connect the port lattice to + # the statement-level ddg; without them the SDG is two disconnected + # graphs and no end-to-end flows_to walk can cross a call. Emitted with + # ``prov=["reaching-defs"]`` (the label codeanalyzer-typescript ships for + # its port-routing ddg edges, so the vocabulary stays keystone-shared). + # CDG-typed extras (callsite → actual_in containment) are skipped — the + # actual vertices already carry that anchoring in ``parent``. + for sig, fg in ir.functions.items(): + pycallable = sig_to_callable.get(sig) + im = ims.get(sig) + if pycallable is None or im is None: + continue + # Idempotency under cache reuse, mirroring the points-to delta: strip + # any reaching-defs edges a prior run appended before re-emitting. + pycallable.ddg = [e for e in pycallable.ddg if e.prov != ["reaching-defs"]] + seen: set = set() + rows = [] + for e in fg.extra_edges: + if e.type != "DDG": + continue + src, dst = im.local(e.source), im.local(e.target) + if src not in pycallable.body or dst not in pycallable.body: + continue + key = (src, dst, e.var) + if key in seen: + continue + seen.add(key) + rows.append( + DdgEdge(src=src, dst=dst, var=e.var, prov=["reaching-defs"]) + ) + rows.sort(key=lambda r: (r.src, r.dst, r.var or "")) + pycallable.ddg.extend(rows) + def _ddg_local_set(im, pdg) -> Set[Tuple[str, str, Optional[str]]]: """The DDG edges of ``pdg`` as a set of ``(local_src, local_dst, var)``. diff --git a/test/conftest_v2.py b/test/conftest_v2.py index d0b099e..e1ac9a4 100644 --- a/test/conftest_v2.py +++ b/test/conftest_v2.py @@ -68,14 +68,16 @@ def assert_conformant(payload: dict, max_level: int) -> None: f"L3 ddg edge must have prov ['ssa'], got {e.get('prov')} in {c['id']}" ) elif max_level >= 4: - # L4 layers an alias-derived (points-to) def-use delta additively on top - # of the unchanged L3 ssa edges, so every ddg edge carries exactly one of - # those two provenances — and no other. + # L4 layers two additive deltas on the unchanged L3 ssa edges: the + # alias-derived points-to def-use delta, and the statement ↔ port + # binding edges (#115) tagged reaching-defs — the keystone-shared label + # codeanalyzer-typescript also emits for its port-routing edges. Every + # ddg edge carries exactly one of those three provenances — no other. for mod, c in _iter_callables(app): for e in c.get("ddg", []): - assert e.get("prov") in (["ssa"], ["points-to"]), ( - f"L4 ddg edge must have prov ['ssa'] or ['points-to'], " - f"got {e.get('prov')} in {c['id']}" + assert e.get("prov") in (["ssa"], ["points-to"], ["reaching-defs"]), ( + f"L4 ddg edge must have prov ['ssa'], ['points-to'] or " + f"['reaching-defs'], got {e.get('prov')} in {c['id']}" ) if max_level >= 4: diff --git a/test/test_v2_l4_ports.py b/test/test_v2_l4_ports.py new file mode 100644 index 0000000..ea67d88 --- /dev/null +++ b/test/test_v2_l4_ports.py @@ -0,0 +1,200 @@ +"""Regression tests for #115: the L4 SDG port layer must be *connected* to the +statement-level ddg, and call vertices must be anchored to their statement. + +Before the fix, the interprocedural port lattice (``actual_in → formal_in``, +``formal_out → actual_out`` via param_in/param_out/summary) was an island: no +ddg edge touched any port, so an end-to-end ``flows_to(def, callee_formal)`` +walk was inexpressible. The wiring existed in the IR (``fg.extra_edges``, +emitted by the old v1 ``program_graphs`` projection) but the v2 emission +dropped it. The restored edges carry ``prov=["reaching-defs"]`` — the same +label codeanalyzer-typescript ships for its port-routing ddg edges. +""" +from pathlib import Path + +from codeanalyzer.dataflow.builder import ( + _base_types, + build_function_pdgs, + build_program_graphs, + emit_ddg_pointsto_delta, + emit_l3_body, + emit_l4, +) +from codeanalyzer.dataflow.scalpel_oracle import make_alias_oracle +from codeanalyzer.dataflow.syntactic import SyntacticOracle +from codeanalyzer.schema import PyApplication +from codeanalyzer.schema.assign_ids import assign_ids +from codeanalyzer.schema.l1_body import populate_l1_body +from codeanalyzer.schema.py_schema import PyCallEdge +from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder + +_SOURCE = """\ +def build(flag): + result = flag + 1 + return result + + +def main(): + x = 5 + y = build(x) + z = y * 2 + return z +""" + + +def _build_l4_app(tmp_path: Path): + f = tmp_path / "app.py" + f.write_text(_SOURCE, encoding="utf-8") + mod = SymbolTableBuilder(tmp_path, None).build_pymodule_from_file(f) + app = PyApplication(symbol_table={"app.py": mod}) + sig_to_id = assign_ids(app, "portfix") + app.call_graph = [ + PyCallEdge(src="app.main", dst="app.build", prov=["jedi"], weight=1) + ] + populate_l1_body(app) + syn_infos, _ = build_function_pdgs( + app, k=3, oracle_factory=lambda c, fast: SyntacticOracle() + ) + emit_l3_body(app, syn_infos, sig_to_id, graphs={"cfg", "dfg", "pdg"}) + ir = build_program_graphs( + app, k=3, + oracle_factory=lambda c, fast: make_alias_oracle(c, fast, _base_types(c)), + ) + emit_l4(app, ir, sig_to_id) + emit_ddg_pointsto_delta(app, syn_infos, ir, sig_to_id) + mod = app.symbol_table["app.py"] + return app, mod.functions["build"], mod.functions["main"] + + +def _edges(c, prov=None): + out = set() + for e in c.ddg or []: + if prov is None or e.prov == prov: + out.add((e.src, e.dst, e.var)) + return out + + +def test_def_stmt_flows_into_actual_in(tmp_path): + """`x = 5` (7:4) must feed the argument port of the `build(x)` callsite.""" + _, _, main = _build_l4_app(tmp_path) + rd = _edges(main, prov=["reaching-defs"]) + assert any( + src == "7:4" and dst.endswith("/actual_in:0") for src, dst, _ in rd + ), f"missing def→actual_in binding edge; reaching-defs edges: {sorted(rd)}" + + +def test_actual_out_flows_back_to_callsite(tmp_path): + """The return-value port must flow back into the callsite statement.""" + _, _, main = _build_l4_app(tmp_path) + rd = _edges(main, prov=["reaching-defs"]) + assert any( + src.endswith("/actual_out") and dst == "8:4" for src, dst, _ in rd + ), f"missing actual_out→use binding edge; reaching-defs edges: {sorted(rd)}" + + +def test_formal_in_flows_to_first_use(tmp_path): + """Inside the callee, the parameter port must reach its first-use stmt.""" + _, build, _ = _build_l4_app(tmp_path) + rd = _edges(build, prov=["reaching-defs"]) + assert any( + src == "@formal_in:0" and dst == "2:4" for src, dst, _ in rd + ), f"missing formal_in→use edge; reaching-defs edges: {sorted(rd)}" + + +def test_return_stmt_flows_into_formal_out(tmp_path): + """`return result` (3:4) must feed the callee's formal_out port.""" + _, build, _ = _build_l4_app(tmp_path) + rd = _edges(build, prov=["reaching-defs"]) + assert any( + src == "3:4" and dst == "@formal_out" for src, dst, _ in rd + ), f"missing return→formal_out edge; reaching-defs edges: {sorted(rd)}" + + +def test_port_edges_never_replace_or_retag_l3_edges(tmp_path): + """The wiring is additive: every ssa edge survives, and no reaching-defs + edge duplicates an (src, dst, var) triple that ssa already carries.""" + _, build, main = _build_l4_app(tmp_path) + for c in (build, main): + ssa = _edges(c, prov=["ssa"]) + rd = _edges(c, prov=["reaching-defs"]) + assert ssa, "L3 ssa edges must still be present" + assert not (ssa & rd), "reaching-defs must not duplicate ssa triples" + + +def test_port_edge_endpoints_exist_in_body(tmp_path): + _, build, main = _build_l4_app(tmp_path) + for c in (build, main): + for e in c.ddg or []: + assert e.src in c.body, f"dangling ddg src {e.src}" + assert e.dst in c.body, f"dangling ddg dst {e.dst}" + + +def test_emission_is_idempotent_under_reemit(tmp_path): + """Re-running emit_l4 + the delta against the same live tree (cache-reuse + shape) must not duplicate the reaching-defs edges.""" + f = tmp_path / "app.py" + f.write_text(_SOURCE, encoding="utf-8") + mod = SymbolTableBuilder(tmp_path, None).build_pymodule_from_file(f) + app = PyApplication(symbol_table={"app.py": mod}) + sig_to_id = assign_ids(app, "portfix") + app.call_graph = [ + PyCallEdge(src="app.main", dst="app.build", prov=["jedi"], weight=1) + ] + populate_l1_body(app) + syn_infos, _ = build_function_pdgs( + app, k=3, oracle_factory=lambda c, fast: SyntacticOracle() + ) + emit_l3_body(app, syn_infos, sig_to_id, graphs={"cfg", "dfg", "pdg"}) + ir = build_program_graphs( + app, k=3, + oracle_factory=lambda c, fast: make_alias_oracle(c, fast, _base_types(c)), + ) + emit_l4(app, ir, sig_to_id) + emit_ddg_pointsto_delta(app, syn_infos, ir, sig_to_id) + main = app.symbol_table["app.py"].functions["main"] + first = sorted((e.src, e.dst, e.var, tuple(e.prov)) for e in main.ddg) + emit_l4(app, ir, sig_to_id) + emit_ddg_pointsto_delta(app, syn_infos, ir, sig_to_id) + main = app.symbol_table["app.py"].functions["main"] + second = sorted((e.src, e.dst, e.var, tuple(e.prov)) for e in main.ddg) + assert first == second, "re-emission must be idempotent" + + +# ---------------------------------------------------------------------------------------------- +# #115 part 2: call vertices anchored to their statement via `parent`. +# ---------------------------------------------------------------------------------------------- + + +def test_nested_call_vertex_is_parented_to_its_statement(tmp_path): + """`y = build(x)`: the call vertex (8:8) floats off the CFG spine; from L3 + it must carry `parent` = its enclosing statement's local (8:4).""" + _, _, main = _build_l4_app(tmp_path) + call_nodes = {k: n for k, n in main.body.items() if n.kind == "call"} + assert call_nodes, "fixture must materialize a call vertex" + for key, node in call_nodes.items(): + assert node.parent == "8:4", ( + f"call vertex {key} must be parented to its statement, " + f"got parent={node.parent!r}" + ) + + +def test_bare_call_statement_needs_no_parent(tmp_path): + """A bare call (`g(b)`) shares its key with the statement node — no + self-parent is emitted.""" + f = tmp_path / "m.py" + f.write_text( + "def g(x):\n return x\n\n\ndef f(a):\n g(a)\n return a\n", + encoding="utf-8", + ) + mod = SymbolTableBuilder(tmp_path, None).build_pymodule_from_file(f) + app = PyApplication(symbol_table={"m.py": mod}) + sig_to_id = assign_ids(app, "barefix") + populate_l1_body(app) + syn_infos, _ = build_function_pdgs( + app, k=3, oracle_factory=lambda c, fast: SyntacticOracle() + ) + emit_l3_body(app, syn_infos, sig_to_id, graphs={"cfg", "dfg", "pdg"}) + fcallable = app.symbol_table["m.py"].functions["f"] + call_nodes = {k: n for k, n in fcallable.body.items() if n.kind == "call"} + assert call_nodes, "fixture must materialize the bare call vertex" + for key, node in call_nodes.items(): + assert node.parent != key, "a call must never parent to itself" From baaa418efcc2cb974364d027ee2e2593ca4f9315 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Mon, 27 Jul 2026 15:09:39 -0400 Subject: [PATCH 20/22] chore(release): 1.1.1 --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06a2c2b..7d1790c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.1] - 2026-07-27 + ### Fixed - **L4 SDG port layer is connected to the statement ddg** (#115): the binding edges `def stmt → actual_in`, `actual_out → callsite`, `formal_in → first use` diff --git a/pyproject.toml b/pyproject.toml index eada73d..79b8c70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codeanalyzer-python" -version = "1.1.0" +version = "1.1.1" description = "Static analysis for Python — canonical schema v2 (symbol table, call graph, and native CFG/PDG/SDG dataflow) as analysis.json or a Neo4j property graph." readme = "README.md" authors = [ From 8b376b41c7fda6ec307dd9381d89cc2d3a164901 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Tue, 28 Jul 2026 11:19:40 -0400 Subject: [PATCH 21/22] feat(cli)!: drop msgpack output; enforce --emit neo4j full-depth contract (#118, #119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit msgpack (#118, TS parity): remove the --format msgpack choice, the analysis.msgpack emit path, the msgpack serialization mixin on schema models, and the msgpack dependency. analysis.json is the single wire format; --format json is unchanged. neo4j gate (#119): --emit neo4j documented itself as always-full-depth but neither rejected -a/--graphs nor forced the depth — it silently emitted a partial graph at the default level 1. It now runs at level 4 with every graph section, and explicitly passing -a/--graphs alongside it is an error. -a and --graphs use None sentinels so explicit flags are distinguishable from defaults (click's parameter-source API misreports under the Typer group context), which also closes a latent hole where an explicitly-passed default --graphs value slipped the level-3 validation. --- CHANGELOG.md | 14 +++++ CLAUDE.md | 4 +- README.md | 5 +- codeanalyzer/__main__.py | 64 ++++++++++++++------ codeanalyzer/config/config.py | 1 - codeanalyzer/options/options.py | 1 - codeanalyzer/schema/py_schema.py | 101 ------------------------------- pyproject.toml | 3 - test/test_cli_emit_gates.py | 97 +++++++++++++++++++++++++++++ 9 files changed, 160 insertions(+), 130 deletions(-) create mode 100644 test/test_cli_emit_gates.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d1790c..e8d462a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- **BREAKING: the msgpack output format is removed** (#118, TS parity): the + `--format msgpack` CLI choice, the `analysis.msgpack` artifact, the msgpack + serialization mixin on schema models, and the `msgpack` dependency are gone. + `analysis.json` is the single wire format; anyone passing `--format msgpack` + must drop the flag. `--format json` still works unchanged. +- **`--emit neo4j` now enforces its always-full-depth contract** (#119): it + runs at level 4 with every graph section regardless of defaults, and + explicitly passing `-a`/`--graphs` alongside it is now the documented + explicit error (previously accepted silently — and worse, the emission + actually ran at the default level 1, producing a partial graph). Passing + `--graphs` below `-a 3` is also now consistently rejected even when the + value equals the default. + ## [1.1.1] - 2026-07-27 ### Fixed diff --git a/CLAUDE.md b/CLAUDE.md index 6816472..941824c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -135,8 +135,8 @@ declared callables by their `can://` tree id, imported/builtin targets by a ### Two projections -1. **`analysis.json`** — the tree itself (the `Analysis` envelope above), also - available as gzip'd msgpack. +1. **`analysis.json`** — the tree itself (the `Analysis` envelope above); the + single wire format (the msgpack variant was removed in 1.2.0, #118). 2. **Neo4j** (`codeanalyzer/neo4j/`) — a near-identity projection, keyed on the same `can://` / global ordinal ids: containment → typed `PY_HAS_*` / `PY_DECLARES` edges (`PY_HAS_MODULE`, `PY_DECLARES`, `PY_HAS_METHOD`, diff --git a/README.md b/README.md index 78fccd5..ebbc062 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ needs. checked in as `schema.neo4j.json` (`2.0.0`) and shipped with every release. - **Incremental cache** — per-file results are cached under `.codeanalyzer`; `--lazy` (default) reuses them, `--eager` forces a clean rebuild. `--ray` distributes the work across cores. -- **Compact output** — canonical `analysis.json`, or binary `analysis.msgpack` for smaller artifacts. +- **Compact output** — one canonical `analysis.json` per run. ## Installation @@ -143,8 +143,7 @@ canpy --input /path/to/python/project ``` With no `--output`, the analysis is printed to stdout as compact JSON; with `--output ` it is -written to `analysis.json` (or `graph.cypher` for `--emit neo4j`, or `analysis.msgpack` with -`--format msgpack`) in that directory. +written to `analysis.json` (or `graph.cypher` for `--emit neo4j`) in that directory. ### Options diff --git a/codeanalyzer/__main__.py b/codeanalyzer/__main__.py index 3f8ae54..c98a620 100644 --- a/codeanalyzer/__main__.py +++ b/codeanalyzer/__main__.py @@ -85,7 +85,7 @@ def main( typer.Option( "-f", "--format", - help="Output format for --emit json: json or msgpack.", + help="Output format for --emit json: json.", case_sensitive=False, ), ] = OutputFormat.JSON, @@ -141,26 +141,31 @@ def main( ), ] = None, analysis_level: Annotated[ - int, + Optional[int], typer.Option( "-a", "--analysis-level", help="Analysis depth: 1=symbol table+Jedi call graph, 2=+PyCG call " "graph, 3=+native intraprocedural dataflow (CFG/PDG), " - "4=+interprocedural SDG (param/summary edges, alias-aware DDG).", + "4=+interprocedural SDG (param/summary edges, alias-aware DDG). " + "[default: 1; incompatible with --emit neo4j, which is always " + "full-depth]", min=1, max=4, + show_default="1", ), - ] = 1, + ] = None, graphs: Annotated[ - str, + Optional[str], typer.Option( "--graphs", help="Level 3+ only: comma-separated program-graph sections to emit " "(cfg, dfg, pdg, sdg). Default: cfg,dfg,pdg. `dfg` emits the PDG's data " - "edges only; `sdg` requires -a 4.", + "edges only; `sdg` requires -a 4. Incompatible with --emit neo4j " + "(always full-depth).", + show_default="cfg,dfg,pdg", ), - ] = "cfg,dfg,pdg", + ] = None, graph_field_depth: Annotated[ int, typer.Option( @@ -300,9 +305,40 @@ def main( _pin_hash_seed() # Flag validation (strict: unrecognized values error out, never fall back). - selected_graphs = [g.strip() for g in graphs.split(",") if g.strip()] + # -a and --graphs use None sentinels so an explicitly-passed flag is + # distinguishable from the default (#119). + explicit_level = analysis_level is not None + explicit_graphs = graphs is not None + + # Neo4j is always full-depth (#119): the graph carries every level's + # facts, so depth/section selectors cannot be combined with it — reject + # explicitly-passed flags and force level 4 with every graph section. from codeanalyzer.dataflow.builder import VALID_GRAPHS + if emit == EmitTarget.NEO4J: + explicit = [ + flag + for flag, was_explicit in ( + ("-a/--analysis-level", explicit_level), + ("--graphs", explicit_graphs), + ) + if was_explicit + ] + if explicit: + logger.error( + "--emit neo4j is always full-depth (level 4, all graph " + f"sections); {' and '.join(explicit)} cannot be combined with it." + ) + raise typer.Exit(code=2) + analysis_level = 4 + graphs = ",".join(VALID_GRAPHS) + + if analysis_level is None: + analysis_level = 1 + if graphs is None: + graphs = "cfg,dfg,pdg" + selected_graphs = [g.strip() for g in graphs.split(",") if g.strip()] + unknown_graphs = [g for g in selected_graphs if g not in VALID_GRAPHS] if unknown_graphs: logger.error( @@ -316,7 +352,7 @@ def main( if "sdg" in selected_graphs and analysis_level < 4: logger.error("--graphs sdg requires -a 4 (interprocedural SDG).") raise typer.Exit(code=2) - if analysis_level < 3 and graphs != "cfg,dfg,pdg": + if analysis_level < 3 and explicit_graphs: logger.error("--graphs is a level-3 option; pass -a 3 to emit program graphs.") raise typer.Exit(code=2) if analysis_level < 3 and graph_field_depth != 3: @@ -408,16 +444,6 @@ def _write_output(artifacts, output_dir: Path, format: OutputFormat): f.write(json_str) logger.info(f"Analysis saved to {output_file}") - elif format == OutputFormat.MSGPACK: - output_file = output_dir / "analysis.msgpack" - msgpack_data = artifacts.to_msgpack_bytes() - with output_file.open("wb") as f: - f.write(msgpack_data) - logger.info(f"Analysis saved to {output_file}") - logger.info( - f"Compression ratio: {artifacts.get_compression_ratio():.1%} of JSON size" - ) - app = typer.Typer( callback=main, diff --git a/codeanalyzer/config/config.py b/codeanalyzer/config/config.py index 70bbf81..e6aea0c 100644 --- a/codeanalyzer/config/config.py +++ b/codeanalyzer/config/config.py @@ -5,4 +5,3 @@ class OutputFormat(str, Enum): """String-based enum for output formats to support typer case-insensitive options.""" JSON = "json" - MSGPACK = "msgpack" diff --git a/codeanalyzer/options/options.py b/codeanalyzer/options/options.py index bf0feab..1be9ec4 100644 --- a/codeanalyzer/options/options.py +++ b/codeanalyzer/options/options.py @@ -6,7 +6,6 @@ class OutputFormat(str, Enum): JSON = "json" - MSGPACK = "msgpack" class EmitTarget(str, Enum): diff --git a/codeanalyzer/schema/py_schema.py b/codeanalyzer/schema/py_schema.py index 25cd45f..2d2ee98 100644 --- a/codeanalyzer/schema/py_schema.py +++ b/codeanalyzer/schema/py_schema.py @@ -22,85 +22,8 @@ from __future__ import annotations from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -import gzip - from pydantic import BaseModel from typing_extensions import Literal -import msgpack - - -def msgpk(cls): - """ - Decorator that adds MessagePack serialization methods to Pydantic models. - - Adds methods: - - to_msgpack_bytes() -> bytes: Serialize to compact binary format - - from_msgpack_bytes(data: bytes) -> cls: Deserialize from binary format - - to_msgpack_dict() -> dict: Convert to msgpack-compatible dict - - from_msgpack_dict(data: dict) -> cls: Create instance from msgpack dict - """ - - def _prepare_for_serialization(obj: Any) -> Any: - """Convert objects to serialization-friendly format.""" - if isinstance(obj, Path): - return str(obj) - elif isinstance(obj, dict): - return { - _prepare_for_serialization(k): _prepare_for_serialization(v) - for k, v in obj.items() - } - elif isinstance(obj, list): - return [_prepare_for_serialization(item) for item in obj] - elif isinstance(obj, tuple): - return tuple(_prepare_for_serialization(item) for item in obj) - elif isinstance(obj, set): - return [_prepare_for_serialization(item) for item in obj] - elif hasattr(obj, "model_dump"): # Pydantic model - return _prepare_for_serialization(obj.model_dump()) - else: - return obj - - def to_msgpack_bytes(self) -> bytes: - """Serialize the model to compact binary format using MessagePack + gzip.""" - data = _prepare_for_serialization(self.model_dump()) - msgpack_data = msgpack.packb(data, use_bin_type=True) - return gzip.compress(msgpack_data) - - @classmethod - def from_msgpack_bytes(cls_obj, data: bytes): - """Deserialize from MessagePack + gzip binary format.""" - decompressed_data = gzip.decompress(data) - obj_dict = msgpack.unpackb(decompressed_data, raw=False) - return cls_obj.model_validate(obj_dict) - - def to_msgpack_dict(self) -> dict: - """Convert to msgpack-compatible dictionary format.""" - return _prepare_for_serialization(self.model_dump()) - - @classmethod - def from_msgpack_dict(cls_obj, data: dict): - """Create instance from msgpack-compatible dictionary.""" - return cls_obj.model_validate(data) - - def get_msgpack_size(self) -> int: - """Get the size of the msgpack serialization in bytes.""" - return len(self.to_msgpack_bytes()) - - def get_compression_ratio(self) -> float: - """Get compression ratio compared to JSON.""" - json_size = len(self.model_dump_json().encode("utf-8")) - msgpack_gzip_size = self.get_msgpack_size() - return msgpack_gzip_size / json_size if json_size > 0 else 1.0 - - # Add methods to the class - cls.to_msgpack_bytes = to_msgpack_bytes - cls.from_msgpack_bytes = from_msgpack_bytes - cls.to_msgpack_dict = to_msgpack_dict - cls.from_msgpack_dict = from_msgpack_dict - cls.get_msgpack_size = get_msgpack_size - cls.get_compression_ratio = get_compression_ratio - - return cls def builder(cls): @@ -192,7 +115,6 @@ def offset(line: int, col: int) -> int: @builder -@msgpk class Span(BaseModel): """Where a node lives in source. `start`/`end` are [line, col] (1-based line, 0-based col, ast semantics); `bytes` are utf-8 offsets into module.source.""" @@ -202,7 +124,6 @@ class Span(BaseModel): @builder -@msgpk class BodyNode(BaseModel): """A node in a callable's `body`: an AST region (statement/call/branch/…) or a synthetic analysis vertex (entry/exit/formal_in/out/actual_in/out).""" @@ -214,37 +135,31 @@ class BodyNode(BaseModel): @builder -@msgpk class CfgEdge(BaseModel): src: str; dst: str; kind: str = "fallthrough" @builder -@msgpk class CdgEdge(BaseModel): src: str; dst: str @builder -@msgpk class DdgEdge(BaseModel): src: str; dst: str; var: Optional[str] = None; prov: List[str] = [] @builder -@msgpk class SummaryEdge(BaseModel): src: str; dst: str @builder -@msgpk class ParamEdge(BaseModel): src: str; dst: str @builder -@msgpk class PyImport(BaseModel): """Represents a Python import statement.""" @@ -259,7 +174,6 @@ class PyImport(BaseModel): @builder -@msgpk class PyComment(BaseModel): """Represents a Python comment.""" @@ -272,7 +186,6 @@ class PyComment(BaseModel): @builder -@msgpk class PySymbol(BaseModel): """Represents a symbol used or declared in Python code.""" @@ -287,7 +200,6 @@ class PySymbol(BaseModel): @builder -@msgpk class PyVariableDeclaration(BaseModel): """Represents a Python variable declaration.""" @@ -306,7 +218,6 @@ class PyVariableDeclaration(BaseModel): @builder -@msgpk class PyCallableParameter(BaseModel): """Represents a parameter of a Python callable (function/method).""" @@ -320,7 +231,6 @@ class PyCallableParameter(BaseModel): @builder -@msgpk class PyCallArgument(BaseModel): """One call-site argument: AST category + inferred type, kept separate. @@ -333,7 +243,6 @@ class PyCallArgument(BaseModel): @builder -@msgpk class PyCallsite(BaseModel): """Represents a Python call site (function or method invocation) with contextual metadata.""" @@ -352,7 +261,6 @@ class PyCallsite(BaseModel): @builder -@msgpk class PyCallable(BaseModel): """Represents a Python callable (function/method).""" @@ -389,7 +297,6 @@ def __hash__(self) -> int: @builder -@msgpk class PyClassAttribute(BaseModel): """Represents a Python class attribute.""" @@ -402,7 +309,6 @@ class PyClassAttribute(BaseModel): @builder -@msgpk class PyClass(BaseModel): """Represents a Python class.""" @@ -425,7 +331,6 @@ def __hash__(self): @builder -@msgpk class PyModule(BaseModel): """Represents a Python module.""" @@ -446,7 +351,6 @@ class PyModule(BaseModel): @builder -@msgpk class PyCallEdge(BaseModel): """Identity-only call-graph edge with weight (keystone shape: the list name IS the edge type, so there is no ``type`` field). @@ -464,7 +368,6 @@ class PyCallEdge(BaseModel): @builder -@msgpk class PyExternalSymbol(BaseModel): """A call-graph target outside the analyzed project -- an imported library or builtin member. An edge-endpoint id home, not a tree node: keyed in @@ -477,7 +380,6 @@ class PyExternalSymbol(BaseModel): @builder -@msgpk class PyRepositoryInfo(BaseModel): """Where the analyzed source came from: git provenance captured at analysis time.""" @@ -487,7 +389,6 @@ class PyRepositoryInfo(BaseModel): @builder -@msgpk class PyAnalyzerInfo(BaseModel): """Which analyzer produced this snapshot, and how it was configured. Lives on the ``Analysis`` envelope (keystone ``analyzer{name,version}``; @@ -499,7 +400,6 @@ class PyAnalyzerInfo(BaseModel): @builder -@msgpk class PyApplication(BaseModel): """Represents a Python application.""" @@ -519,7 +419,6 @@ class PyApplication(BaseModel): @builder -@msgpk class Analysis(BaseModel): """v2 payload root: envelope + the application tree node. ``k_limit`` is an L3+ envelope key (None below the dataflow levels; exclude_none drops it).""" diff --git a/pyproject.toml b/pyproject.toml index 79b8c70..3336aab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,9 +15,6 @@ dependencies = [ # parso 0.8.5 is the first release shipping the Python 3.14 grammar; older # resolutions make jedi reject every file in a 3.14 analysis env (#107) "parso>=0.8.5", - # msgpack - "msgpack>=1.0.0,<1.0.7; python_version < '3.11'", - "msgpack>=1.0.7,<2.0.0; python_version >= '3.11'", # networkx "networkx>=2.6.0,<3.2.0; python_version < '3.11'", "networkx>=3.0.0,<4.0.0; python_version >= '3.11'", diff --git a/test/test_cli_emit_gates.py b/test/test_cli_emit_gates.py new file mode 100644 index 0000000..6516340 --- /dev/null +++ b/test/test_cli_emit_gates.py @@ -0,0 +1,97 @@ +"""Regression tests for #119 (Neo4j emission is always full-depth — enforced, +not just documented) and #118 (the msgpack output format is gone). +""" +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from codeanalyzer.__main__ import app + +_ENV = {"NO_COLOR": "1", "TERM": "dumb"} + + +@pytest.fixture() +def cli_runner(): + return CliRunner() + + +@pytest.fixture() +def tiny_project(tmp_path): + proj = tmp_path / "tiny" + proj.mkdir() + (proj / "m.py").write_text( + "def g(x):\n return x\n\n\ndef f(a):\n b = g(a)\n return b\n", + encoding="utf-8", + ) + return proj + + +def _invoke(cli_runner, *args): + return cli_runner.invoke(app, list(args), env=_ENV) + + +# ---------------------------------------------------------------------------------------------- +# #119 — --emit neo4j is always full-depth. +# ---------------------------------------------------------------------------------------------- + + +def test_emit_neo4j_rejects_explicit_analysis_level(cli_runner, tiny_project, tmp_path): + r = _invoke( + cli_runner, + "--input", str(tiny_project), "--output", str(tmp_path / "out"), + "--no-venv", "--emit", "neo4j", "-a", "2", + ) + assert r.exit_code != 0, "--emit neo4j with an explicit -a must be an error" + + +def test_emit_neo4j_rejects_explicit_graphs(cli_runner, tiny_project, tmp_path): + r = _invoke( + cli_runner, + "--input", str(tiny_project), "--output", str(tmp_path / "out"), + "--no-venv", "--emit", "neo4j", "--graphs", "cfg", + ) + assert r.exit_code != 0, "--emit neo4j with an explicit --graphs must be an error" + + +def test_emit_neo4j_runs_full_depth_by_default(cli_runner, tiny_project, tmp_path): + """Plain --emit neo4j must produce the FULL graph: the cypher snapshot has + to contain the L3/L4 families (PyCFGNode rows, PY_CFG_NEXT/PY_DDG edges, + param vertices), not just the L1 symbol table.""" + out = tmp_path / "out" + r = _invoke( + cli_runner, + "--input", str(tiny_project), "--output", str(out), + "--no-venv", "--emit", "neo4j", + ) + assert r.exit_code == 0, f"plain --emit neo4j must succeed: {r.output}" + cypher = (out / "graph.cypher").read_text() + for marker in ("PyCFGNode", "PY_CFG_NEXT", "PY_DDG", "formal_in"): + assert marker in cypher, ( + f"full-depth neo4j emission must contain {marker}; " + "the graph appears to be shallow" + ) + + +# ---------------------------------------------------------------------------------------------- +# #118 — msgpack is gone. +# ---------------------------------------------------------------------------------------------- + + +def test_format_msgpack_is_rejected(cli_runner, tiny_project, tmp_path): + r = _invoke( + cli_runner, + "--input", str(tiny_project), "--output", str(tmp_path / "out"), + "--no-venv", "--format", "msgpack", + ) + assert r.exit_code != 0, "--format msgpack must no longer be accepted" + + +def test_msgpack_absent_from_model_and_help(cli_runner): + r = _invoke(cli_runner, "--help") + assert "msgpack" not in r.output.lower() + from codeanalyzer.schema.py_schema import PyModule + + assert not hasattr(PyModule, "to_msgpack_bytes"), ( + "the msgpack serialization mixin must be removed" + ) From fec0f140cf5934e83359add09e9ab59f1e054d46 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Tue, 18 Aug 2026 10:36:14 -0400 Subject: [PATCH 22/22] chore(deps): drop unused numpy and pandas dependencies (#124) Neither package is imported anywhere in the analyzer -- `git grep` finds first-party hits only in the vendored xarray test fixture and in a comment in the Homebrew formula generator. Both were nonetheless declared in `[project].dependencies` with tight upper caps (`numpy<1.24` below Python 3.11, `numpy<2.0` above it). Those caps forced resolution onto numpy releases with no prebuilt wheel for some targets -- Red Hat UBI images in particular -- so `pip install codeanalyzer-python` fell back to building numpy from source and failed. On Python 3.11+ numpy now disappears from the resolved tree entirely, since ray 2.55 declares no numpy dependency and pandas was its only other route. On Python 3.9/3.10 `ray==2.0.0` still pulls numpy in transitively; raising that pin carries real compatibility risk and is left for a separate change. --- CHANGELOG.md | 11 +++++++++++ packaging/homebrew/generate_formula.sh | 6 +++--- pyproject.toml | 7 ------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8d462a..b9cc68e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 actually ran at the default level 1, producing a partial graph). Passing `--graphs` below `-a 3` is also now consistently rejected even when the value equals the default. +- **`numpy` and `pandas` are no longer dependencies** (#124): neither was ever + imported by the analyzer, but both were declared in `[project].dependencies` + with tight upper caps (`numpy<1.24` below Python 3.11, `numpy<2.0` above it). + The caps forced resolution onto numpy releases with no prebuilt wheel for some + targets — Red Hat UBI images in particular — so installation fell back to + building numpy from source and failed. On Python 3.11+ numpy is now absent + from the resolved tree entirely (`ray` 2.55 does not require it). On Python + 3.9/3.10 `ray==2.0.0` still requires numpy transitively — that pin is + untouched — but with the cap gone it resolves to numpy 2.0.2, which ships + cp39 manylinux wheels for x86_64 and aarch64, so the source build stops + happening there too. ## [1.1.1] - 2026-07-27 diff --git a/packaging/homebrew/generate_formula.sh b/packaging/homebrew/generate_formula.sh index 70846cf..56f3977 100755 --- a/packaging/homebrew/generate_formula.sh +++ b/packaging/homebrew/generate_formula.sh @@ -4,7 +4,7 @@ # # Unlike the codeanalyzer-typescript sibling -- which ships a single self-contained # binary that the formula just downloads -- codeanalyzer-python is a pure-Python -# package published to PyPI with heavy native dependencies (ray, pandas, numpy). +# package published to PyPI with a heavy native dependency (ray). # Vendoring every transitive dependency as a Homebrew `resource` is impractical # (ray is not buildable from an sdist), and pip-installing at build time is blocked # by Homebrew's network sandbox. @@ -41,8 +41,8 @@ class CodeanalyzerPython < Formula version "${VERSION}" license "Apache-2.0" - # codeanalyzer-python is a pure-Python PyPI package with heavy native deps - # (ray, pandas, numpy). Rather than vendor every transitive dependency as a + # codeanalyzer-python is a pure-Python PyPI package with a heavy native dep + # (ray). Rather than vendor every transitive dependency as a # Homebrew resource, install the pinned PyPI release as an isolated uv tool; # uv resolves and caches the environment on first run. depends_on "uv" diff --git a/pyproject.toml b/pyproject.toml index 3336aab..6789093 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,13 +18,6 @@ dependencies = [ # networkx "networkx>=2.6.0,<3.2.0; python_version < '3.11'", "networkx>=3.0.0,<4.0.0; python_version >= '3.11'", - # pandas - "pandas>=1.3.0,<2.0.0; python_version < '3.11'", - "pandas>=2.0.0,<3.0.0; python_version >= '3.11'", - # numpy - "numpy>=1.21.0,<1.24.0; python_version < '3.11'", - "numpy>=1.24.0,<2.0.0; python_version >= '3.11' and python_version < '3.12'", - "numpy>=1.26.0,<2.0.0; python_version >= '3.12'", # pydantic "pydantic>=1.8.0,<2.0.0; python_version < '3.11'", "pydantic>=2.0.0,<3.0.0; python_version >= '3.11'",