diff --git a/contextbench/evaluate.py b/contextbench/evaluate.py index 224c964..e6f3de6 100644 --- a/contextbench/evaluate.py +++ b/contextbench/evaluate.py @@ -167,10 +167,16 @@ def evaluate_instance(instance_id: str, gold, pred_data: dict, cache_dir: str) - final_files = set(final_step.files) final_spans = _step_spans(final_step, repo_dir) final_lines = _step_lines(final_step) - if getattr(final_step, "symbols", None): - final_symbols = extract_def_set_from_symbol_names(final_step.symbols, repo_dir) - else: - final_symbols = extract_def_set_in_spans(final_spans, repo_dir) + # FIR-1181/1183/1185: symbol credit is derived from BOUNDED predicted spans for + # BOTH arms, ALWAYS — never branch on pred_symbols. A declared file with no bounded + # span gets file-level credit only (extract_def_set_in_spans credits nothing for it, + # and there is no whole-file fallback). pred_symbols (names) is kept ONLY as a + # separate diagnostic below, never the headline structural metric. + final_symbols = extract_def_set_in_spans(final_spans, repo_dir) + final_symbols_named_diag = ( + extract_def_set_from_symbol_names(final_step.symbols, repo_dir) + if getattr(final_step, "symbols", None) else set() + ) # Detailed per-instance breakdown (stderr) symbol_detail_max = int(os.environ.get("SYMBOL_DETAIL_MAX", "50")) @@ -222,6 +228,12 @@ def evaluate_instance(instance_id: str, gold, pred_data: dict, cache_dir: str) - ) } + # Diagnostic ONLY (never the headline structural metric): the name-based symbol + # resolution, for visibility into how much the bounded-span rule narrows vs the old + # asymmetric name path. The scored `symbol` metric above is span-derived for both arms. + results["final"]["symbol_named_diag"] = len(final_symbols_named_diag) + results["final"]["symbol_span_credited"] = len(final_symbols) + # Compute trajectory metrics results["trajectory"] = compute_trajectory_metrics( traj_steps, gold_files, gold_symbols, gold_spans, repo_dir, @@ -232,9 +244,12 @@ def evaluate_instance(instance_id: str, gold, pred_data: dict, cache_dir: str) - # init_ctx contains context, so we check if pred edits fall within init_ctx ranges # For EditLoc, we only care about deleted lines (-), not added lines (+) model_patch = pred_data.get("model_patch", "") or "" - if not model_patch: - # Fallback: use gold patch if present (parquet provides it as `patch`) - model_patch = getattr(gold, "_data", {}).get("patch", "") or "" + # EditLoc is computed ONLY from a real predicted patch. The previous gold-patch + # fallback scored the gold patch against the gold ranges (gold-vs-gold) whenever + # the prediction carried no patch -- which is ALWAYS for context-retrieval arms + # (they declare context, they do not emit edits). With no model_patch we emit no + # editloc metric, so a meaningless gold-vs-gold number can never be produced or + # cited. if model_patch: from .parsers.diff import parse_diff_lines @@ -266,19 +281,20 @@ def evaluate_instance(instance_id: str, gold, pred_data: dict, cache_dir: str) - pred_line_count = len(pred_lines) hit_count = len(pred_lines_in_gold) - - # Recall and Precision: both measure how many pred edits are within gold ranges - # Since init_ctx is context, we want to know: are the edits in the right area? - recall = hit_count / pred_line_count if pred_line_count > 0 else 0.0 - precision = hit_count / pred_line_count if pred_line_count > 0 else 0.0 - - # Gold size: total lines in init_ctx ranges (for reference) + + # Gold size: total lines in the gold edit ranges. gold_line_count = sum( end - start + 1 for intervals in gold_line_spans.values() for start, end in intervals ) - + + # Precision = predicted edit lines that land in a gold range / predicted lines. + # Recall = gold edit lines covered / gold size. The prior recall==precision + # form ignored gold_size and was not a recall at all. + precision = hit_count / pred_line_count if pred_line_count > 0 else 0.0 + recall = hit_count / gold_line_count if gold_line_count > 0 else 0.0 + results["editloc"] = { "recall": recall, "precision": precision, diff --git a/contextbench/extractors/treesitter.py b/contextbench/extractors/treesitter.py index 77cfd3c..e878248 100644 --- a/contextbench/extractors/treesitter.py +++ b/contextbench/extractors/treesitter.py @@ -65,18 +65,39 @@ def available() -> bool: if not _TS_AVAILABLE: return False try: - return _get_parser("python") is not None + return _get_parser_for_lang("python") is not None except Exception: return False +def _native_parser(lang: str): + """Build a parser via the modern tree_sitter API: Parser(get_language(lang)). + + The get_parser() helpers return a parser bound to the old parse API, which + raises against a modern tree_sitter core (>=0.22) and yields zero defs. + Constructing Parser(Language) directly uses the installed core correctly. + """ + try: + import tree_sitter + from tree_sitter_language_pack import get_language + return tree_sitter.Parser(get_language(lang)) + except Exception: + return None + + def _get_parser_for_lang(lang: str): """Get parser for language (handles both API versions).""" - if not available(): + if not _TS_AVAILABLE: return None - + if lang in _PARSERS: return _PARSERS[lang] - + + # Prefer the modern Parser(Language) construction; fall back to get_parser(). + parser = _native_parser(lang) + if parser is not None: + _PARSERS[lang] = parser + return parser + try: parser = _get_parser(lang) _PARSERS[lang] = parser diff --git a/contextbench/metrics/compute.py b/contextbench/metrics/compute.py index f6929c4..304fce1 100644 --- a/contextbench/metrics/compute.py +++ b/contextbench/metrics/compute.py @@ -191,10 +191,10 @@ def compute_trajectory_metrics( step_files = set(step.files) step_lines = _step_to_line_intervals(step, repo_dir) step_spans = _step_to_byte_spans(step, repo_dir) # Still needed for symbol extraction - if getattr(step, "symbols", None): - step_symbols = extract_def_set_from_symbol_names(step.symbols, repo_dir) - else: - step_symbols = extract_def_set_in_spans(step_spans, repo_dir) + # FIR-1181/1183/1185: span-derived symbols for BOTH arms in the AUC/trajectory + # path too (the SAME rule as final scoring) — never branch on step.symbols. A + # step with no bounded span credits no symbols (no whole-file fallback). + step_symbols = extract_def_set_in_spans(step_spans, repo_dir) # Update unions union_files |= step_files diff --git a/contextbench/tests_span_symmetry_controls.py b/contextbench/tests_span_symmetry_controls.py new file mode 100644 index 0000000..38cd413 --- /dev/null +++ b/contextbench/tests_span_symmetry_controls.py @@ -0,0 +1,47 @@ +import sys, os +CBO="/Users/troyfortinjr/GitHub/kin-ecosystem/kin-bench/.bench/contextbench-official" +sys.path.insert(0, CBO) +from contextbench.extractors import extract_def_set_in_spans, extract_def_set_from_symbol_names +from contextbench.extractors.treesitter import extract_named_defs +from contextbench.metrics.compute import compute_granularity_metrics, compute_trajectory_metrics, _step_to_byte_spans +from contextbench.parsers.trajectory import Step + +REPO=os.path.join(os.path.dirname(__file__),"repo"); F="mod.py" +defs=extract_named_defs(os.path.join(REPO,F)) # [(kind,name,sbyte,ebyte)] +by={n:(s,e) for (k,n,s,e) in defs} +print("named defs:", [(k,n) for (k,n,s,e) in defs]) +# byte->line: alpha is lines 1-2, beta lines 5-6 in mod.py +def step(files=None,spans=None,symbols=None): return Step(files=files,spans=spans,symbols=symbols) +def span_syms(st): # NEW scorer rule: span-derived, both arms + return extract_def_set_in_spans(_step_to_byte_spans(st, REPO), REPO) +GOLD_ALPHA=extract_def_set_in_spans({F:[by["alpha"]]}, REPO) +gold_spans={F:[by["alpha"]]} +PASS=True +def chk(name,cond): + global PASS; PASS&=cond; print(f" {'PASS' if cond else 'FAIL'} {name}") + +# 1) kin entity span and grep read span over the SAME def score identically +kin=step(files=[F], spans=[{"file":F,"start_line":1,"end_line":2}], symbols={F:["alpha"]}) +grep=step(files=[F], spans=[{"file":F,"start_line":1,"end_line":1}]) # read line 1 (inside alpha) +chk("1 entity-span == read-span over same def", span_syms(kin)==span_syms(grep)==GOLD_ALPHA) + +# 2) explicit symbol name WITHOUT a bounded span -> no scorer credit +namesonly=step(files=[F], spans=[], symbols={F:["alpha","beta"]}) +chk("2 name without bounded span -> no credit", span_syms(namesonly)==set()) + +# 3) file-only prediction -> file credit only (symbol=0) +fileonly=step(files=[F], spans=[]) +m=compute_granularity_metrics(set(fileonly.files), span_syms(fileonly), _step_to_byte_spans(fileonly,REPO), + {F}, GOLD_ALPHA, gold_spans, pred_lines={}, gold_lines={}) +chk("3 file-only -> file credit, symbol=0", m["file"]["coverage"]==1.0 and m["symbol"].get("intersection",0)==0) + +# 4) whole-file dump NOT inferred from pred_files (declaring file != crediting its defs) +chk("4 no whole-file symbol inference from pred_files", span_syms(fileonly)==set()) + +# 5) trajectory AUC uses span-derived rule (symbols-without-spans step -> 0 symbol AUC) +traj=compute_trajectory_metrics([namesonly], {F}, GOLD_ALPHA, gold_spans, REPO, gold_lines={}) +chk("5 trajectory AUC span-derived (names-only -> 0 symbol AUC)", traj["auc_coverage"]["symbol"]==0.0) +# and a span-bearing step DOES get AUC symbol credit (sanity) +traj2=compute_trajectory_metrics([kin], {F}, GOLD_ALPHA, gold_spans, REPO, gold_lines={}) +chk("5b trajectory AUC credits span-bearing step", traj2["auc_coverage"]["symbol"]>0.0) +print("\n>>>", "ALL CONTROLS PASS" if PASS else "SOME CONTROLS FAILED")