Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 62 additions & 2 deletions evaluate.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,19 @@ class SharedStateEvidence:
references: set[str]


@dataclass(frozen=True)
class EffectfulReference:
kind: str
category: str
reference: str
origin: str
enclosing_identifier: str
package_name: str
summary_id: str
receiver_type: str
evidence: tuple[str, ...]


@dataclass(frozen=True)
class SourceFile:
path: str
Expand All @@ -111,6 +124,7 @@ class SourceFile:
qualified_references: set[str]
effectful_imports: set[str]
effectful_identifiers: set[str]
effectful_references: tuple[EffectfulReference, ...]
has_effectful_imports: bool
has_effectful_identifiers: bool
shared_state: tuple[SharedStateEvidence, ...]
Expand Down Expand Up @@ -339,6 +353,20 @@ def parse_source_file(item: dict[str, Any]) -> SourceFile:
)
for evidence in item["sharedState"]
)
effectful_references = tuple(
EffectfulReference(
kind=evidence["kind"],
category=evidence["category"],
reference=evidence["reference"],
origin=evidence["origin"],
enclosing_identifier=evidence["enclosingIdentifier"],
package_name=evidence["packageName"],
summary_id=evidence["summaryId"],
receiver_type=evidence["receiverType"],
evidence=tuple(evidence["evidence"]),
)
for evidence in item.get("effectfulReferences", [])
)
return SourceFile(
path=item["path"],
test_scope=item["testScope"],
Expand All @@ -358,8 +386,11 @@ def parse_source_file(item: dict[str, Any]) -> SourceFile:
qualified_references=set(item["qualifiedReferences"]),
effectful_imports=set(item["effectfulImports"]),
effectful_identifiers=set(item["effectfulIdentifiers"]),
has_effectful_imports=bool(item["effectfulImports"]),
has_effectful_identifiers=bool(item["effectfulIdentifiers"]),
effectful_references=effectful_references,
has_effectful_imports=bool(item["effectfulImports"])
or any(effect.kind == "import" for effect in effectful_references),
has_effectful_identifiers=bool(item["effectfulIdentifiers"])
or any(effect.kind != "import" for effect in effectful_references),
shared_state=shared_state,
has_shared_mutable_state=bool(shared_state),
property_checks=property_checks,
Expand Down Expand Up @@ -421,6 +452,35 @@ def evaluate_fact_consistency(files: list[SourceFile]) -> list[Violation]:
+ ", ".join(effectful_identifiers_outside_identifiers),
)
)
effect_enclosing_identifiers_outside_identifiers = sorted(
{
effect.enclosing_identifier
for effect in source_file.effectful_references
if effect.enclosing_identifier
}
- source_file.identifiers
)
if effect_enclosing_identifiers_outside_identifiers:
violations.append(
Violation(
source_file.path,
"effectful reference enclosing identifiers must be structural identifiers: "
+ ", ".join(effect_enclosing_identifiers_outside_identifiers),
)
)
dependency_effects_without_summary = sorted(
effect.reference
for effect in source_file.effectful_references
if effect.origin == "dependency-summary" and (not effect.package_name or not effect.summary_id)
)
if dependency_effects_without_summary:
violations.append(
Violation(
source_file.path,
"dependency effectful references must include packageName and summaryId: "
+ ", ".join(dependency_effects_without_summary),
)
)
references_outside_property = sorted(
source_file.property_test_references - source_file.api_references
)
Expand Down
79 changes: 79 additions & 0 deletions evaluate_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ def source_file(**overrides):
"qualifiedReferences": [],
"effectfulImports": [],
"effectfulIdentifiers": [],
"effectfulReferences": [],
"sharedState": [],
"propertyChecks": [],
"interfaceLogicEvidence": empty_interface_logic_evidence(),
Expand Down Expand Up @@ -110,6 +111,82 @@ def test_effectful_identifiers_must_be_structural_identifiers(self):
[violation.message for violation in violations],
)

def test_effectful_reference_enclosing_identifier_must_be_structural_identifier(self):
source = source_file(
path="/repo/Core.swift",
identifiers=[],
effectfulReferences=[
{
"kind": "call",
"category": "network",
"reference": "messages.create",
"origin": "dependency-summary",
"enclosingIdentifier": "run",
"packageName": "@anthropic-ai/sdk",
"summaryId": "anthropic-sdk-messages-create-network",
"receiverType": "Anthropic",
"evidence": ["summary"],
}
],
)

violations = evaluate.evaluate([source])

self.assertIn(
"effectful reference enclosing identifiers must be structural identifiers: run",
[violation.message for violation in violations],
)

def test_dependency_effectful_reference_requires_summary_identity(self):
source = source_file(
path="/repo/Core.swift",
effectfulReferences=[
{
"kind": "call",
"category": "network",
"reference": "messages.create",
"origin": "dependency-summary",
"enclosingIdentifier": "decideSync",
"packageName": "",
"summaryId": "",
"receiverType": "Anthropic",
"evidence": ["summary"],
}
],
)

violations = evaluate.evaluate([source])

self.assertIn(
"dependency effectful references must include packageName and summaryId: messages.create",
[violation.message for violation in violations],
)

def test_structured_effectful_reference_counts_as_effectful_identifier(self):
source = source_file(
path="/repo/Core.swift",
effectfulReferences=[
{
"kind": "call",
"category": "network",
"reference": "messages.create",
"origin": "dependency-summary",
"enclosingIdentifier": "decideSync",
"packageName": "@anthropic-ai/sdk",
"summaryId": "anthropic-sdk-messages-create-network",
"receiverType": "Anthropic",
"evidence": ["summary"],
}
],
)

violations = evaluate.evaluate([source])

self.assertIn(
"effectful identifiers may only appear in shell, state, interface, test, stateTest, or exempt modules",
[violation.message for violation in violations],
)

def test_property_test_references_must_be_api_references(self):
source = source_file(
path="/repo/CoreTests.swift",
Expand Down Expand Up @@ -1432,6 +1509,7 @@ def test_run_adapter_parses_adapter_json_without_evaluating_policy(self):
"qualifiedReferences": [],
"effectfulImports": [],
"effectfulIdentifiers": [],
"effectfulReferences": [],
"sharedState": [],
"propertyChecks": [],
"interfaceLogicEvidence": empty_interface_logic_evidence(),
Expand Down Expand Up @@ -1573,6 +1651,7 @@ def valid_fact_document():
"qualifiedReferences": [],
"effectfulImports": [],
"effectfulIdentifiers": [],
"effectfulReferences": [],
"sharedState": [],
"propertyChecks": [],
"interfaceLogicEvidence": empty_interface_logic_evidence(),
Expand Down
27 changes: 27 additions & 0 deletions fact.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,28 @@
"references": { "$ref": "#/$defs/stringList" }
}
},
"effectfulReference": {
"type": "object",
"additionalProperties": false,
"required": ["kind", "category", "reference", "origin", "enclosingIdentifier", "packageName", "summaryId", "receiverType", "evidence"],
"properties": {
"kind": {
"enum": ["call", "new", "import", "identifier"]
},
"category": {
"enum": ["network", "filesystem", "database", "process", "timer", "console", "storage", "browser", "unknown"]
},
"reference": { "$ref": "#/$defs/nonEmptyString" },
"origin": {
"enum": ["standard-library", "global", "dependency-summary", "local-call-expansion"]
},
"enclosingIdentifier": { "type": "string" },
"packageName": { "type": "string" },
"summaryId": { "type": "string" },
"receiverType": { "type": "string" },
"evidence": { "$ref": "#/$defs/stringList" }
}
},
"interfaceLogicEvidence": {
"type": "object",
"additionalProperties": false,
Expand Down Expand Up @@ -140,6 +162,11 @@
"qualifiedReferences": { "$ref": "#/$defs/stringList" },
"effectfulImports": { "$ref": "#/$defs/stringList" },
"effectfulIdentifiers": { "$ref": "#/$defs/stringList" },
"effectfulReferences": {
"type": "array",
"uniqueItems": true,
"items": { "$ref": "#/$defs/effectfulReference" }
},
"sharedState": {
"type": "array",
"uniqueItems": true,
Expand Down
Loading
Loading