agent-framework/python/scripts/sample_validation/models.py at main · microsoft/agent-framework · GitHub

History

174 lines (135 loc) · 4.86 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

"""Data models for the sample validation system."""

from dataclasses import dataclass, field

from datetime import datetime

from enum import Enum

from pathlib import Path

from agent_framework import Workflow

from agent_framework.github import GitHubCopilotAgent

@dataclass

class ValidationConfig:

"""Configuration for the validation workflow."""

samples_dir: Path

python_root: Path

subdir: str | None = None

exclude: list[str] | None = None

max_parallel_workers: int = 10

playbooks_dir: Path | None = None

use_cache: bool = True

agent_timeout: int = 120

@dataclass

class SampleInfo:

"""Information about a discovered sample file."""

path: Path

relative_path: str

@classmethod

def from_path(cls, path: Path, samples_dir: Path) -> "SampleInfo":

"""Create SampleInfo from a file path."""

return cls(

path=path,

relative_path=str(path.relative_to(samples_dir)),

)

@dataclass

class DiscoveryResult:

"""Result of sample discovery."""

samples: list[SampleInfo]

@dataclass

class ReplayResult:

"""Outcome of attempting to replay cached playbooks for discovered samples.

Samples whose cached playbook replayed successfully are captured in

``cached_results`` and skip the agent entirely. Everything else flows to the

agent-driven validation as ``remaining_samples``.

"""

remaining_samples: list[SampleInfo]

cached_results: list["RunResult"] = field(default_factory=list) # type: ignore[assignment]

@dataclass

class WorkflowCreationResult:

"""Result of creating a nested per-sample concurrent workflow."""

samples: list[SampleInfo]

workflow: Workflow | None

agents: list[GitHubCopilotAgent]

cached_results: list["RunResult"] = field(default_factory=list) # type: ignore[assignment]

class RunStatus(Enum):

"""Status of a sample run."""

SUCCESS = "success"

FAILURE = "failure"

MISSING_SETUP = "missing_setup"

@dataclass

class RunResult:

"""Result of running a single sample."""

sample: SampleInfo

status: RunStatus

output: str

error: str

@dataclass

class ExecutionResult:

"""Result of sample execution."""

results: list[RunResult]

@dataclass

class Report:

"""Final validation report."""

timestamp: datetime

total_samples: int

success_count: int

failure_count: int

missing_setup_count: int

results: list[RunResult] = field(default_factory=list) # type: ignore

def to_markdown(self) -> str:

"""Generate a markdown report."""

lines = [

"# Sample Validation Report",

"",

f"**Generated:** {self.timestamp.isoformat()}",

"",

"## Summary",

"",

"| Metric | Count |",

"|--------|-------|",

f"| Total Samples | {self.total_samples} |",

f"| [PASS] Success | {self.success_count} |",

f"| [FAIL] Failure | {self.failure_count} |",

f"| [MISSING_SETUP] Missing Setup | {self.missing_setup_count} |",

"",

"## Detailed Results",

"",

]

# Group by status

for status in [RunStatus.FAILURE, RunStatus.MISSING_SETUP, RunStatus.SUCCESS]:

status_results = [r for r in self.results if r.status == status]

if not status_results:

continue

status_label = {

RunStatus.SUCCESS: "[PASS]",

RunStatus.FAILURE: "[FAIL]",

RunStatus.MISSING_SETUP: "[MISSING_SETUP]",

}

lines.append(f"### {status_label[status]} {status.value.title()} ({len(status_results)})")

lines.append("")

for result in status_results:

lines.append(f"- **{result.sample.relative_path}**")

if result.error:

# Truncate long errors

error_preview = result.error[:200] + "..." if len(result.error) > 200 else result.error

lines.append(f" - Error: `{error_preview}`")

lines.append("")

return "\n".join(lines)

def to_dict(self) -> dict[str, object]:

"""Convert report to dictionary for JSON serialization."""

return {

"timestamp": self.timestamp.isoformat(),

"summary": {

"total_samples": self.total_samples,

"success_count": self.success_count,

"failure_count": self.failure_count,

"missing_setup_count": self.missing_setup_count,

"results": [

{

"path": r.sample.relative_path,

"status": r.status.value,

"output": r.output,

"error": r.error,

}

for r in self.results

}

agent-framework/python/scripts/sample_validation/models.py at main · microsoft/agent-framework · GitHub

Uh oh!

Expand file tree

Breadcrumbs

models.py

Latest commit

History

Breadcrumbs

models.py

File metadata and controls