Advanced Features

.youbencha-workspace/
└── run-{timestamp}-{hash}/
    ├── src-modified/              # Code after agent execution
    ├── src-expected/              # Reference code (if configured)
    ├── artifacts/
    │   ├── results.json           # Machine-readable results
    │   ├── report.md              # Human-readable report
    │   ├── youbencha.log.json     # Agent execution log
    │   └── git-diff.patch         # Git diff output
    └── .youbencha.lock            # Workspace metadata

agent:
  type: copilot-cli
  model: claude-sonnet-4.5  # Or gpt-5, gemini-3-pro-preview

evaluators:
  - name: agentic-judge
    config:
      type: copilot-cli
      model: gpt-5.1  # Use different model for evaluation

repo: https://github.com/example/repo.git
branch: main
expected_source: branch      # Or: commit
expected: feature/completed  # Reference branch/commit

evaluators:
  - name: expected-diff
    config:
      threshold: 0.85  # Require 85% similarity to pass

evaluators:
  # Judge 1: Error Handling
  - name: agentic-judge-error-handling
    config:
      type: copilot-cli
      agent_name: agentic-judge
      assertions:
        has_try_catch: "Includes try-catch blocks. Score 1 if present, 0 if absent."
        proper_logging: "Errors are logged appropriately. Score 1 if yes, 0 if no."
  
  # Judge 2: Documentation
  - name: agentic-judge-documentation
    config:
      type: copilot-cli
      agent_name: agentic-judge
      assertions:
        functions_documented: "Functions have JSDoc. Score 1 if all, 0 if none."
        readme_updated: "README.md reflects changes. Score 1 if yes, 0 if no."

# evaluators/test-coverage.yaml
name: agentic-judge:test-coverage
description: "Ensures code changes include appropriate test coverage"

config:
  type: copilot-cli
  agent_name: agentic-judge
  assertions:
    tests_added: "New tests were added. Score 1 if yes, 0 if no."
    coverage_adequate: "Test coverage is adequate. Score 1 if ≥80%, 0.5 if 60-80%, 0 if <60%."

# Reference in test case:
# evaluators:
#   - file: ./evaluators/test-coverage.yaml

evaluators:
  - name: git-diff
    config:
      assertions:
        max_files_changed: 5        # Max files modified
        max_lines_added: 100        # Max lines added
        max_lines_removed: 50       # Max lines removed
        max_total_changes: 150      # Max total changes
        min_change_entropy: 0.5     # Min entropy (distributed)
        max_change_entropy: 2.0     # Max entropy (focused)

pre_execution:
  - name: script
    config:
      command: bash
      args:
        - "-c"
        - |
          mkdir -p ${WORKSPACE_DIR}/config
          cat > ${WORKSPACE_DIR}/config/auth.json << EOF
          {"jwtSecret": "test-key"}
          EOF
      timeout_ms: 30000

post_evaluation:
  - name: database
    config:
      type: json-file
      output_path: ./results-history.jsonl
      include_full_bundle: true
      append: true

post_evaluation:
  - name: webhook
    config:
      url: ${SLACK_WEBHOOK_URL}
      method: POST
      headers:
        Content-Type: "application/json"
      retry_on_failure: true

post_evaluation:
  - name: script
    config:
      command: ./scripts/analyze.sh
      args: ["${RESULTS_PATH}"]
      env:
        SLACK_WEBHOOK: "${SLACK_WEBHOOK_URL}"
        ANALYSIS_MODE: "detailed"

agent:
  type: copilot-cli
  config:
    prompt_file: ./prompts/add-authentication.md

evaluators:
  - name: agentic-judge
    config:
      prompt_file: ./prompts/strict-evaluation.txt
      assertions:
        # ... assertions here

agent:
  type: copilot-cli
  agent_name: my-custom-agent  # Uses .github/agents/my-custom-agent/
  config:
    prompt: "Your task description"

# Test case level
timeout: 600000  # 10 minutes (default: 5 minutes)

# Pre-execution level
pre_execution:
  - name: script
    config:
      command: ./long-setup.sh
      timeout_ms: 120000  # 2 minutes

# View results summary
jq '.summary' results.json

# Calculate pass rate across suite
jq -s 'map(.summary.overall_status == "passed") | 
       map(select(.) | 1) | add / length' results/*.json

# Track metrics over time
jq '.metrics' results.json

# Detect regression
PREV=$(tail -n 2 history.jsonl | head -n 1 | jq '.summary.passed')
CURR=$(tail -n 1 history.jsonl | jq '.summary.passed')
if [ "$CURR" -lt "$PREV" ]; then
    echo "⚠️ REGRESSION DETECTED"
    exit 1
fi

# Track cost trends
jq '.execution.token_usage' history.jsonl

# Markdown report (default)
yb report --from results.json --format markdown

# JSON report for programmatic use
yb report --from results.json --format json --output report.json