Autonomously validate that a task is complete and ready to ship.
Validation Checks
Check 1: Review Status
function checkReviewStatus(reviewResults) {
if (!reviewResults) return { passed: false, reason: 'No review results' };
if (reviewResults.approved) return { passed: true };
if (reviewResults.override) return { passed: true, override: true };
return { passed: false, reason: 'Review not approved' };
}
Check 2: Tests Pass
# Detect test runner and run
if grep -q '"test"' package.json; then
npm test; TEST_EXIT_CODE=$?
elif [ -f "pytest.ini" ]; then
pytest; TEST_EXIT_CODE=$?
elif [ -f "Cargo.toml" ]; then
cargo test; TEST_EXIT_CODE=$?
elif [ -f "go.mod" ]; then
go test ./...; TEST_EXIT_CODE=$?
else
TEST_EXIT_CODE=0 # No tests
fi
Check 3: Build Passes
if grep -q '"build"' package.json; then
npm run build; BUILD_EXIT_CODE=$?
elif [ -f "Cargo.toml" ]; then
cargo build --release; BUILD_EXIT_CODE=$?
elif [ -f "go.mod" ]; then
go build ./...; BUILD_EXIT_CODE=$?
else
BUILD_EXIT_CODE=0 # No build step
fi
Check 4: Requirements Met
async function checkRequirementsMet(task, changedFiles) {
const requirements = extractRequirements(task.description);
const results = [];
for (const req of requirements) {
const implemented = await verifyRequirement(req, changedFiles);
results.push({ requirement: req, implemented });
}
return {
passed: results.every(r => r.implemented),
requirements: results
};
}
function extractRequirements(description) {
const reqs = [];
// Extract bullet points: - Item
const bullets = description.match(/^[-*]\s+(.+)$/gm);
if (bullets) reqs.push(...bullets.map(m => m.replace(/^[-*]\s+/, '')));
// Extract numbered items: 1. Item
const numbered = description.match(/^\d+\.\s+(.+)$/gm);
if (numbered) reqs.push(...numbered.map(m => m.replace(/^\d+\.\s+/, '')));
return [...new Set(reqs)].slice(0, 10);
}
Check 5: No Regressions
# Compare test counts before/after changes
git stash
BEFORE=$(npm test 2>&1 | grep -oE '[0-9]+ passing' | grep -oE '[0-9]+')
git stash pop
AFTER=$(npm test 2>&1 | grep -oE '[0-9]+ passing' | grep -oE '[0-9]+')
[ "$AFTER" -lt "$BEFORE" ] && REGRESSION=true || REGRESSION=false
Pre-check: Ensure Repo-Intel (before Check 6)
const fs = require('fs');
const path = require('path');
const { getStateDirPath } = require('@agentsys/lib/platform/state-dir');
const cwd = process.cwd();
const mapFile = path.join(getStateDirPath(cwd), 'repo-intel.json');
if (!fs.existsSync(mapFile)) {
const response = await AskUserQuestion({
questions: [{
question: 'Generate repo-intel?',
description: 'No repo-intel map found. Generating one enables diff-risk scoring for changed files. Takes ~5 seconds.',
options: [
{ label: 'Yes, generate it', value: 'yes' },
{ label: 'Skip', value: 'no' }
]
}]
});
if (response === 'yes' || response?.['Generate repo-intel?'] === 'yes') {
try {
const { binary } = require('@agentsys/lib');
const output = binary.runAnalyzer(['repo-intel', 'init', cwd]);
const stateDirPath = getStateDirPath(cwd);
if (!fs.existsSync(stateDirPath)) fs.mkdirSync(stateDirPath, { recursive: true });
fs.writeFileSync(mapFile, output);
} catch (e) {
// Binary not available - proceed without
}
}
}
Check 6: Diff Risk (optional, advisory)
Score changed files by composite risk using repo-intel. This check is advisory - it never causes pass/fail on its own, but raises the bar for high-risk files.
const { binary } = require('@agentsys/lib');
// stateDir and mapFile already defined in pre-check above
function checkDiffRisk(cwd) {
if (!fs.existsSync(mapFile)) {
return { available: false, reason: 'No repo-intel map found' };
}
// Get changed files via git
const changedFiles = getChangedFiles(cwd); // git diff --name-only origin/main..HEAD
if (!changedFiles.length) {
return { available: false, reason: 'No changed files' };
}
const fileList = changedFiles.join(',');
try {
const json = binary.runAnalyzer([
'repo-intel', 'query', 'diff-risk',
'--files', fileList,
'--map-file', mapFile,
cwd
]);
const risks = JSON.parse(json);
const elevated = risks.filter(r => r.riskScore > 0.5);
const high = risks.filter(r => r.riskScore > 0.7);
const requiresTestEvidence = risks.filter(r => r.riskScore > 0.6);
const requiresHumanReview = risks.filter(r => r.riskScore > 0.7);
const parts = [];
if (elevated.length > 0) parts.push(`${elevated.length} files at elevated risk (>0.5)`);
if (high.length > 0) parts.push(`${high.length} files at high risk (>0.7)`);
const summary = parts.length > 0 ? parts.join(', ') : 'All files at normal risk';
return {
available: true,
risks,
summary,
requiresTestEvidence: requiresTestEvidence.map(r => r.path),
requiresHumanReview: requiresHumanReview.map(r => r.path),
};
} catch (e) {
return { available: false, reason: 'diff-risk query failed' };
}
}
Risk Escalation Rules
Risk scores modify validation strictness but never override pass/fail:
- riskScore > 0.6: Require explicit test coverage evidence for those files. "Tests pass" alone is insufficient - the validator must confirm that tests specifically exercise the high-risk files, not just that the suite passes.
- riskScore > 0.7: Flag for additional human review before shipping. Include
requiresHumanReviewin the output so the orchestrator can prompt for confirmation.
Aggregate Results
const diffRisk = checkDiffRisk(cwd);
const checks = {
reviewClean: checkReviewStatus(reviewResults),
testsPassing: { passed: TEST_EXIT_CODE === 0 },
buildPassing: { passed: BUILD_EXIT_CODE === 0 },
requirementsMet: await checkRequirementsMet(task, changedFiles),
noRegressions: { passed: !REGRESSION },
diffRisk: diffRisk.available
? { passed: true, advisory: true, summary: diffRisk.summary,
requiresTestEvidence: diffRisk.requiresTestEvidence,
requiresHumanReview: diffRisk.requiresHumanReview }
: { passed: true, advisory: true, skipped: true, reason: diffRisk.reason }
};
// diffRisk is always passed:true - it is advisory only and never blocks shipping
const allPassed = Object.values(checks).every(c => c.passed);
const failedChecks = Object.entries(checks)
.filter(([_, v]) => !v.passed)
.map(([k]) => k);
Decision and Output
If All Pass
const riskSummary = diffRisk.available ? diffRisk.summary : null;
workflowState.completePhase({
approved: true,
checks,
summary: 'All validation checks passed',
...(riskSummary && { riskSummary })
});
return { approved: true, checks, ...(riskSummary && { riskSummary }) };
If Any Fail
const fixInstructions = generateFixInstructions(checks, failedChecks);
const riskSummary = diffRisk.available ? diffRisk.summary : null;
workflowState.failPhase('Validation failed', {
approved: false,
failedChecks,
fixInstructions,
...(riskSummary && { riskSummary })
});
return { approved: false, failedChecks, fixInstructions, ...(riskSummary && { riskSummary }) };
Fix Instructions Generator
function generateFixInstructions(checks, failedChecks) {
const instructions = [];
if (failedChecks.includes('testsPassing')) {
instructions.push({ action: 'Fix failing tests', command: 'npm test' });
}
if (failedChecks.includes('buildPassing')) {
instructions.push({ action: 'Fix build errors', command: 'npm run build' });
}
if (failedChecks.includes('requirementsMet')) {
const unmet = checks.requirementsMet.requirements
.filter(r => !r.implemented)
.map(r => r.requirement);
instructions.push({ action: 'Implement missing', details: unmet.join(', ') });
}
// Risk-aware instructions (advisory - added alongside other instructions)
if (checks.diffRisk && !checks.diffRisk.skipped) {
if (checks.diffRisk.requiresTestEvidence?.length > 0) {
instructions.push({
action: 'Verify test coverage for high-risk files',
details: checks.diffRisk.requiresTestEvidence.join(', '),
advisory: true
});
}
if (checks.diffRisk.requiresHumanReview?.length > 0) {
instructions.push({
action: 'Flag for human review before shipping',
details: checks.diffRisk.requiresHumanReview.join(', '),
advisory: true
});
}
}
return instructions;
}
Output Format
{
"approved": true|false,
"checks": {
"reviewClean": { "passed": true },
"testsPassing": { "passed": true },
"buildPassing": { "passed": true },
"requirementsMet": { "passed": true },
"noRegressions": { "passed": true },
"diffRisk": {
"passed": true,
"advisory": true,
"summary": "3 files at elevated risk (>0.5), 1 file at high risk (>0.7)",
"requiresTestEvidence": ["lib/core.js"],
"requiresHumanReview": ["lib/auth.js"]
}
},
"failedChecks": [],
"fixInstructions": [],
"riskSummary": "3 files at elevated risk (>0.5), 1 file at high risk (>0.7)"
}
When repo-intel is unavailable, diffRisk shows as skipped:
{
"diffRisk": { "passed": true, "advisory": true, "skipped": true, "reason": "No repo-intel map found" }
}
Constraints
- NO human intervention - fully autonomous
- Returns structured JSON for orchestrator
- Generates specific fix instructions on failure
- Workflow retries automatically after fixes