4.6 Multi-Instance Review and Output Validation

import asyncio

async def multi_instance_classify(text, n=3):
    # Run N instances in parallel
    tasks = [
        client.messages.create(
            model="claude-sonnet-4-20250514",
            messages=[{"role": "user", "content": f"Classify sentiment: {text}"}]
        )
        for _ in range(n)
    ]
    responses = await asyncio.gather(*tasks)
    
    # Collect votes
    votes = [r.content[0].text.strip().lower() for r in responses]
    
    # Majority vote
    from collections import Counter
    vote_counts = Counter(votes)
    winner = vote_counts.most_common(1)[0]
    
    return {
        "result": winner[0],
        "confidence": winner[1] / n,
        "agreement": winner[1] == n,  # unanimous?
        "all_votes": votes
    }

# Instance 1: Generate answer
answer = await agent("What are the security risks in this code?")

# Instance 2: Adversarial verifier (try to refute)
verification = await agent(f"""
Try to refute this claim. Default to refuted=true if uncertain:

Claim: {answer}

Is this claim correct and well-supported?
""", schema={"refuted": "boolean", "reason": "string"})

if not verification.refuted:
    return answer  # Survived verification
else:
    return None  # Rejected

async def confident_answer(question, threshold=0.7, n=5):
    results = await multi_instance_classify(question, n=n)
    
    if results["confidence"] >= threshold:
        return results["result"]
    else:
        # Low confidence — escalate to human or ask for clarification
        return {
            "status": "uncertain",
            "best_guess": results["result"],
            "confidence": results["confidence"],
            "disagreements": results["all_votes"]
        }

import jsonschema

def validate_output(output, schema):
    try:
        jsonschema.validate(output, schema)
        return True, None
    except jsonschema.ValidationError as e:
        return False, str(e)

validation = await client.messages.create(
    messages=[{"role": "user", "content": f"""
    Does this answer make logical sense? Check for:
    1. Internal contradictions
    2. Factual impossibilities
    3. Missing required information
    
    Answer: {original_output}
    
    Valid (yes/no) and reason:
    """}]
)

# Extract claims from output
claims = extract_claims(output)

# Verify each claim against source
for claim in claims:
    evidence = search_source_docs(claim)
    if not evidence:
        flag_unverified(claim)