Everything is a Trace: Stephanie Enters Full Reflective Mode

class PlanTrace:
    """
    Represents the complete execution trace of a reasoning plan.
    This is Stephanie's cognitive mirror the foundation for 
    self-reflection and self-improvement.
    """
    # --- Core Identifiers ---
    trace_id: str  # Unique identifier for this specific trace/execution
    
    # --- Initial Context ---
    goal_text: str  # The original goal or query
    goal_id: int
    input_data: Dict[str, Any]  # Any initial data or variables provided to the plan
    
    # --- Plan Definition (Optional but useful for context) ---
    plan_signature: str  # e.g., "knowledge_db_loader_document_ebt_inference"

    # --- Execution Details ---
    execution_steps: List[ExecutionStep]  # The sequence of cognitive steps
    
    # --- Final Outcome ---
    final_output_text: str  # The final output produced by the plan
    pipeline_score: Optional[Dict[str, float]] = None  # e.g., {"helpfulness": 0.85, "truthfulness": 0.78}

    # --- Target for Epistemic Quality Assessment ---
    target_epistemic_quality: Optional[float] = None 
    target_epistemic_quality_source: Optional[str] = None 

    # --- Metadata ---
    extra_data: Optional[Dict[str, Any]] = field(default_factory=dict) 

@dataclass
class ExecutionStep:
    """
    Represents a single cognitive step in the execution of a reasoning plan.
    The atomic unit of Stephanie's self-awareness.
    """
    step_id: str  # Unique identifier (trace_id_step_1)
    step_order: int
    step_type: str  # e.g., "knowledge_db_loader", "document_scorer"
    description: str  # What this step accomplishes
    
    # Core inputs/outputs
    input_text: Optional[str] = None
    output_text: Optional[str] = None
    
    # CRITICAL INNOVATION: Flexible attributes dictionary
    # This is the 4th dimension of understanding
    attributes: Dict[str, Any] = field(default_factory=dict)
    
    # Standard metadata
    agent_name: Optional[str] = None
    start_time: Optional[float] = None
    end_time: Optional[float] = None
    duration: Optional[float] = None
    error: Optional[Dict[str, Any]] = None
    output_keys: Optional[List[str]] = None
    output_size: Optional[int] = None

{
  "trace_id": "trace_01f6af9f4c804425a9c654f0157cb172",
  "goal_text": "Will AI ever be able to reprogram itself?",
  "plan_signature": "SimplifiedLATS_10_steps",
  "execution_steps": [
    {
      "step_id": "1754096022981",
      "step_order": 1,
      "step_type": "reasoning",
      "description": "Simplified LATS Step 1",
      "output_text": "Examine existing technologies and research initiatives that explore self-modifying AI, such as neural architecture search, meta-learning, or reinforcement learning, to assess their alignment with \"self-reprogramming\" and identify gaps in current capabilities.",
      "scores": {
        "alignment": { "score": 98.1153, "source": "sicql"},
        "clarity": { "score": 80.9811, "source": "sicql"},
        "implementability": { "score": 69.6087, "source": "sicql"},
        "novelty": { "score": 73.8141, "source": "sicql"},
        "relevance": {"score": 72.836, "source": "sicql"}
      }
    },
    {
      "step_id": "1754096022982",
      "output_text": "Step 3: Evaluate potential future advancements, such as recursive self-improvement frameworks or hybrid human-AI collaboration models, and assess their feasibility based on existing research trends.",
    },
    {
      "step_id": "1754096022983",
      "output_text": "Step 4: Analyze current research progress and technical barriers in developing AI capable of autonomous self-reprogramming, including computational limits, verification risks, and ethical implications.",
    }
    ...
  ],
  "final_output_text": "AI may eventually achieve self-reprogramming through advancements in self-improving algorithms and recursive learning, but this would require overcoming significant technical, ethical, and safety challenges, making it a possibility rather than a certainty.",
  "final_scores": {
    "alignment": { "score": 97.9853, "source": "sicql"},
    "clarity": { "score": 80.2211, "source": "sicql"},
    "implementability": {  "score": 69.9953, "source": "sicql" },
    "novelty": {"score": 74.5296, "source": "sicql" },
    "relevance": {"score": 72.6343, "source": "sicql" }
  },
  "target_epistemic_quality": 79.07,
  "target_epistemic_quality_source": "sicql",
  "created_at": "",
}

# Find documents with high uncertainty in reasoning quality
high_uncertainty_docs = corpus.get_metric_matrix("reasoning_quality", "uncertainty")
high_uncertainty_docs = high_uncertainty_docs[
    high_uncertainty_docs.mean(axis=1) > 0.3
].index.tolist()

# Analyze which step type had highest uncertainty
step_types = [step.step_type for step_id, step in high_uncertainty_docs]
problematic_step = max(set(step_types), key=step_types.count)

register("plan_trace_monitor", PlanTraceMonitor(cfg, self.memory, self.logger))

plan_trace_monitor: PlanTraceMonitor = get_registered_component("plan_trace_monitor")

plan_trace_monitor.start_pipeline(self.context(), run_id)

await plan_trace_monitor.complete_pipeline(result_context)
await plan_trace_monitor.score_pipeline(result_context)

plan_trace_monitor.reset()

class PlanTraceMonitor:
    """Monitors pipeline execution and creates PlanTraces for self-improvement.
    
    This component handles all PlanTrace-related functionality, keeping the Supervisor clean.
    It creates PlanTraces at pipeline start, tracks stage execution, and scores completed traces.
    """

    def __init__(self, cfg: Dict, memory, logger):
        self.cfg = cfg
        self.memory = memory
        self.logger = logger
        self.current_plan_trace: Optional[PlanTrace] = None
        self.plan_trace_scorer = PlanTraceScorerAgent(cfg, memory, logger)
        self.stage_start_times: Dict[int, float] = {}
        
        self.logger.log("PlanTraceMonitorInitialized", {
            "cfg_keys": list(cfg.keys())
        })
    
    def start_pipeline(self, context: Dict, pipeline_run_id: str) -> None:
        """Create PlanTrace when pipeline starts"""
        goal = context.get("goal", {})
        essential_config = {
            k: v for k, v in OmegaConf.to_container(self.cfg, resolve=True).items()
            if k in ["pipeline", "model", "scorer", "dimensions", "scorer_types"]
        }
        
        # Create PlanTrace for this pipeline execution
        self.current_plan_trace = PlanTrace(
            trace_id=str(pipeline_run_id),  # Use pipeline_run_id as trace_id
            goal_id=goal.get("id"),
            goal_text=goal.get("goal_text", ""),
            plan_signature=self._generate_plan_signature(context),
            input_data=self._extract_input_data(context),
            final_output_text="",
            execution_steps=[],
            target_epistemic_quality=None,
            target_epistemic_quality_source=None,
            extra_data={
                "agent_name": "PlanTraceMonitor",
                "started_at": time.time(),
                "pipeline_run_id": pipeline_run_id,
                "pipeline_config": essential_config
            }
        )
        
        # Log PlanTrace creation
        self.logger.log("PlanTraceCreated", {
            "trace_id": pipeline_run_id,
            "goal_id": goal.get("id"),
            "goal_text": (goal.get("goal_text", "")[:100] + "...") if goal.get("goal_text") else None
        })
    
    def _generate_plan_signature(self, context: Dict) -> str:
        """Generate a signature identifying this pipeline configuration"""
        pipeline = context.get("pipeline", [])
        return f"{'_'.join(pipeline)}"
    
    def _extract_input_data(self, context: Dict) -> Dict:
        """Extract relevant input data for the PlanTrace"""
        # Only capture essential input data, not the entire context
        return {
            "input_keys": list(context.keys()),
            "goal_id": context.get("goal", {}).get("id"),
            "goal_text_preview": (context.get("goal", {}).get("goal_text", "")[:100] + "...")
                if context.get("goal", {}).get("goal_text") else None
        }
    
    def start_stage(self, stage_name: str, context: Dict, stage_idx: int) -> None:
        """Create ExecutionStep when stage starts"""
        if not self.current_plan_trace:
            return
            
        # Record start time
        self.stage_start_times[stage_idx] = time.time()
        
        # Create step ID
        step_id = f"{self.current_plan_trace.trace_id}_step_{stage_idx + 1}"
        
        # Create step description
        description = f"Stage {stage_idx + 1}: {stage_name}"
        
        # Extract input data (simplified)
        input_preview = "Context keys: " + ", ".join(list(context.keys())[:3])
        if len(context.keys()) > 3:
            input_preview += f" + {len(context.keys()) - 3} more"
        
        # Create ExecutionStep
        execution_step = ExecutionStep(
            step_id=step_id,
            step_order=stage_idx + 1,
            step_type=stage_name,
            description=description,
            input_text=input_preview,
            output_text="",
            agent_name=stage_name,
            start_time=time.time(),
            error=None,
            scores=None
        )
        
        # Add to PlanTrace
        self.current_plan_trace.execution_steps.append(execution_step)
        
        # Log stage start
        self.logger.log("PipelineStageStarted", {
            "trace_id": self.current_plan_trace.trace_id,
            "stage_idx": stage_idx + 1,
            "stage_name": stage_name
        })
    
    def complete_stage(self, stage_name: str, context: Dict, stage_idx: int) -> None:
        """Update ExecutionStep when stage completes"""
        if not self.current_plan_trace or stage_idx >= len(self.current_plan_trace.execution_steps):
            return
            
        # Calculate duration
        start_time = self.stage_start_times.get(stage_idx, time.time())
        duration = time.time() - start_time
        
        # Update the current step
        step = self.current_plan_trace.execution_steps[stage_idx]
        step.end_time = time.time()
        step.duration = duration
        
        # Capture output preview
        output_keys = list(context.keys())
        output_preview = "Context keys: " + ", ".join(output_keys[:3])
        if len(output_keys) > 3:
            output_preview += f" + {len(output_keys) - 3} more"
        
        step.output_text = output_preview
        step.output_keys = output_keys
        step.output_size = len(str(context))
        
        # Log stage completion
        self.logger.log("PipelineStageCompleted", {
            "trace_id": self.current_plan_trace.trace_id,
            "stage_idx": stage_idx + 1,
            "stage_name": stage_name,
            "stage_time": duration,
            "output_keys": output_keys
        })
    
    def handle_stage_error(self, stage_name: str, error: Exception, stage_idx: int) -> None:
        """Update ExecutionStep when stage errors"""
        if not self.current_plan_trace or stage_idx >= len(self.current_plan_trace.execution_steps):
            return
            
        # Calculate duration
        start_time = self.stage_start_times.get(stage_idx, time.time())
        duration = time.time() - start_time
        
        # Update the current step with error information
        step = self.current_plan_trace.execution_steps[stage_idx]
        step.end_time = time.time()
        step.duration = duration
        step.error = {
            "type": type(error).__name__,
            "message": str(error),
            "traceback": traceback.format_exc()
        }
        
        # Log error
        self.logger.log("PipelineStageError", {
            "trace_id": self.current_plan_trace.trace_id,
            "stage_idx": stage_idx + 1,
            "stage_name": stage_name,
            "error_type": type(error).__name__,
            "error_message": str(error),
            "stage_duration": duration
        })
    
    @time_function()
    async def complete_pipeline(self, context: Dict) -> None:
        """Complete the PlanTrace when pipeline ends"""
        if not self.current_plan_trace:
            return
            
        # Set final output text
        final_output = context.get("final_output", "")
        if isinstance(final_output, str):
            self.current_plan_trace.final_output_text = (
                final_output[:1000] + "..." if len(final_output) > 1000 else final_output
            )
        elif isinstance(final_output, dict):
            self.current_plan_trace.final_output_text = str(final_output)[:1000] + "..."
        else:
            self.current_plan_trace.final_output_text = str(final_output)[:1000] + "..."
        
        # Set completion time
        self.current_plan_trace.extra_data["completed_at"] = time.time()
        
        # Calculate total pipeline time
        start_time = self.current_plan_trace.extra_data.get("started_at", time.time())
        self.current_plan_trace.extra_data["total_time"] = time.time() - start_time
        
        # Store in memory
        try:
            self.memory.plan_traces.add(self.current_plan_trace)
            self.logger.log("PlanTraceStored", {
                "trace_id": self.current_plan_trace.trace_id,
                "step_count": len(self.current_plan_trace.execution_steps)
            })
        except Exception as e:
            self.logger.log("PlanTraceStorageError", {
                "trace_id": self.current_plan_trace.trace_id,
                "error": str(e)
            })
        
        self.logger.log("PlanTraceCompleted", {
            "trace_id": self.current_plan_trace.trace_id,
            "step_count": len(self.current_plan_trace.execution_steps),
            "total_time": self.current_plan_trace.extra_data["total_time"]
        })

    @time_function()
    async def score_pipeline(self, context: Dict) -> None:
        """Score the completed PlanTrace"""
        if not self.current_plan_trace:
            return
            
        try:
            # Run PlanTraceScorerAgent
            scoring_context = {
                "plan_traces": [self.current_plan_trace],
                "goal": context.get("goal", {})
            }
            
            # Score the PlanTrace
            scored_context = await self.plan_trace_scorer.run(scoring_context)
            
            # Update PlanTrace with scores
            self.current_plan_trace.step_scores = scored_context.get("step_scores", [])
            self.current_plan_trace.pipeline_score = scored_context.get("pipeline_score", {})
            self.current_plan_trace.mars_analysis = scored_context.get("mars_analysis", {})
            
            # Update in memory
            self.memory.plan_traces.update(self.current_plan_trace)
            
            self.logger.log("PlanTraceScored", {
                "trace_id": self.current_plan_trace.trace_id,
                "step_count": len(self.current_plan_trace.execution_steps),
                "pipeline_score": scored_context.get("pipeline_score", {})
            })
        except Exception as e:
            self.logger.log("PlanTraceScoringError", {
                "trace_id": self.current_plan_trace.trace_id,
                "error": str(e),
                "traceback": traceback.format_exc()
            })
    
    def handle_pipeline_error(self, error: Exception, context: Dict) -> None:
        """Handle errors that occur during pipeline execution"""
        if not self.current_plan_trace:
            return
            
        # Update PlanTrace with error information
        self.current_plan_trace.final_output_text = f"Pipeline failed: {str(error)}"
        self.current_plan_trace.extra_data["error"] = {
            "type": type(error).__name__,
            "message": str(error),
            "traceback": traceback.format_exc()
        }
        self.current_plan_trace.extra_data["completed_at"] = time.time()
        
        # Store in memory
        try:
            self.memory.plan_traces.add(self.current_plan_trace)
        except Exception as e:
            self.logger.log("PlanTraceSaveError", {
                "trace_id": self.current_plan_trace.trace_id,
                "error": str(e)
            })
        
        self.logger.log("PlanTraceError", {
            "trace_id": self.current_plan_trace.trace_id,
            "error_type": type(error).__name__,
            "error_message": str(error)
        })
    
    def reset(self) -> None:
        """Reset the monitor for the next pipeline"""
        self.current_plan_trace = None
        self.stage_start_times = {}

{
  "dimension": "reasoning_quality",
  "score": 0.82,
  "attributes": {
    "q_value": 0.76,
    "v_value": 0.79,
    "uncertainty": 0.12,
    "advantage": 0.03
  }
}

class PlanTraceScorerAgent(BaseAgent):
    """
    Scores pipeline execution traces at multiple levels:
    - Individual execution steps (granular reasoning quality)
    - Complete pipeline execution (overall quality)
    - Step relationships and flow patterns
    
    Uses HRM as primary reasoning quality scorer with MARS meta-analysis
    to enable self-tuning of pipeline execution patterns.
    """
    
    def __init__(self, cfg, memory=None, logger=None):
        super().__init__(cfg, memory, logger)
        self.dimensions = cfg.get("dimensions", [])
        self.include_mars = cfg.get("include_mars", True)
        
        # Configure which scorers to use
        self.scorer_types = cfg.get("scorer_types", [
            "hrm", "sicql", "contrastive_ranker"
        ])
        
        # Initialize scorers
        self.scorers = self._initialize_scorers()
        
        # Initialize MARS calculator
        dimension_config = cfg.get("dimension_config", {})
        self.mars_calculator = MARSCalculator(dimension_config)
        
        # Pattern extraction parameters
        self.high_agreement_threshold = cfg.get("high_agreement_threshold", 0.8)
        self.low_uncertainty_threshold = cfg.get("low_uncertainty_threshold", 0.2)
        self.pattern_min_count = cfg.get("pattern_min_count", 3)
        
        self.export_dir = cfg.get("export_dir", "exports/plan_traces")

        self.logger.log("PlanTraceScorerInitialized", {
            "dimensions": self.dimensions,
            "scorers": self.scorer_types,
            "high_agreement_threshold": self.high_agreement_threshold,
            "low_uncertainty_threshold": self.low_uncertainty_threshold
        })

    def _initialize_scorers(self) -> Dict[str, Any]:
        """Initialize all configured scorers"""
        scorers = {}
        
        if "hrm" in self.scorer_types:
            scorers["hrm"] = HRMScorer(self.cfg.scorer.hrm, memory=self.memory, logger=self.logger)
        if "sicql" in self.scorer_types:
            scorers["sicql"] = SICQLScorer(self.cfg.scorer.sicql, memory=self.memory, logger=self.logger)
        if "contrastive_ranker" in self.scorer_types:
            scorers["contrastive_ranker"] = ContrastiveRankerScorer(
                self.cfg.scorer.contrastive_ranker, memory=self.memory, logger=self.logger
            )
            
        return scorers

    async def run(self, context: dict) -> dict:
        """Score pipeline execution traces with self-tuning capability"""
        start_time = time.time()
        
        # --- 1. Load and Prepare Training Data
        raw_traces_data = context.get("plan_traces", [])
        if not raw_traces_data:
            # If no traces are provided, try loading from export directory
            self.logger.log(
                "EpistemicPlanHRMTrainingNoTraces",
                {
                    "message": "No plan traces found in context['plan_traces']. Attempting to load from export directory.",
                    "export_dir": self.export_dir,
                }, 
            ) 
            raw_traces_data = load_plan_traces_from_export_dir(self.export_dir)

        for raw_trace in raw_traces_data:
            # Convert raw trace data to PlanTrace object
            if isinstance(raw_trace, dict):
                # If raw_trace is a dict, convert it to PlanTrace
                plan_trace = PlanTrace.from_dict(raw_trace)
            elif isinstance(raw_trace, PlanTrace):
                plan_trace = raw_trace
            if not plan_trace.execution_steps:
                self.logger.log("EmptyPlanTrace", {"trace_id": plan_trace.trace_id})
                continue
            
            # Score individual execution steps
            step_results = []
            all_step_bundles = {}  # step_id -> ScoreBundle
            
            # Process steps with progress tracking
            pbar = tqdm(
                plan_trace.execution_steps,
                desc="Scoring Steps",
                disable=not self.cfg.get("progress", True)
            )
            
            for step in pbar:
                # Create scorable for this step
                scorable = ScorableFactory.from_plan_trace(
                    plan_trace, 
                    mode="single_step",
                    step=step
                )
                
                # Score the step
                step_bundle = self._score_scorable(scorable, plan_trace.goal_text)
                all_step_bundles[step.step_id] = step_bundle
                
                # Prepare results for reporting
                step_scores = {
                    dim: {
                        "score": result.score,
                        "rationale": result.rationale,
                        "source": result.source
                    } for dim, result in step_bundle.results.items()
                }
                
                step_results.append({
                    "step_id": step.step_id,
                    "step_order": step.step_order,
                    "step_type": step.step_type,
                    "agent": step.agent_name,
                    "description": step.description,
                    "scores": step_scores
                })
                
                # Update progress bar
                pbar.set_postfix({"steps": f"{len(step_results)}/{len(plan_trace.execution_steps)}"})
            
            # Score the complete pipeline
            full_scorable = ScorableFactory.from_plan_trace(plan_trace, mode="full_trace")
            full_bundle = self._score_scorable(full_scorable, plan_trace.goal_text)
            
            # Create ScoreCorpus for MARS analysis
            corpus = ScoreCorpus(bundles=all_step_bundles)
            
            # Run MARS analysis across all steps
            mars_results = {}
            if self.include_mars:
                mars_results = self.mars_calculator.calculate(corpus)
                
                # Log MARS analysis metrics
                self.logger.log("MARSAnalysisCompleted", {
                    "trace_id": plan_trace.trace_id,
                    "step_count": len(plan_trace.execution_steps),
                    "dimensions": list(mars_results.keys()),
                    "overall_agreement": self.mars_calculator.get_aggregate_score(mars_results)
                })
                
                # Identify high-quality patterns for self-tuning
                self._update_self_tuning_patterns(corpus, mars_results, plan_trace)
            
            # Save results to context
            context["step_scores"] = step_results
            context["pipeline_score"] = {dim: result.score for dim, result in full_bundle.results.items()}
            context["mars_analysis"] = mars_results
            context["scoring_time"] = time.time() - start_time
            context["score_corpus"] = corpus.to_dict()
            
            self.logger.log("PlanTraceScoringComplete", {
                "trace_id": plan_trace.trace_id,
                "step_count": len(plan_trace.execution_steps),
                "dimensions": self.dimensions,
                "scorers": len(self.scorers)
            })
            
            return context

    def _score_scorable(self, scorable, goal_text) -> ScoreBundle:
        """Score a single scorable with all configured scorers"""
        score_results = {}
        
        for scorer_name, scorer in self.scorers.items():
            try:
                # Score with this scorer
                score_bundle = scorer.score(
                    goal={"goal_text": goal_text},
                    scorable=scorable,
                    dimensions=self.dimensions,
                )
                
                # Add results (prefer HRM for reasoning quality)
                for dim, result in score_bundle.results.items():
                    # If HRM is available for reasoning quality, prefer it
                    if dim == "reasoning_quality" and scorer_name == "hrm":
                        score_results[dim] = result
                    # For other dimensions, use the first available scorer
                    elif dim not in score_results:
                        score_results[dim] = result
            
            except Exception as e:
                self.logger.log("ScorerError", {
                    "scorer": scorer_name,
                    "error": str(e)
                })
                continue
        
        return ScoreBundle(results=score_results)

    def _update_self_tuning_patterns(self, corpus: ScoreCorpus, 
                                  mars_results: Dict, 
                                  plan_trace: PlanTrace):
        """Update self-tuning patterns based on high-quality pipeline executions"""
        # Find high-quality steps (high agreement, low uncertainty)
        high_quality_steps = []
        pattern_metrics = {}
        
        for dimension, results in mars_results.items():
            # Get steps with high agreement and low uncertainty
            agreement_threshold = results.get("agreement_score", 0.0) * 0.9
            high_agreement_steps = corpus.get_high_disagreement_scorables(
                dimension, 
                threshold=1.0 - agreement_threshold
            )
            
            # Get steps with low uncertainty
            low_uncertainty_steps = []
            if "uncertainty" in corpus.metrics:
                uncertainty_matrix = corpus.get_metric_matrix(dimension, "uncertainty")
                low_uncertainty_steps = uncertainty_matrix[
                    uncertainty_matrix.mean(axis=1) < self.low_uncertainty_threshold
                ].index.tolist()
            
            # Intersection: steps that are both high agreement AND low uncertainty
            high_quality_for_dim = list(set(high_agreement_steps) & set(low_uncertainty_steps))
            high_quality_steps.extend(high_quality_for_dim)
            
            # Track metrics for pattern extraction
            pattern_metrics[dimension] = {
                "high_agreement_steps": high_agreement_steps,
                "low_uncertainty_steps": low_uncertainty_steps,
                "high_quality_steps": high_quality_for_dim
            }
        
        # Remove duplicates
        high_quality_steps = list(set(high_quality_steps))
        
        if high_quality_steps:
            # Extract patterns from high-quality steps
            patterns = self._extract_patterns(high_quality_steps, corpus, plan_trace)
            
            # Store patterns for future pipeline construction
            self.memory.pipeline_patterns.store_patterns(patterns)
            
            self.logger.log("SelfTuningPatternsUpdated", {
                "pattern_count": len(patterns),
                "step_count": len(high_quality_steps),
                "trace_id": plan_trace.trace_id
            })
            
            # Generate recommendations for immediate improvement
            recommendations = self._generate_immediate_recommendations(
                corpus, mars_results, high_quality_steps
            )
            self.logger.log("SelfTuningRecommendations", {
                "trace_id": plan_trace.trace_id,
                "recommendations": recommendations
            })

    def _extract_patterns(self, step_ids: List[str], 
                         corpus: ScoreCorpus, 
                         plan_trace: PlanTrace) -> List[Dict]:
        """Extract patterns from high-quality steps for self-tuning"""
        patterns = []
        
        # Map step IDs to step objects for quick lookup
        step_map = {step.step_id: step for step in plan_trace.execution_steps}
        
        for step_id in step_ids:
            step = step_map.get(step_id)
            if not step:
                continue
                
            # Extract pattern features
            pattern = {
                "step_type": step.step_type,
                "agent": step.agent_name,
                "input_type": step.input_type,
                "output_type": step.output_type,
                "success_metrics": {}
            }
            
            # Add success metrics from MARS analysis
            for dimension in self.dimensions:
                # Get metric values for this dimension
                uncertainty_values = corpus.get_metric_values(dimension, "hrm", ["uncertainty"])
                if step_id in uncertainty_values["uncertainty"]:
                    pattern["success_metrics"][dimension] = {
                        "uncertainty": uncertainty_values["uncertainty"][step_id],
                        "agreement_score": corpus.get_dimension_matrix(dimension).std().mean()
                    }
            
            # Add contextual information
            pattern["context"] = {
                "previous_step_type": self._get_previous_step_type(step, plan_trace),
                "next_step_type": self._get_next_step_type(step, plan_trace),
                "position_in_pipeline": step.step_order / len(plan_trace.execution_steps)
            }
            
            patterns.append(pattern)
        
        return patterns

    def _get_previous_step_type(self, step: ExecutionStep, plan_trace: PlanTrace) -> Optional[str]:
        """Get the type of the previous step in the pipeline"""
        if step.step_order > 1:
            prev_step = next(
                (s for s in plan_trace.execution_steps if s.step_order == step.step_order - 1), 
                None
            )
            return prev_step.step_type if prev_step else None
        return None

    def _get_next_step_type(self, step: ExecutionStep, plan_trace: PlanTrace) -> Optional[str]:
        """Get the type of the next step in the pipeline"""
        if step.step_order < len(plan_trace.execution_steps):
            next_step = next(
                (s for s in plan_trace.execution_steps if s.step_order == step.step_order + 1), 
                None
            )
            return next_step.step_type if next_step else None
        return None

    def _generate_immediate_recommendations(self, 
                                         corpus: ScoreCorpus, 
                                         mars_results: Dict, 
                                         high_quality_steps: List[str]) -> List[str]:
        """Generate recommendations for immediate pipeline improvement"""
        recommendations = []
        
        # 1. Identify problematic dimensions
        for dimension, results in mars_results.items():
            if results["agreement_score"] < 0.7:
                recommendations.append(
                    f"⚠️ Low agreement in {dimension} scoring. "
                    "Consider reviewing pipeline steps for consistency."
                )
            
            if results["high_disagreement"]:
                primary_conflict = results["primary_conflict"]
                recommendations.append(
                    f"⚠️ Significant conflict between {primary_conflict[0]} and {primary_conflict[1]} "
                    f"in {dimension} scoring (Δ={results['delta']:.3f}). "
                    "This may indicate ambiguous pipeline steps."
                )
        
        # 2. Identify unreliable scorers
        scorer_reliability = {}
        for dimension in self.dimensions:
            reliability = corpus.analyze_scorer_reliability(dimension)
            for scorer, score in reliability.items():
                if scorer not in scorer_reliability:
                    scorer_reliability[scorer] = []
                scorer_reliability[scorer].append(score)
        
        # Average reliability across dimensions
        avg_reliability = {
            scorer: mean(scores) for scorer, scores in scorer_reliability.items()
        }
        
        # Find least reliable scorer
        if avg_reliability:
            least_reliable = min(avg_reliability, key=avg_reliability.get)
            if avg_reliability[least_reliable] < 0.6:
                recommendations.append(
                    f"⚠️ {least_reliable} shows low reliability across dimensions. "
                    "Consider retraining or adjusting its configuration."
                )
        
        # 3. Identify opportunities for improvement
        if high_quality_steps:
            # Find common patterns in high-quality steps
            step_types = [step.step_type for step_id, step in self._get_steps_by_id(high_quality_steps)]
            common_step_type = max(set(step_types), key=step_types.count)
            
            recommendations.append(
                f"💡 High-quality steps frequently use {common_step_type} pattern. "
                "Consider applying this pattern to similar pipeline sections."
            )
        
        return recommendations

    def _get_steps_by_id(self, step_ids: List[str]) -> Dict[str, ExecutionStep]:
        """Get step objects by their IDs"""
        # This would be implemented based on your memory structure
        # For now, return a mock implementation
        return {step_id: ExecutionStep(
            step_id=step_id,
            step_order=0,
            step_type="unknown",
            description="",
            output_text="",
            scores=None
        ) for step_id in step_ids}

class PreferenceRanker(nn.Module):
    """Siamese network architecture (must match trainer)"""
    def __init__(self, embedding_dim=1024, hidden_dim=256):
        super().__init__()
        self.encoder = nn.Sequential(
            nn.Linear(embedding_dim, hidden_dim),
            nn.ReLU(),
            nn.Dropout(0.2),
            nn.Linear(hidden_dim, hidden_dim)
        )
        self.comparator = nn.Sequential(
            nn.Linear(hidden_dim * 2, hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim, 1)
        )

    def forward(self, emb_a, emb_b):
        feat_a = self.encoder(emb_a)
        feat_b = self.encoder(emb_b)
        combined = torch.cat([feat_a, feat_b], dim=1)
        return self.comparator(combined).squeeze(1)


class ContrastiveRankerScorer(BaseScorer):
    def __init__(self, cfg: dict, memory, logger):
        super().__init__(cfg, memory, logger)
        self.model_type = "contrastive_ranker"
        self.models = {}        # dim -> (scaler, model)
        self.tuners = {}        # dim -> RegressionTuner
        self.metas = {}         # dim -> model metadata
        self.baselines = {}     # dim -> baseline embedding
        self._load_all_dimensions()

    def _load_all_dimensions(self):
        """Preload all dimension models with baseline caching"""
        for dim in tqdm(self.dimensions, desc="Loading contrastive rankers"):
            locator = self.get_locator(dim)
            
            # Load metadata first
            meta = load_json(locator.meta_file())
            self.metas[dim] = meta
            
            # Load scaler
            scaler = load(locator.scaler_file())
            
            # Initialize model with correct dimensions
            input_dim = scaler.mean_.shape[0]
            model = PreferenceRanker(
                embedding_dim=input_dim,
                hidden_dim=meta["hidden_dim"]
            )
            
            # Load weights
            model.load_state_dict(torch.load(locator.model_file(suffix=".pt")))
            model.eval()
            self.models[dim] = (scaler, model)
            
            # Load tuner
            tuner = RegressionTuner(dimension=dim, logger=self.logger)
            tuner.load(locator.tuner_file())
            self.tuners[dim] = tuner
            
            # Precompute baseline embedding
            baseline_text = meta["baseline"]
            baseline_emb = np.array(self.memory.embedding.get_or_create(baseline_text))
            self.baselines[dim] = baseline_emb

    def score(self, goal: dict, scorable: Scorable, dimensions: list[str]) -> ScoreBundle:
        """Generate absolute scores via baseline comparison"""
        goal_text = goal.get("goal_text", "")
        ctx_emb = np.array(self.memory.embedding.get_or_create(goal_text))
        doc_emb = np.array(self.memory.embedding.get_or_create(scorable.text))
        
        results = {}
        for dim in dimensions:
            scaler, model = self.models[dim]
            tuner = self.tuners[dim]
            meta = self.metas[dim]
            baseline_emb = self.baselines[dim]
            
            # Create comparison inputs
            input_doc = np.concatenate([ctx_emb, doc_emb])
            input_baseline = np.concatenate([ctx_emb, baseline_emb])
            
            # Scale inputs
            input_doc_scaled = scaler.transform(input_doc.reshape(1, -1))
            input_baseline_scaled = scaler.transform(input_baseline.reshape(1, -1))
            
            # Convert to tensors
            doc_tensor = torch.tensor(input_doc_scaled, dtype=torch.float32)
            baseline_tensor = torch.tensor(input_baseline_scaled, dtype=torch.float32)
            
            # Get preference score
            with torch.no_grad():
                raw_score = model(doc_tensor, baseline_tensor).item()
            
            # Calibrate to absolute score
            tuned_score = tuner.transform(raw_score)
            final_score = max(min(tuned_score, meta["max_score"]), meta["min_score"])

            attributes = {
                "raw_score": round(raw_score, 4),
                "normalized_score": round(tuned_score, 4),
                "final_score": final_score,
                "energy": raw_score,  # Using raw_score as energy
            }

            results[dim] = ScoreResult(
                dimension=dim,
                score=final_score,
                rationale=f"PrefScore(raw={raw_score:.4f}, tuned={tuned_score:.2f})",
                weight=1.0,
                source=self.model_type,
                attributes=attributes,
                )
        
        return ScoreBundle(results=results)

input_a = np.concatenate([ctx_emb, a_emb])
input_b = np.concatenate([ctx_emb, b_emb])
y.append(1 if pair["preferred"] == "A" else 0)

for epoch in range(self.epochs):
    for xa, xb, labels in dataloader:
        logits = model(xa, xb)
        loss = criterion(logits, labels)
        loss.backward()
        optimizer.step()

logits = model(batch_tensor, baseline_tensor)
tuner.train_single(float(logits[j]), abs_score)

# Find high-uncertainty steps across all pipelines
matrix = corpus.get_metric_matrix("reasoning_quality", "uncertainty")
high_uncertainty = matrix[matrix > 0.3]

ScoreResult(
  dimension="clarity",
  score=0.82,
  source="sicql",
  attributes={
    "energy": -3.12,
    "uncertainty": 0.21,
    "advantage": 0.44
  }
)

corpus.to_tensor()
# Returns a shape like: [scorables × dimensions × scorers × metrics]

# Get all uncertainty values across reasoning quality dimension
uncertainty_matrix = corpus.get_metric_matrix("reasoning_quality", 
           "uncertainty")

# Find documents with high uncertainty
high_uncertainty_docs = uncertainty_matrix[
    uncertainty_matrix.mean(axis=1) > 0.3
].index.tolist()

# Analyze which step type correlates with high uncertainty
step_types = []
for doc_id in high_uncertainty_docs:
    for step in corpus.bundles[doc_id].execution_steps:
        step_types.append(step.step_type)
        
problematic_step = max(set(step_types), key=step_types.count)

[scorables × dimensions × scorers × metrics]

class ScoreCorpus:
    """
    Collection of ScoreBundles across multiple documents/scorables for tensor-based analysis.
    
    This class implements the true 4D tensor structure [scorables × dimensions × scorers × metrics]
    that enables powerful slicing and analysis capabilities.
    
    Key features:
    - Convert to 4D tensor for ML integration
    - Slice by metric type (energy, uncertainty, etc.)
    - Analyze scoring agreement patterns
    - Identify systematic scorer biases
    - Support for MARS calculator integration
    """
    
    def __init__(self, bundles: Dict[str, ScoreBundle], meta: Dict[str, Any] = None):
        """
        Initialize a ScoreCorpus from a collection of ScoreBundles.
        
        Args:
            bundles: Dictionary mapping scorable IDs to ScoreBundles
            meta: Optional metadata about the corpus
        """
        self.bundles = bundles
        self.meta = meta or {}
        self._dimensions = None
        self._scorers = None
        self._metrics = None
        self._dimension_matrix_cache = {}
        self._metric_matrix_cache = {}
    
    @property
    def dimensions(self) -> List[str]:
        """Get all dimensions present across bundles"""
        if self._dimensions is None:
            self._dimensions = self._discover_dimensions()
        return self._dimensions
    
    @property
    def scorers(self) -> List[str]:
        """Get all scorers present across bundles"""
        if self._scorers is None:
            self._scorers = self._discover_scorers()
        return self._scorers
    
    @property
    def metrics(self) -> Set[str]:
        """Get all metrics present across bundles (including 'score')"""
        if self._metrics is None:
            self._metrics = self._discover_metrics()
        return self._metrics
    
    def _discover_dimensions(self) -> List[str]:
        """Discover all dimensions present in the corpus"""
        dimensions = set()
        for bundle in self.bundles.values():
            dimensions.update(bundle.results.keys())
        return sorted(list(dimensions))
    
    def _discover_scorers(self) -> List[str]:
        """Discover all scorers present in the corpus"""
        scorers = set()
        for bundle in self.bundles.values():
            for result in bundle.results.values():
                scorers.add(result.source)
        return sorted(list(scorers))
    
    def _discover_metrics(self) -> Set[str]:
        """Discover all metrics present in the corpus"""
        metrics = {"score"}  # Always include the core score
        for bundle in self.bundles.values():
            for result in bundle.results.values():
                if result.attributes:
                    metrics.update(result.attributes.keys())
        return metrics
    
    def get_dimension_matrix(self, dimension: str) -> pd.DataFrame:
        """
        Get scores as a DataFrame: [scorables × scorers]
        
        Args:
            dimension: The dimension to extract
            
        Returns:
            DataFrame where rows are scorables and columns are scorers
        """
        # Check cache first
        if dimension in self._dimension_matrix_cache:
            return self._dimension_matrix_cache[dimension]
        
        # Build matrix
        data = {}
        for scorable_id, bundle in self.bundles.items():
            if dimension in bundle.results:
                result = bundle.results[dimension]
                data[scorable_id] = {result.source: result.score}
        
        # Create DataFrame
        df = pd.DataFrame.from_dict(data, orient='index')
        
        # Ensure all scorers are present as columns
        for scorer in self.scorers:
            if scorer not in df.columns:
                df[scorer] = np.nan
        
        # Sort columns by scorers list
        df = df[self.scorers]
        
        # Cache result
        self._dimension_matrix_cache[dimension] = df
        
        return df
    
    def get_metric_matrix(self, dimension: str, metric: str) -> pd.DataFrame:
        """
        Get a specific metric as a DataFrame: [scorables × scorers]
        
        Args:
            dimension: The dimension to extract
            metric: The metric to extract (e.g., "uncertainty", "q_value")
            
        Returns:
            DataFrame where rows are scorables and columns are scorers
        """
        # Check cache first
        cache_key = (dimension, metric)
        if cache_key in self._metric_matrix_cache:
            return self._metric_matrix_cache[cache_key]
        
        # Build matrix
        data = {}
        for scorable_id, bundle in self.bundles.items():
            if dimension in bundle.results:
                result = bundle.results[dimension]
                value = result.attributes.get(metric, np.nan) if result.attributes else np.nan
                data[scorable_id] = {result.source: value}
        
        # Create DataFrame
        df = pd.DataFrame.from_dict(data, orient='index')
        
        # Ensure all scorers are present as columns
        for scorer in self.scorers:
            if scorer not in df.columns:
                df[scorer] = np.nan
        
        # Sort columns by scorers list
        df = df[self.scorers]
        
        # Cache result
        self._metric_matrix_cache[cache_key] = df
        
        return df
    
    def get_metric_values(self, dimension: str, scorer: str, metrics: List[str]) -> Dict[str, List[Any]]:
        """
        Get values for specific metrics across all scorables for a dimension and scorer.
        
        Args:
            dimension: The dimension to extract
            scorer: The scorer to extract
            metrics: List of metrics to extract
            
        Returns:
            Dictionary mapping metric names to lists of values
        """
        results = {metric: [] for metric in metrics}
        
        for bundle in self.bundles.values():
            if dimension in bundle.results:
                result = bundle.results[dimension]
                if result.source == scorer:
                    for metric in metrics:
                        if result.attributes and metric in result.attributes:
                            results[metric].append(result.attributes[metric])
                        else:
                            results[metric].append(None)
        
        return results
    
    def get_all_metric_values(self, dimension: str, metrics: List[str]) -> Dict[str, List[Any]]:
        """
        Get values for specific metrics across all scorables and scorers for a dimension.
        
        Args:
            dimension: The dimension to extract
            metrics: List of metrics to extract
            
        Returns:
            Dictionary mapping metric names to lists of values
        """
        results = {metric: [] for metric in metrics}
        
        for bundle in self.bundles.values():
            if dimension in bundle.results:
                result = bundle.results[dimension]
                for metric in metrics:
                    if result.attributes and metric in result.attributes:
                        results[metric].append(result.attributes[metric])
                    else:
                        results[metric].append(None)
        
        return results
    
    def to_tensor(self, dimensions: List[str] = None, 
                 scorers: List[str] = None, 
                 metrics: List[str] = None) -> np.ndarray:
        """
        Convert to 4D tensor: [scorables × dimensions × scorers × metrics]
        
        Args:
            dimensions: Optional list of dimensions to include (defaults to all)
            scorers: Optional list of scorers to include (defaults to all)
            metrics: Optional list of metrics to include (defaults to all)
            
        Returns:
            4D numpy array of shape (n_scorables, n_dimensions, n_scorers, n_metrics)
        """
        # Default to all dimensions/scorers/metrics if not specified
        dimensions = dimensions or self.dimensions
        scorers = scorers or self.scorers
        metrics = metrics or list(self.metrics)
        
        # Create tensor with zeros
        tensor = np.zeros((len(self.bundles), len(dimensions), len(scorers), len(metrics)))
        
        # Fill tensor with values
        for scorable_idx, (scorable_id, bundle) in enumerate(self.bundles.items()):
            for dim_idx, dimension in enumerate(dimensions):
                if dimension in bundle.results:
                    result = bundle.results[dimension]
                    scorer_idx = scorers.index(result.source)
                    
                    # Fill in metric values
                    for metric_idx, metric in enumerate(metrics):
                        if metric == "score":
                            tensor[scorable_idx, dim_idx, scorer_idx, metric_idx] = result.score
                        elif result.attributes and metric in result.attributes:
                            try:
                                tensor[scorable_idx, dim_idx, scorer_idx, metric_idx] = float(result.attributes[metric])
                            except (TypeError, ValueError):
                                tensor[scorable_idx, dim_idx, scorer_idx, metric_idx] = 0.0
                        # Otherwise leave as 0.0
        
        return tensor
    
    def to_dataframe(self, dimensions: List[str] = None, 
                    scorers: List[str] = None, 
                    metrics: List[str] = None) -> pd.DataFrame:
        """
        Convert to multi-index DataFrame for analysis.
        
        The DataFrame will have:
        - Index: scorable IDs
        - Columns: MultiIndex of (dimension, scorer, metric)
        
        Args:
            dimensions: Optional list of dimensions to include (defaults to all)
            scorers: Optional list of scorers to include (defaults to all)
            metrics: Optional list of metrics to include (defaults to all)
            
        Returns:
            Multi-index DataFrame
        """
        # Default to all dimensions/scorers/metrics if not specified
        dimensions = dimensions or self.dimensions
        scorers = scorers or self.scorers
        metrics = metrics or list(self.metrics)
        
        # Create column index
        column_tuples = [(dim, scorer, metric) 
                        for dim in dimensions 
                        for scorer in scorers 
                        for metric in metrics]
        columns = pd.MultiIndex.from_tuples(column_tuples, 
                                         names=['dimension', 'scorer', 'metric'])
        
        # Create DataFrame
        df = pd.DataFrame(index=list(self.bundles.keys()), columns=columns)
        
        # Fill DataFrame
        for scorable_id, bundle in self.bundles.items():
            for dim in dimensions:
                if dim in bundle.results:
                    result = bundle.results[dim]
                    for metric in metrics:
                        if metric == "score":
                            value = result.score
                        elif result.attributes and metric in result.attributes:
                            value = result.attributes[metric]
                        else:
                            value = None
                        
                        df.loc[scorable_id, (dim, result.source, metric)] = value
        
        return df
    
    def analyze_scorer_reliability(self, dimension: str, 
                                 trust_reference: str = "llm") -> Dict[str, float]:
        """
        Analyze which scorers are most reliable for a dimension.
        
        Args:
            dimension: The dimension to analyze
            trust_reference: The scorer to use as gold standard
            
        Returns:
            Dictionary mapping scorers to reliability scores (higher = more reliable)
        """
        if trust_reference not in self.scorers:
            warnings.warn(f"Trust reference '{trust_reference}' not found. Using median scorer instead.")
            return self._analyze_scorer_consistency(dimension)
        
        # Get the document × scorer matrix
        matrix = self.get_dimension_matrix(dimension)
        
        # Calculate correlation with trust reference
        reliability = {}
        trust_scores = matrix[trust_reference]
        
        for scorer in self.scorers:
            if scorer == trust_reference:
                reliability[scorer] = 1.0  # Perfect correlation with itself
                continue
            
            # Calculate correlation
            valid_pairs = matrix[[scorer, trust_reference]].dropna()
            if len(valid_pairs) > 1:
                try:
                    corr = valid_pairs[scorer].corr(valid_pairs[trust_reference])
                    reliability[scorer] = float(corr) if not pd.isna(corr) else 0.0
                except:
                    reliability[scorer] = 0.0
            else:
                reliability[scorer] = 0.0
        
        return reliability
    
    def _analyze_scorer_consistency(self, dimension: str) -> Dict[str, float]:
        """Analyze scorer consistency when no trust reference is available"""
        matrix = self.get_dimension_matrix(dimension)
        scorer_std = matrix.std()
        max_std = scorer_std.max()
        
        # Higher reliability for lower standard deviation
        return {scorer: 1.0 - (std / max_std) if max_std > 0 else 1.0 
                for scorer, std in scorer_std.items()}
    
    def get_high_disagreement_scorables(self, dimension: str, 
                                     threshold: float = 0.15) -> List[str]:
        """
        Get scorables with high disagreement across scorers for a dimension.
        
        Args:
            dimension: The dimension to analyze
            threshold: Threshold for disagreement (standard deviation)
            
        Returns:
            List of scorable IDs with high disagreement
        """
        # Get the document × scorer matrix
        matrix = self.get_dimension_matrix(dimension)
        
        # Calculate disagreement per document (standard deviation across scorers)
        disagreement = matrix.std(axis=1)
        
        # Return scorables with disagreement above threshold
        return disagreement[disagreement > threshold].index.tolist()
    
    def get_outlier_scorables(self, dimension: str, scorer: str, 
                            threshold: float = 2.0) -> List[str]:
        """
        Get scorables where a specific scorer significantly differs from consensus.
        
        Args:
            dimension: The dimension to analyze
            scorer: The scorer to check
            threshold: Threshold in standard deviations
            
        Returns:
            List of scorable IDs where the scorer is an outlier
        """
        # Get the document × scorer matrix
        matrix = self.get_dimension_matrix(dimension)
        if scorer not in matrix.columns:
            return []
        
        # Calculate consensus (mean excluding the scorer)
        consensus = matrix.drop(columns=[scorer]).mean(axis=1)
        
        # Calculate difference from consensus
        diff = (matrix[scorer] - consensus).abs()
        std_dev = diff.std()
        
        # Return scorables where difference is above threshold
        if std_dev > 0:
            return diff[diff > threshold * std_dev].index.tolist()
        return []
    
    def get_metric_correlations(self, dimension: str, 
                              metrics: List[str] = None) -> Dict[Tuple[str, str], float]:
        """
        Get correlations between different metrics for a dimension.
        
        Args:
            dimension: The dimension to analyze
            metrics: Optional list of metrics to analyze (defaults to all)
            
        Returns:
            Dictionary mapping (metric1, metric2) to correlation coefficient
        """
        metrics = metrics or list(self.metrics - {"score"})
        if len(metrics) < 2:
            return {}
        
        # Get all metric matrices
        metric_matrices = {
            metric: self.get_metric_matrix(dimension, metric)
            for metric in metrics
        }
        
        # Calculate correlations
        correlations = {}
        for i in range(len(metrics)):
            for j in range(i+1, len(metrics)):
                metric1, metric2 = metrics[i], metrics[j]
                
                # Stack values
                values1 = []
                values2 = []
                for scorable_id in self.bundles.keys():
                    val1 = metric_matrices[metric1].loc.get(scorable_id, np.nan)
                    val2 = metric_matrices[metric2].loc.get(scorable_id, np.nan)
                    
                    # Skip if either value is NaN
                    if not pd.isna(val1) and not pd.isna(val2):
                        values1.append(val1)
                        values2.append(val2)
                
                # Calculate correlation
                if len(values1) > 1:
                    try:
                        corr = pd.Series(values1).corr(pd.Series(values2))
                        if not pd.isna(corr):
                            correlations[(metric1, metric2)] = float(corr)
                    except:
                        pass
        
        return correlations
    
    def find_metric_outliers(self, dimension: str, metric: str, 
                           threshold: float = 2.0) -> List[Tuple[str, float]]:
        """
        Find scorables with outlier values for a specific metric.
        
        Args:
            dimension: The dimension to analyze
            metric: The metric to check
            threshold: Threshold in standard deviations
            
        Returns:
            List of (scorable_id, z_score) tuples
        """
        # Get the metric matrix
        matrix = self.get_metric_matrix(dimension, metric)
        
        # Stack all values
        all_values = []
        for scorer in self.scorers:
            values = matrix[scorer].dropna().values
            all_values.extend(values)
        
        if not all_values:
            return []
        
        # Calculate mean and std
        mean_val = np.mean(all_values)
        std_val = np.std(all_values)
        
        if std_val == 0:
            return []
        
        # Find outliers
        outliers = []
        for scorable_id in self.bundles.keys():
            for scorer in self.scorers:
                value = matrix.loc.get((scorable_id, scorer), np.nan)
                if not pd.isna(value):
                    z_score = (value - mean_val) / std_val
                    if abs(z_score) > threshold:
                        outliers.append((scorable_id, z_score))
        
        # Sort by absolute z-score
        outliers.sort(key=lambda x: abs(x[1]), reverse=True)
        return outliers
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for serialization"""
        return {
            "scorable_ids": list(self.bundles.keys()),
            "dimensions": self.dimensions,
            "scorers": self.scorers,
            "metrics": list(self.metrics),
            "meta": self.meta
        }
    
    @classmethod
    def from_dict(cls, data: Dict[str, Any], 
                 bundles: Dict[str, ScoreBundle] = None) -> "ScoreCorpus":
        """Reconstruct from dictionary (with optional bundles)"""
        # If bundles are provided, filter to match scorable IDs
        if bundles:
            scorable_ids = data.get("scorable_ids", [])
            filtered_bundles = {k: v for k, v in bundles.items() if k in scorable_ids}
            return cls(bundles=filtered_bundles, meta=data.get("meta", {}))
        
        # Without bundles, just return empty corpus with metadata
        return cls(bundles={}, meta=data.get("meta", {}))
    
    def __len__(self) -> int:
        """Return number of scorables in the corpus"""
        return len(self.bundles)
    
    def __getitem__(self, scorable_id: str) -> ScoreBundle:
        """Get a specific ScoreBundle by scorable ID"""
        return self.bundles[scorable_id]
    
    def __iter__(self):
        """Iterate over scorables"""
        return iter(self.bundles.items())
    
    def __repr__(self):
        return (f"<ScoreCorpus(scorables={len(self.bundles)}, "
                f"dimensions={len(self.dimensions)}, "
                f"scorers={len(self.scorers)}, "
                f"metrics={len(self.metrics)})>")

[num_scorables, num_dimensions, num_scorers]

class ScoreAttributeORM(Base):
    id          # primary key
    score_id    # FK to ScoreORM
    key         # e.g. "energy", "certainty", "advantage"
    value       # stored as text, cast dynamically
    data_type   # e.g. "float", "json", "str"
    created_at  # timestamp

ScoreDeltaCalculator:
    def __init__(self, cfg: dict, memory, logger=None):
        self.cfg = cfg
        self.memory = memory
        self.logger = logger

    def log_score_delta(self, scorable, new_score, goal_id=None):
        prev = self.memory.evaluations.get_latest_score(
            scorable, agent_name=self.cfg.get("name")
        )
        if prev is not None:
            delta = round(new_score - prev, 2)
            if self.logger:
                self.logger.log(
                    "ScoreDelta",
                    {
                        "delta": delta,
                        "id": scorable.id,
                        "target_type": scorable.target_type,
                        "text": scorable.text[:60],
                        "goal_id": goal_id,
                        "prev_score": prev,
                        "new_score": new_score,
                        "stage": self.cfg.get("name"),
                    },
                )
            return delta
        return None

class MARSCalculator(BaseScoreCalculator):
    """
    Model Agreement and Reasoning Signal (MARS) Calculator

    Analyzes agreement patterns across multiple scoring models/adapters to:
    - Quantify scoring consensus or divergence across documents
    - Identify which scorers disagree systematically
    - Determine which model aligns best with trust reference
    - Measure uncertainty in the overall assessment
    - Provide diagnostic insights for scoring system improvement

    Unlike traditional aggregators, MARS operates at the ScoreCorpus level (multiple documents)
    to detect reliability patterns rather than just computing an average score.
    """

    def __init__(self, config: Dict = None):
        """
        Initialize MARS calculator with configuration

        Args:
            config: Optional configuration with:
                - trust_reference: Which scorer to use as gold standard (default: "llm")
                - variance_threshold: Threshold for flagging high disagreement (default: 0.15)
                - dimensions: Dimension-specific configurations
                - metrics: Which metrics to analyze (default: ["score"] for core score)
        """
        self.config = config or {}
        self.trust_reference = self.config.get("trust_reference", "llm")
        self.variance_threshold = self.config.get("variance_threshold", 0.15)
        self.metrics = self.config.get(
            "metrics", ["score"]
        )  # Core score by default
        self.dimension_configs = self.config.get("dimensions", {})

    def calculate(self, corpus: "ScoreCorpus") -> Dict[str, Any]:
        """
        Calculate MARS metrics across all scoring models in the corpus

        Args:
            corpus: ScoreCorpus containing results from multiple scorers across multiple documents

        Returns:
            Dictionary containing comprehensive MARS analysis metrics
        """
        # Calculate MARS metrics for each dimension
        mars_results = {}
        for dimension in corpus.dimensions:
            mars_results[dimension] = self._calculate_dimension_mars(
                corpus, dimension
            )

        return mars_results

    def _get_dimension_config(self, dimension: str) -> Dict:
        """Get dimension-specific configuration with fallbacks"""
        return self.dimension_configs.get(
            dimension,
            {
                "trust_reference": self.trust_reference,
                "variance_threshold": self.variance_threshold,
                "metrics": self.metrics,
            },
        )

    def _calculate_dimension_mars(
        self, corpus: "ScoreCorpus", dimension: str
    ) -> Dict[str, Any]:
        """
        Calculate MARS metrics for a specific dimension

        Args:
            corpus: ScoreCorpus containing evaluation results
            dimension: The dimension being analyzed

        Returns:
            Dictionary with MARS metrics for this dimension
        """
        # Get dimension-specific configuration
        dim_config = self._get_dimension_config(dimension)
        trust_ref = dim_config["trust_reference"]
        metrics = dim_config["metrics"]

        # Get the document × scorer matrix for this dimension
        matrix = corpus.get_dimension_matrix(dimension)

        # If no data for this dimension, return empty results
        if matrix.empty:
            return {
                "dimension": dimension,
                "agreement_score": 0.0,
                "std_dev": 0.0,
                "preferred_model": "none",
                "primary_conflict": ("none", "none"),
                "delta": 0.0,
                "high_disagreement": False,
                "explanation": "No data available for this dimension",
                "scorer_metrics": {},
                "metric_correlations": {},
            }

        # Calculate basic statistics
        avg_score = matrix.mean().mean()  # Overall average score
        std_dev = (
            matrix.std().mean()
        )  # Average standard deviation across documents

        # Calculate agreement score (1.0 = perfect agreement)
        agreement_score = 1.0 - min(std_dev, 1.0)

        # Identify primary conflict (largest average score difference)
        scorer_means = matrix.mean()
        max_scorer = scorer_means.idxmax()
        min_scorer = scorer_means.idxmin()
        delta = scorer_means[max_scorer] - scorer_means[min_scorer]
        primary_conflict = (max_scorer, min_scorer)

        # Determine which model aligns best with trust reference
        preferred_model = "unknown"
        if trust_ref in matrix.columns:
            trust_scores = matrix[trust_ref]
            closest = None
            min_diff = float("inf")

            for scorer in matrix.columns:
                if scorer == trust_ref:
                    continue

                # Calculate average absolute difference
                diff = (matrix[scorer] - trust_scores).abs().mean()
                if diff < min_diff:
                    min_diff = diff
                    closest = scorer

            preferred_model = closest if closest else "unknown"
        else:
            # If trust reference isn't available, use median scorer
            sorted_scorers = scorer_means.sort_values()
            median_idx = len(sorted_scorers) // 2
            preferred_model = sorted_scorers.index[median_idx]

        # Identify high-disagreement areas
        high_disagreement = std_dev > dim_config["variance_threshold"]

        # Analyze scorer metrics (q_value, uncertainty, etc.)
        scorer_metrics = self._analyze_scorer_metrics(
            corpus, dimension, metrics
        )

        # Calculate metric correlations
        metric_correlations = self._calculate_metric_correlations(
            corpus, dimension, metrics
        )

        # Generate explanation
        explanation_parts = [
            f"MARS agreement: {agreement_score:.3f} (std: {std_dev:.3f})"
        ]

        if high_disagreement:
            explanation_parts.append(
                f"⚠️ High disagreement detected (threshold: {dim_config['variance_threshold']})"
            )

        if preferred_model != "unknown":
            explanation_parts.append(
                f"Most aligned with {trust_ref}: {preferred_model}"
            )

        explanation_parts.append(
            f"Primary conflict: {primary_conflict[0]} vs {primary_conflict[1]} (Δ={delta:.3f})"
        )

        # Check for systematic bias
        above_mean = [
            scorer
            for scorer, mean_score in scorer_means.items()
            if mean_score > avg_score
        ]
        below_mean = [
            scorer
            for scorer, mean_score in scorer_means.items()
            if mean_score < avg_score
        ]

        if len(above_mean) == 1 or len(below_mean) == 1:
            outlier = above_mean[0] if len(above_mean) == 1 else below_mean[0]
            explanation_parts.append(f"⚠️ {outlier} appears to be an outlier")

        explanation = " | ".join(explanation_parts)

        return {
            "dimension": dimension,
            "agreement_score": round(agreement_score, 3),
            "std_dev": round(std_dev, 3),
            "preferred_model": preferred_model,
            "primary_conflict": primary_conflict,
            "delta": round(delta, 3),
            "high_disagreement": high_disagreement,
            "explanation": explanation,
            "scorer_metrics": scorer_metrics,
            "metric_correlations": metric_correlations,
            "source": "mars",
            "average_score": round(avg_score, 3),
        }

    def _analyze_scorer_metrics(
        self, corpus: "ScoreCorpus", dimension: str, metrics: List[str]
    ) -> Dict[str, Dict[str, float]]:
        """
        Analyze extended metrics for each scorer in this dimension
        """
        scorer_metrics = {}

        for scorer in corpus.scorers:
            # Get all attribute values for this scorer and dimension
            metric_values = corpus.get_metric_values(
                dimension, scorer, metrics
            )

            # Calculate statistics for each metric
            metrics_stats = {}
            for metric, values in metric_values.items():
                if not values:
                    continue

                # Filter out None/NaN values
                valid_values = [v for v in values if v is not None]
                if not valid_values:
                    continue

                metrics_stats[metric] = {
                    "mean": float(np.mean(valid_values)),
                    "std": float(np.std(valid_values)),
                    "min": float(min(valid_values)),
                    "max": float(max(valid_values)),
                    "count": len(valid_values),
                }

            if metrics_stats:
                scorer_metrics[scorer] = metrics_stats

        return scorer_metrics

    def _calculate_metric_correlations(
        self, corpus: "ScoreCorpus", dimension: str, metrics: List[str]
    ) -> Dict[str, Dict[str, float]]:
        """
        Calculate correlations between different metrics for this dimension
        """
        if len(metrics) < 2:
            return {}

        # Get all metric values for this dimension
        metric_values = corpus.get_all_metric_values(dimension, metrics)

        # Calculate correlations
        correlations = {}
        for i in range(len(metrics)):
            for j in range(i + 1, len(metrics)):
                metric1, metric2 = metrics[i], metrics[j]

                # Get valid pairs of values
                pairs = [
                    (v1, v2)
                    for v1, v2 in zip(
                        metric_values[metric1], metric_values[metric2]
                    )
                    if v1 is not None and v2 is not None
                ]

                if len(pairs) > 1:
                    values1, values2 = zip(*pairs)
                    try:
                        corr, _ = stats.pearsonr(values1, values2)
                        if metric1 not in correlations:
                            correlations[metric1] = {}
                        correlations[metric1][metric2] = float(corr)
                    except:
                        pass

        return correlations

    def get_aggregate_score(self, mars_results: Dict[str, Dict]) -> float:
        """
        Get a single aggregate score from MARS analysis

        This provides a weighted average of dimension scores based on agreement reliability

        Args:
            mars_results: Results from calculate() method

        Returns:
            Weighted aggregate score where dimensions with higher agreement contribute more
        """
        total = 0
        weight_sum = 0

        for dimension, results in mars_results.items():
            # Weight by agreement score (higher agreement = more weight)
            weight = results["agreement_score"]
            total += results["average_score"] * weight
            weight_sum += weight

        return round(total / weight_sum, 3) if weight_sum > 0 else 0.0

    def get_high_disagreement_documents(
        self, corpus: "ScoreCorpus", dimension: str, threshold: float = None
    ) -> List[str]:
        """
        Identify documents with high scoring disagreement for this dimension

        Args:
            corpus: ScoreCorpus to analyze
            dimension: Dimension to check
            threshold: Custom disagreement threshold (uses config default if None)

        Returns:
            List of document IDs with high disagreement
        """
        if threshold is None:
            dim_config = self._get_dimension_config(dimension)
            threshold = dim_config["variance_threshold"]

        # Get the document × scorer matrix
        matrix = corpus.get_dimension_matrix(dimension)
        if matrix.empty:
            return []

        # Calculate disagreement per document (standard deviation across scorers)
        disagreement = matrix.std(axis=1)

        # Return documents with disagreement above threshold
        return disagreement[disagreement > threshold].index.tolist()

    def get_scorer_reliability(
        self, corpus: "ScoreCorpus", dimension: str
    ) -> Dict[str, float]:
        """
        Calculate reliability score for each scorer in this dimension

        Args:
            corpus: ScoreCorpus to analyze
            dimension: Dimension to check

        Returns:
            Dictionary mapping scorer names to reliability scores (higher = more reliable)
        """
        # Get dimension-specific configuration
        dim_config = self._get_dimension_config(dimension)
        trust_ref = dim_config["trust_reference"]

        # Get the document × scorer matrix
        matrix = corpus.get_dimension_matrix(dimension)
        if matrix.empty:
            return {}

        # Calculate reliability as correlation with trust reference
        reliability = {}
        if trust_ref in matrix.columns:
            trust_scores = matrix[trust_ref]

            for scorer in matrix.columns:
                if scorer == trust_ref:
                    reliability[scorer] = (
                        1.0  # Perfect correlation with itself
                    )
                    continue

                # Calculate correlation with trust reference
                valid_pairs = matrix[[scorer, trust_ref]].dropna()
                if len(valid_pairs) > 1:
                    try:
                        corr, _ = stats.pearsonr(
                            valid_pairs[scorer], valid_pairs[trust_ref]
                        )
                        reliability[scorer] = float(corr)
                    except:
                        reliability[scorer] = 0.0
                else:
                    reliability[scorer] = 0.0

        # If no trust reference, use consistency across documents
        else:
            scorer_std = matrix.std()
            max_std = scorer_std.max()
            for scorer, std in scorer_std.items():
                # Higher reliability for lower standard deviation
                reliability[scorer] = (
                    1.0 - (std / max_std) if max_std > 0 else 1.0
                )

        return reliability

    def generate_recommendations(
        self, mars_results: Dict[str, Dict]
    ) -> List[str]:
        """
        Generate actionable recommendations based on MARS analysis

        Args:
            mars_results: Results from calculate() method

        Returns:
            List of actionable recommendations
        """
        recommendations = []

        for dimension, results in mars_results.items():
            # High disagreement recommendations
            if results["high_disagreement"]:
                primary_conflict = results["primary_conflict"]
                recommendations.append(
                    f"⚠️ High disagreement in {dimension}: {primary_conflict[0]} and {primary_conflict[1]} "
                    f"differ by {results['delta']:.3f}. Consider human review for ambiguous cases."
                )

            # Outlier scorer recommendations
            scorer_metrics = results["scorer_metrics"]
            if (
                len(scorer_metrics) > 2
            ):  # Need at least 3 scorers to identify outliers
                # Check for scorers with unusual metric patterns
                for scorer, metrics in scorer_metrics.items():
                    if (
                        "uncertainty" in metrics
                        and metrics["uncertainty"]["std"] > 0.2
                    ):
                        recommendations.append(
                            f"⚠️ {scorer} shows high uncertainty variability in {dimension}. "
                            "Consider retraining or adding calibration."
                        )

            # Correlation-based recommendations
            metric_correlations = results["metric_correlations"]
            for metric1, correlations in metric_correlations.items():
                for metric2, corr in correlations.items():
                    if abs(corr) > 0.7:  # Strong correlation
                        recommendations.append(
                            f"💡 In {dimension}, {metric1} and {metric2} are strongly correlated ({corr:.2f}). "
                            "Consider using one as a proxy for the other."
                        )

        # Overall system recommendations
        overall_agreement = mean(
            [r["agreement_score"] for r in mars_results.values()]
        )
        if overall_agreement < 0.7:
            recommendations.append(
                "⚠️ Overall scoring agreement is low (<0.7). Consider implementing human review "
                "for documents with high disagreement."
            )

        return recommendations