{
  "format_version": 3,
  "generator": "prepare_memory_router_dataset.py",
  "seed": 20260909,
  "candidate_count": 32,
  "schema": {
    "query": "natural-language routing query",
    "candidates": "bounded memory records with text and provenance",
    "positive_indices": "one or more supporting memory records; empty means abstain",
    "need_memory": "1 if evidence is required and present, 0 for unknown/unsupported queries",
    "hop": "0 for abstention, otherwise number of supporting records clipped to router max_hops",
    "group_id": "conversation or QA episode identity; no group may cross train/eval"
  },
  "files": {
    "train": {
      "path": "H:\\Memory\\V2_dpskw\\data\\_probe_original\\train.jsonl",
      "episodes": 300,
      "sha256": "433df3400497202828aab3a7857151e8191052c5e1d6d2bd03bbde4de7d97e26"
    },
    "eval": {
      "path": "H:\\Memory\\V2_dpskw\\data\\_probe_original\\eval.jsonl",
      "episodes": 100,
      "sha256": "9e62a5e24cf27828a4dfb011e2f5b86975353eb2a373a8ed65f1133ee143e032"
    }
  },
  "counts": {
    "train_groups": 84155,
    "eval_groups": 21120,
    "train_episodes": 300,
    "eval_episodes": 100,
    "train_unknown": 162,
    "eval_unknown": 73,
    "train_candidates": 9600,
    "eval_candidates": 3200
  },
  "local_sources": [
    {
      "source": "dynamic_memory_lab/data/benchmark_train.jsonl",
      "rows": 128,
      "kind": "benchmark",
      "split": "train"
    },
    {
      "source": "dynamic_memory_lab/data/native_memory/train.jsonl",
      "rows": 512,
      "kind": "native_memory",
      "split": "train"
    },
    {
      "source": "dynamic_memory_lab/data/production_memory/train.jsonl",
      "rows": 1485,
      "kind": "normalized_policy",
      "split": "train"
    },
    {
      "source": "dynamic_memory_lab/data/production_memory_hard_v2/train.jsonl",
      "rows": 27000,
      "kind": "normalized_policy",
      "split": "train"
    },
    {
      "source": "dynamic_memory_lab/data/mega_validation/memory_validation_80k_train.jsonl",
      "rows": 80000,
      "kind": "mega_validation",
      "split": "train"
    },
    {
      "source": "dynamic_memory_lab/data/benchmark_eval.jsonl",
      "rows": 64,
      "kind": "benchmark",
      "split": "eval"
    },
    {
      "source": "dynamic_memory_lab/data/native_memory/eval.jsonl",
      "rows": 128,
      "kind": "native_memory",
      "split": "eval"
    },
    {
      "source": "dynamic_memory_lab/data/production_memory/eval.jsonl",
      "rows": 363,
      "kind": "normalized_policy",
      "split": "eval"
    },
    {
      "source": "dynamic_memory_lab/data/production_memory_hard_v2/eval.jsonl",
      "rows": 7200,
      "kind": "normalized_policy",
      "split": "eval"
    },
    {
      "source": "dynamic_memory_lab/data/mega_validation/memory_validation_20k_eval.jsonl",
      "rows": 20000,
      "kind": "mega_validation",
      "split": "eval"
    }
  ],
  "public_sources": [],
  "public_catalog": [
    {
      "name": "hotpotqa_train",
      "dataset_id": "hotpot_qa",
      "config": "distractor",
      "split": "train",
      "split_kind": "train",
      "task": "qa",
      "url": "https://huggingface.co/datasets/hotpot_qa"
    },
    {
      "name": "hotpotqa_validation",
      "dataset_id": "hotpot_qa",
      "config": "distractor",
      "split": "validation",
      "split_kind": "eval",
      "task": "qa",
      "url": "https://huggingface.co/datasets/hotpot_qa"
    },
    {
      "name": "codesearchnet_python_train",
      "dataset_id": "code_search_net",
      "config": "python",
      "split": "train",
      "split_kind": "train",
      "task": "code",
      "url": "https://huggingface.co/datasets/code_search_net"
    },
    {
      "name": "codesearchnet_python_validation",
      "dataset_id": "code_search_net",
      "config": "python",
      "split": "validation",
      "split_kind": "eval",
      "task": "code",
      "url": "https://huggingface.co/datasets/code_search_net"
    },
    {
      "name": "fever_train",
      "dataset_id": "fever",
      "config": "v1.0",
      "split": "train",
      "split_kind": "train",
      "task": "evidence",
      "url": "https://huggingface.co/datasets/fever"
    },
    {
      "name": "fever_validation",
      "dataset_id": "fever",
      "config": "v1.0",
      "split": "labelled_dev",
      "split_kind": "eval",
      "task": "evidence",
      "url": "https://huggingface.co/datasets/fever"
    }
  ],
  "split_stats": {
    "train": {
      "kept": 300,
      "unknown": 162,
      "family:benchmark_qa": 128,
      "family:native_memory": 172,
      "written": 300
    },
    "eval": {
      "kept": 100,
      "unknown": 73,
      "family:benchmark_qa": 64,
      "family:native_memory": 36,
      "written": 100
    }
  },
  "leakage_check": {
    "group_overlap": 0,
    "passed": true
  },
  "conflict_policy": {
    "enabled": true,
    "answerable_positive_conflicts_excluded": true,
    "unknown_hard_conflicts_retained": true
  },
  "evaluation_policy": "eval.jsonl is generated and hashed before router training; the trainer refuses an optional hash mismatch.",
  "warnings": [
    "Local generated memory-policy files are useful hard negatives but are not public-human chat data.",
    "Add redacted real user traces only after consent and PII removal.",
    "Public download failures are recorded; failed sources contribute zero rows."
  ]
}