Natural Memory NM2.1: 记忆路由器分叉、数据集缺陷修复与全轴评测证据
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
This commit is contained in:
WpyQwq
@@ -0,0 +1,571 @@
|
||||
[
|
||||
{
|
||||
"model": "hf-internal-testing/tiny-random-LlamaForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.layers",
|
||||
"4_convention": "MISMATCH: second positional is 'attention_mask', adapter forwards position_embeddings positionally",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "LlamaForCausalLM",
|
||||
"model_type": "llama",
|
||||
"hidden_size": 16,
|
||||
"num_layers": 2,
|
||||
"vocab_size": 32000,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "LlamaForCausalLM",
|
||||
"container_path": "model.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"attention_mask",
|
||||
"position_ids",
|
||||
"past_key_values",
|
||||
"use_cache",
|
||||
"position_embeddings",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": false,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "hf-internal-testing/tiny-random-MistralForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.layers",
|
||||
"4_convention": "MISMATCH: second positional is 'attention_mask', adapter forwards position_embeddings positionally",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "MistralForCausalLM",
|
||||
"model_type": "mistral",
|
||||
"hidden_size": 32,
|
||||
"num_layers": 2,
|
||||
"vocab_size": 32000,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "MistralForCausalLM",
|
||||
"container_path": "model.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"attention_mask",
|
||||
"position_ids",
|
||||
"past_key_values",
|
||||
"use_cache",
|
||||
"position_embeddings",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": false,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.layers",
|
||||
"4_convention": "MISMATCH: second positional is 'attention_mask', adapter forwards position_embeddings positionally",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "Qwen2ForCausalLM",
|
||||
"model_type": "qwen2",
|
||||
"hidden_size": 8,
|
||||
"num_layers": 2,
|
||||
"vocab_size": 152064,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "Qwen2ForCausalLM",
|
||||
"container_path": "model.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"attention_mask",
|
||||
"position_ids",
|
||||
"past_key_values",
|
||||
"use_cache",
|
||||
"position_embeddings",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": false,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "trl-internal-testing/tiny-Qwen3ForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.layers",
|
||||
"4_convention": "MISMATCH: second positional is 'attention_mask', adapter forwards position_embeddings positionally",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "Qwen3ForCausalLM",
|
||||
"model_type": "qwen3",
|
||||
"hidden_size": 8,
|
||||
"num_layers": 2,
|
||||
"vocab_size": 151936,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "Qwen3ForCausalLM",
|
||||
"container_path": "model.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"attention_mask",
|
||||
"position_ids",
|
||||
"past_key_values",
|
||||
"use_cache",
|
||||
"position_embeddings",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": false,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "hf-internal-testing/tiny-random-Gemma2ForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.layers",
|
||||
"4_convention": "ok (position_embeddings second)",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "Gemma2ForCausalLM",
|
||||
"model_type": "gemma2",
|
||||
"hidden_size": 32,
|
||||
"num_layers": 1,
|
||||
"vocab_size": 256000,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "Gemma2ForCausalLM",
|
||||
"container_path": "model.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"position_embeddings",
|
||||
"attention_mask",
|
||||
"position_ids",
|
||||
"past_key_values",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": true,
|
||||
"resolved_layer_indices": [
|
||||
0
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "hf-internal-testing/tiny-random-Gemma3ForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.layers",
|
||||
"4_convention": "ok (position_embeddings second)",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "Gemma3ForCausalLM",
|
||||
"model_type": "gemma3_text",
|
||||
"hidden_size": 16,
|
||||
"num_layers": 2,
|
||||
"vocab_size": 262144,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "Gemma3ForCausalLM",
|
||||
"container_path": "model.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"position_embeddings",
|
||||
"attention_mask",
|
||||
"position_ids",
|
||||
"past_key_values",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": true,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "hf-internal-testing/tiny-random-Starcoder2ForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.layers",
|
||||
"4_convention": "MISMATCH: second positional is 'attention_mask', adapter forwards position_embeddings positionally",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "Starcoder2ForCausalLM",
|
||||
"model_type": "starcoder2",
|
||||
"hidden_size": 32,
|
||||
"num_layers": 2,
|
||||
"vocab_size": 1024,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "Starcoder2ForCausalLM",
|
||||
"container_path": "model.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"attention_mask",
|
||||
"position_ids",
|
||||
"past_key_values",
|
||||
"use_cache",
|
||||
"position_embeddings",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": false,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "hf-internal-testing/tiny-random-OlmoeForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.layers",
|
||||
"4_convention": "MISMATCH: second positional is 'attention_mask', adapter forwards position_embeddings positionally",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "OlmoeForCausalLM",
|
||||
"model_type": "olmoe",
|
||||
"hidden_size": 64,
|
||||
"num_layers": 2,
|
||||
"vocab_size": 50304,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "OlmoeForCausalLM",
|
||||
"container_path": "model.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"attention_mask",
|
||||
"position_ids",
|
||||
"past_key_values",
|
||||
"use_cache",
|
||||
"position_embeddings",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": false,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "hf-internal-testing/tiny-random-GraniteMoeForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.layers",
|
||||
"4_convention": "MISMATCH: second positional is 'attention_mask', adapter forwards position_embeddings positionally",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "GraniteMoeForCausalLM",
|
||||
"model_type": "granitemoe",
|
||||
"hidden_size": 32,
|
||||
"num_layers": 2,
|
||||
"vocab_size": 49155,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "GraniteMoeForCausalLM",
|
||||
"container_path": "model.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"attention_mask",
|
||||
"past_key_values",
|
||||
"position_embeddings",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": false,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "hf-internal-testing/tiny-random-MixtralForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.layers",
|
||||
"4_convention": "ok (position_embeddings second)",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "MixtralForCausalLM",
|
||||
"model_type": "mixtral",
|
||||
"hidden_size": 64,
|
||||
"num_layers": 2,
|
||||
"vocab_size": 32000,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "MixtralForCausalLM",
|
||||
"container_path": "model.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"position_embeddings",
|
||||
"attention_mask",
|
||||
"position_ids",
|
||||
"past_key_values",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": true,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "hf-internal-testing/tiny-random-CohereForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.layers",
|
||||
"4_convention": "MISMATCH: second positional is 'attention_mask', adapter forwards position_embeddings positionally",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "CohereForCausalLM",
|
||||
"model_type": "cohere",
|
||||
"hidden_size": 32,
|
||||
"num_layers": 2,
|
||||
"vocab_size": 1024,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "CohereForCausalLM",
|
||||
"container_path": "model.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"attention_mask",
|
||||
"position_ids",
|
||||
"past_key_values",
|
||||
"use_cache",
|
||||
"position_embeddings",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": false,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "hf-internal-testing/tiny-random-PhiForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.layers",
|
||||
"4_convention": "MISMATCH: second positional is 'attention_mask', adapter forwards position_embeddings positionally",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "PhiForCausalLM",
|
||||
"model_type": "phi",
|
||||
"hidden_size": 32,
|
||||
"num_layers": 2,
|
||||
"vocab_size": 1024,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "PhiForCausalLM",
|
||||
"container_path": "model.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"attention_mask",
|
||||
"position_ids",
|
||||
"past_key_values",
|
||||
"use_cache",
|
||||
"position_embeddings",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": false,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "hf-tiny-model-private/tiny-random-OPTForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.decoder.layers",
|
||||
"4_convention": "MISMATCH: second positional is 'attention_mask', adapter forwards position_embeddings positionally",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "OPTForCausalLM",
|
||||
"model_type": "opt",
|
||||
"hidden_size": 16,
|
||||
"num_layers": 5,
|
||||
"vocab_size": 1024,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "OPTForCausalLM",
|
||||
"container_path": "model.decoder.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"attention_mask",
|
||||
"past_key_values",
|
||||
"use_cache",
|
||||
"position_ids",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": false,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
},
|
||||
{
|
||||
"model": "hf-tiny-model-private/tiny-random-BartForCausalLM",
|
||||
"full_stack": false,
|
||||
"dtype": "float32",
|
||||
"stages": {
|
||||
"1_config": "needs patch: config.text_config",
|
||||
"2_load_causal_lm": "ok",
|
||||
"3_container": "needs patch: repo wants model.language_model.layers, found model.decoder.layers",
|
||||
"4_convention": "MISMATCH: second positional is 'attention_mask', adapter forwards position_embeddings positionally",
|
||||
"5_surgery": "ok (no shims)",
|
||||
"6_identity": "identical",
|
||||
"7_write_read": "ok"
|
||||
},
|
||||
"patches": [],
|
||||
"architecture": "BartForCausalLM",
|
||||
"model_type": "bart",
|
||||
"hidden_size": 16,
|
||||
"num_layers": 2,
|
||||
"vocab_size": 1024,
|
||||
"text_config_present": false,
|
||||
"backbone_class": "BartForCausalLM",
|
||||
"container_path": "model.decoder.layers",
|
||||
"repo_layer_path_ok": false,
|
||||
"layer_forward_params": [
|
||||
"hidden_states",
|
||||
"attention_mask",
|
||||
"encoder_hidden_states",
|
||||
"encoder_attention_mask",
|
||||
"past_key_values",
|
||||
"use_cache",
|
||||
"kwargs"
|
||||
],
|
||||
"second_positional_is_position_embeddings": false,
|
||||
"resolved_layer_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"shims_needed": [],
|
||||
"identity_max_abs_logit_delta": 0.0,
|
||||
"memory_state_changed_by": null
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user