using System.Security.Cryptography; using System.Text; using System.Text.Json; namespace WpywMail.Native; /// /// 存储后端之间的数据搬迁。 /// /// 正向(JSON → SQLite):把 users/messages/queue/sessions 全部导入数据库, /// 报文原文与附件读文件后进 blobs 表(gzip + 内容去重),并**逐封校验** /// 「从数据库读回来的字节」与「原文件字节」SHA256 完全一致,之后才允许删除源文件。 /// /// 反向(SQLite → JSON):把 blobs 还原成 raw/ 与 attachments/ 文件并重写 JSON, /// 用于一键回滚。 /// public static class StorageMigration { private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web) { WriteIndented = true }; private static T? ReadJson(string path) { if (!File.Exists(path)) return default; try { return JsonSerializer.Deserialize(File.ReadAllText(path), Json); } catch (Exception ex) { AppLog.Error($"[迁移] 读取 {Path.GetFileName(path)} 失败:{ex.Message}"); return default; } } private static long DirectoryBytes(string path) => Directory.Exists(path) ? Directory.GetFiles(path, "*", SearchOption.AllDirectories).Sum(f => new FileInfo(f).Length) : 0; private static string Sha(byte[] data) => Convert.ToHexString(SHA256.HashData(data)).ToLowerInvariant(); // ------------------------------------------------------------------ 正向 public static int ToSqlite(AppConfig config, bool deleteSourceFiles) { var dataDir = config.DataDirectory; var usersPath = Path.Combine(dataDir, "users.json"); var messagesPath = Path.Combine(dataDir, "messages.json"); var queuePath = Path.Combine(dataDir, "queue.json"); var sessionsPath = Path.Combine(dataDir, "sessions.json"); var users = ReadJson>(usersPath) ?? []; var messages = ReadJson>(messagesPath) ?? []; var queue = ReadJson>(queuePath) ?? []; var sessions = ReadJson>(sessionsPath) ?? []; if (messages.Count == 0 && users.Count == 0) { Console.WriteLine("没有可迁移的数据(users.json / messages.json 为空或不存在)。"); return 1; } var beforeRaw = DirectoryBytes(Path.Combine(dataDir, "raw")); var beforeAttachments = DirectoryBytes(Path.Combine(dataDir, "attachments")); var beforeJson = new[] { usersPath, messagesPath, queuePath, sessionsPath } .Where(File.Exists).Sum(p => new FileInfo(p).Length); Console.WriteLine($"源数据:用户 {users.Count},邮件 {messages.Count},队列 {queue.Count},会话 {sessions.Count}"); Console.WriteLine($"源占用:raw {beforeRaw:N0} B + attachments {beforeAttachments:N0} B + JSON {beforeJson:N0} B"); using var store = new SqliteStore(config); var existing = (int)store.StorageReport().Messages; if (existing > 0) { Console.WriteLine($"数据库里已有 {existing} 封邮件,将按 id 覆盖导入(幂等)。"); } foreach (var user in users) store.ImportUser(user); // UID 唯一性修复:数据库上有 (owner, folder, uid) 唯一索引 var used = new HashSet<(string, string, int)>(); var nextUid = new Dictionary<(string, string), int>(); var verified = new List<(MailMessage Message, string OriginalSha, bool HadFile)>(); var imported = 0; var missingRaw = 0; foreach (var message in messages.OrderBy(m => m.Date)) { message.OwnerEmail = (message.OwnerEmail ?? "").ToLowerInvariant(); message.Folder = (message.Folder ?? "inbox").ToLowerInvariant(); var key = (message.OwnerEmail, message.Folder); if (!nextUid.TryGetValue(key, out var next)) next = 1; var uid = message.Uid; if (uid <= 0 || !used.Add((key.Item1, key.Item2, uid))) { while (used.Contains((key.Item1, key.Item2, next))) next++; uid = next; used.Add((key.Item1, key.Item2, uid)); } message.Uid = uid; next = Math.Max(next, uid + 1); nextUid[key] = next; // 报文原文:文件 → blobs var originalSha = ""; var hadFile = false; if (!string.IsNullOrWhiteSpace(message.RawPath) && !message.RawPath.StartsWith("db:", StringComparison.OrdinalIgnoreCase)) { var full = Path.Combine(dataDir, message.RawPath); if (File.Exists(full)) { var bytes = File.ReadAllBytes(full); originalSha = Sha(bytes); hadFile = true; message.RawPath = store.SaveRaw(bytes); message.Size = bytes.Length; } else { missingRaw++; message.RawPath = ""; } } // 附件:文件 → blobs foreach (var attachment in message.Attachments) { if (string.IsNullOrWhiteSpace(attachment.StoredAs) || attachment.StoredAs.StartsWith("db:", StringComparison.OrdinalIgnoreCase)) continue; var full = Path.Combine(dataDir, attachment.StoredAs); if (File.Exists(full)) attachment.StoredAs = store.SaveAttachment(File.ReadAllBytes(full), attachment.FileName); } store.SaveMessage(message); verified.Add((message, originalSha, hadFile)); imported++; } foreach (var item in queue) store.ImportQueueItem(item); var now = DateTimeOffset.UtcNow; foreach (var session in sessions.Where(s => s.Expires > now)) store.ImportSession(session); // ---- 校验:从数据库读回来的字节必须与原文件逐字节一致 ---- Console.WriteLine("校验中(逐封比对 SHA256)……"); var bad = 0; foreach (var (message, originalSha, hadFile) in verified) { if (!hadFile || message.RawPath.Length == 0) continue; try { var back = store.ReadRaw(message.RawPath); if (Sha(back) != originalSha) { bad++; Console.Error.WriteLine($" [不一致] {message.Id} {message.Subject}"); } } catch (Exception ex) { bad++; Console.Error.WriteLine($" [读取失败] {message.Id}:{ex.Message}"); } } var report = store.StorageReport(); Console.WriteLine(); Console.WriteLine($"导入完成:邮件 {imported} 封,队列 {queue.Count} 条,会话 {sessions.Count} 个" + (missingRaw > 0 ? $",{missingRaw} 封缺少原始报文文件" : "")); Console.WriteLine($"入库大对象:原始 {report.BlobRawBytes:N0} B → 存储 {report.BlobStoredBytes:N0} B" + (report.BlobRawBytes > 0 ? $"({(100.0 * report.BlobStoredBytes / report.BlobRawBytes):F0}%,{report.BlobGzipped} 个压缩)" : "")); Console.WriteLine($"数据库文件:{report.DbBytes:N0} B + WAL {report.WalBytes:N0} B"); Console.WriteLine($"校验结果:SHA256 不一致 {bad} 封"); if (bad > 0) { Console.Error.WriteLine("存在校验失败,已保留源文件,请勿删除。"); return 2; } if (!deleteSourceFiles) { Console.WriteLine("源文件已保留(想释放空间请加 --delete-source 重跑)。"); return 0; } // ---- 删源文件(校验已全部通过才走到这里)---- var deleted = 0; long freed = 0; foreach (var file in Directory.GetFiles(Path.Combine(dataDir, "raw"), "*.eml", SearchOption.TopDirectoryOnly)) { freed += new FileInfo(file).Length; File.Delete(file); deleted++; } var attachmentDir = Path.Combine(dataDir, "attachments"); foreach (var file in Directory.GetFiles(attachmentDir, "*", SearchOption.TopDirectoryOnly)) { freed += new FileInfo(file).Length; File.Delete(file); deleted++; } Console.WriteLine($"已删除源文件 {deleted} 个,释放 {freed:N0} B"); Console.WriteLine($"提示:JSON 索引文件(users/messages/queue/sessions.json)保留未删,便于随时回滚。"); return 0; } // ------------------------------------------------------------------ 反向 public static int ToJson(AppConfig config) { var dataDir = config.DataDirectory; using var store = new SqliteStore(config); var messages = store.AllMessages().ToList(); var users = store.AllUsers().ToList(); var queue = store.AllQueueItems().ToList(); var sessions = store.AllSessions().ToList(); var rawDir = Path.Combine(dataDir, "raw"); var attachmentDir = Path.Combine(dataDir, "attachments"); Directory.CreateDirectory(rawDir); Directory.CreateDirectory(attachmentDir); var exported = new Dictionary(); foreach (var (id, data) in store.ExportBlobs()) { // 同一个 blob 可能被多封邮件/附件引用,这里各导出一份文件(回滚场景不追求去重) var name = $"{DateTime.UtcNow:yyyyMMddHHmmss}-{id}{GuessExtension(data)}"; var relative = Path.Combine("raw", name); File.WriteAllBytes(Path.Combine(dataDir, relative), data); exported[id] = relative; } static string GuessExtension(byte[] data) => data.Length >= 4 && data[0] == 0x50 && data[1] == 0x4B ? ".zip" : ".eml"; foreach (var message in messages) { if (message.RawPath.StartsWith("db:", StringComparison.OrdinalIgnoreCase) && long.TryParse(message.RawPath.AsSpan(3), out var blobId) && exported.TryGetValue(blobId, out var path)) message.RawPath = path; foreach (var attachment in message.Attachments) { if (attachment.StoredAs.StartsWith("db:", StringComparison.OrdinalIgnoreCase) && long.TryParse(attachment.StoredAs.AsSpan(3), out var attId) && exported.TryGetValue(attId, out var attPath)) { var target = Path.Combine("attachments", Path.GetFileName(attPath)); File.Copy(Path.Combine(dataDir, attPath), Path.Combine(dataDir, target), true); attachment.StoredAs = target; } } } File.WriteAllText(Path.Combine(dataDir, "messages.json"), JsonSerializer.Serialize(messages, Json), new UTF8Encoding(false)); File.WriteAllText(Path.Combine(dataDir, "users.json"), JsonSerializer.Serialize(users, Json), new UTF8Encoding(false)); File.WriteAllText(Path.Combine(dataDir, "queue.json"), JsonSerializer.Serialize(queue, Json), new UTF8Encoding(false)); File.WriteAllText(Path.Combine(dataDir, "sessions.json"), JsonSerializer.Serialize(sessions, Json), new UTF8Encoding(false)); Console.WriteLine($"已导出:邮件 {messages.Count} 封,用户 {users.Count},队列 {queue.Count},会话 {sessions.Count}"); Console.WriteLine($"报文文件 {exported.Count} 个 → {rawDir}"); Console.WriteLine("把 appsettings.json 的 Storage.Provider 改回 json 即可用这套数据启动。"); return 0; } // ------------------------------------------------------------------ 状态 public static int Status(AppConfig config) { var dataDir = config.DataDirectory; var rawBytes = DirectoryBytes(Path.Combine(dataDir, "raw")); var attachmentBytes = DirectoryBytes(Path.Combine(dataDir, "attachments")); var jsonBytes = new[] { "users.json", "messages.json", "queue.json", "sessions.json" } .Select(f => Path.Combine(dataDir, f)).Where(File.Exists).Sum(p => new FileInfo(p).Length); Console.WriteLine($"当前配置的后端:{config.Storage.Provider}"); Console.WriteLine($"raw/ {rawBytes,12:N0} B ({Directory.GetFiles(Path.Combine(dataDir, "raw"), "*").Length} 个文件)"); Console.WriteLine($"attachments/{attachmentBytes,12:N0} B ({Directory.GetFiles(Path.Combine(dataDir, "attachments"), "*").Length} 个文件)"); Console.WriteLine($"JSON 索引 {jsonBytes,12:N0} B"); var dbPath = string.IsNullOrWhiteSpace(config.Storage.DatabasePath) ? Path.Combine(dataDir, "wpywmail.db") : config.Storage.DatabasePath; if (!File.Exists(dbPath)) { Console.WriteLine("SQLite 数据库:尚未创建"); return 0; } using var store = new SqliteStore(config); var report = store.StorageReport(); Console.WriteLine($"SQLite {report.DbBytes + report.WalBytes,12:N0} B ({dbPath})"); Console.WriteLine($" 邮件 {report.Messages} 封,大对象 {report.Blobs} 个" + $"(原始 {report.BlobRawBytes:N0} B → 存储 {report.BlobStoredBytes:N0} B," + $"其中 {report.BlobGzipped} 个启用了压缩)"); Console.WriteLine($" 正文列合计 {report.TextBytes:N0} B(最大一封 {report.LargestTextBytes:N0} B:{Trim(report.LargestTextSubject)})"); Console.WriteLine($" 页统计:{report.PageCount} 页 × {report.PageSize} B = {report.TotalBytes:N0} B," + $"其中空闲 {report.FreeBytes:N0} B" + (report.FreeBytes > 0 ? "(用 --vacuum 回收;注意 VACUUM 需要约 2 倍临时空间,且应在服务停止时做)" : "")); Console.WriteLine($" 孤儿大对象 {report.Orphans} 个(用 --compact 清理)"); return 0; } private static string Trim(string text) => string.IsNullOrEmpty(text) ? "(无)" : text.Length <= 24 ? text : text[..24] + "…"; public static int Compact(AppConfig config) { using var store = new SqliteStore(config); var removed = store.Compact(); Console.WriteLine($"已清理孤儿大对象 {removed} 个。"); return Status(config); } /// 合并 WAL 并回收空闲页,把数据库文件压到实际大小。 public static int Vacuum(AppConfig config) { using var store = new SqliteStore(config); var before = store.StorageReport(); Console.WriteLine($"整理前:主库 {before.DbBytes:N0} B + WAL {before.WalBytes:N0} B,空闲页 {before.FreeBytes:N0} B"); store.Vacuum(); // VACUUM:回收空闲页 store.Persist(); // 再把 VACUUM 期间产生的 WAL 归并回主库并截断 var after = store.StorageReport(); Console.WriteLine($"整理后:主库 {after.DbBytes:N0} B + WAL {after.WalBytes:N0} B,空闲页 {after.FreeBytes:N0} B"); Console.WriteLine($"实际占用:{(before.DbBytes + before.WalBytes):N0} B → {(after.DbBytes + after.WalBytes):N0} B" + $"(释放 {(before.DbBytes + before.WalBytes) - (after.DbBytes + after.WalBytes):N0} B)"); return 0; } }