Zig 文本处理：Unicode、正则与高性能字符串

字节层     []u8        存储与传输，UTF-8 编码后的原始数据
码点层     u21         Unicode 码点，一个「逻辑字符」的编号
字素簇层   []const u8  人类感知的一个「字」，可能由多个码点组成

const std = @import("std");

test "utf8 decode" {
    const s = "héllo 世界";
    var view = std.unicode.Utf8View.init(s) catch unreachable;
    var it = view.iterator();

    var count: usize = 0;
    while (it.nextCodepoint()) |cp| {
        std.debug.print("U+{X:0>4}\n", .{cp});
        count += 1;
    }
    try std.testing.expectEqual(@as(usize, 8), count);
}

👨👩👧  = U+1F468 ZWJ U+1F469 ZWJ U+1F467   (家庭 emoji，5 个码点)
🇨🇳   = U+1F1E8 U+1F1F3                     (国旗，2 个区域指示符)
é    = U+0065 U+0301                        (e + 组合符)

test "byte iterate" {
    const s = "abc";
    for (s, 0..) |byte, i| {
        std.debug.print("{d}: {c}\n", .{ i, byte });
    }
}

test "codepoint iterate" {
    const s = "a世b";
    var view = try std.unicode.Utf8View.init(s);
    var it = view.iterator();
    while (it.nextCodepointSlice()) |slice| {
        std.debug.print("{s} ({d} 字节)\n", .{ slice, slice.len });
    }
}

fn truncateUtf8(s: []const u8, max_bytes: usize) []const u8 {
    if (s.len <= max_bytes) return s;
    var end = max_bytes;
    // 回退到字符边界：UTF-8 续字节是 10xxxxxx
    while (end > 0 and (s[end] & 0xC0) == 0x80) end -= 1;
    return s[0..end];
}

test "ascii case" {
    var buf: [16]u8 = undefined;
    const upper = std.ascii.upperString(&buf, "hello");
    try std.testing.expectEqualStrings("HELLO", upper);
}

NFD → NFC：合并预组合字符，适合存储与比较
NFC → NFD：分解，适合某些文本算法

fn textEqual(a: []const u8, b: []const u8) bool {
    // 先按字节快速比较
    if (std.mem.eql(u8, a, b)) return true;
    // 字节不等时，规范化后再比较（需要 ziglyph 等库）
    return false;
}

fn matchHere(pattern: []const u8, text: []const u8) bool {
    if (pattern.len == 0) return true;

    // 处理 * 量词
    if (pattern.len >= 2 and pattern[1] == '*') {
        return matchStar(pattern[0], pattern[2..], text);
    }
    // 处理 + 量词（至少一次）
    if (pattern.len >= 2 and pattern[1] == '+') {
        if (text.len > 0 and charMatches(pattern[0], text[0])) {
            return matchStar(pattern[0], pattern[2..], text[1..]);
        }
        return false;
    }
    // 处理 ? 量词（零或一次）
    if (pattern.len >= 2 and pattern[1] == '?') {
        if (text.len > 0 and charMatches(pattern[0], text[0])) {
            if (matchHere(pattern[2..], text[1..])) return true;
        }
        return matchHere(pattern[2..], text);
    }
    // 单字符匹配
    if (text.len > 0 and charMatches(pattern[0], text[0])) {
        return matchHere(pattern[1..], text[1..]);
    }
    return false;
}

fn matchStar(c: u8, pattern: []const u8, text: []const u8) bool {
    var t = text;
    while (true) {
        if (matchHere(pattern, t)) return true;
        if (t.len == 0 or !charMatches(c, t[0])) return false;
        t = t[1..];
    }
}

fn charMatches(pattern_char: u8, text_char: u8) bool {
    if (pattern_char == '.') return true;
    return pattern_char == text_char;
}

fn search(pattern: []const u8, text: []const u8) bool {
    if (pattern.len > 0 and pattern[0] == '^') {
        return matchHere(pattern[1..], text);
    }
    var t = text;
    while (true) {
        if (matchHere(pattern, t)) return true;
        if (t.len == 0) return false;
        t = t[1..];
    }
}

const idx = std.mem.indexOf(u8, haystack, needle);
const starts = std.mem.startsWith(u8, s, "prefix");
const token = std.mem.tokenizeScalar(u8, line, ',');

test "tokenize" {
    const line = "a,b,,c";
    var it = std.mem.tokenizeScalar(u8, line, ',');
    var count: usize = 0;
    while (it.next()) |tok| : (count += 1) {
        std.debug.print("[{s}]\n", .{tok});
    }
    // 输出 a / b / c，空字段被跳过
    try std.testing.expectEqual(@as(usize, 3), count);
}

test "find" {
    const haystack = "the quick brown fox";
    const pos = std.mem.indexOf(u8, haystack, "brown").?;
    try std.testing.expectEqual(@as(usize, 10), pos);

    const last = std.mem.lastIndexOf(u8, haystack, "o").?;
    try std.testing.expectEqual(@as(usize, 17), last);
}

test "concat" {
    var out = std.ArrayList(u8).init(std.testing.allocator);
    defer out.deinit();

    try out.appendSlice("Hello, ");
    try out.appendSlice("Zig");
    try out.append('!');
    try std.testing.expectEqualStrings("Hello, Zig!", out.items);
}

try out.ensureTotalCapacity(1024);

test "format" {
    var buf: [64]u8 = undefined;
    const s = try std.fmt.bufPrint(&buf, "id={d} name={s}", .{ 42, "zig" });
    try std.testing.expectEqualStrings("id=42 name=zig", s);
}

test "writer" {
    var out = std.ArrayList(u8).init(std.testing.allocator);
    defer out.deinit();
    const w = out.writer();

    try w.print("count={d}\n", .{3});
    try w.writeAll("done\n");
    try std.testing.expectEqualStrings("count=3\ndone\n", out.items);
}

// 差：每次循环都重新分配
var s: []const u8 = "";
for (items) |item| {
    s = try std.fmt.allocPrint(alloc, "{s}{s}", .{ s, item }); // O(n²)
}

// 好：一次构建
var out = std.ArrayList(u8).init(alloc);
defer out.deinit();
for (items) |item| try out.appendSlice(item);

const LogEntry = struct {
    timestamp: []const u8,
    level: []const u8,
    module: []const u8,
    message: []const u8,
};

fn parseLogLine(line: []const u8) ?LogEntry {
    var it = std.mem.tokenizeScalar(u8, line, ' ');
    const timestamp = it.next() orelse return null;
    const level = it.next() orelse return null;
    var module = it.next() orelse return null;
    if (module.len >= 2 and module[0] == '[') {
        module = module[1 .. module.len - 1];
    }
    // 剩余部分是消息，可能含空格
    const rest_start = std.mem.indexOf(u8, line, module) orelse return null;
    const message = std.mem.trim(u8, line[rest_start + module.len ..], " ");
    return .{
        .timestamp = timestamp,
        .level = level,
        .module = module,
        .message = message,
    };
}

fn parseCsvLine(line: []const u8, alloc: std.mem.Allocator) ![][]const u8 {
    var fields = std.ArrayList([]const u8).init(alloc);
    var it = std.mem.splitScalar(u8, line, ',');
    while (it.next()) |field| {
        try fields.append(std.mem.trim(u8, field, " \t"));
    }
    return fields.toOwnedSlice();
}

pub fn processFile(path: []const u8, alloc: std.mem.Allocator) !void {
    const file = try std.fs.cwd().openFile(path, .{});
    defer file.close();

    var buf_reader = std.io.bufferedReader(file.reader());
    var reader = buf_reader.reader();

    var line_buf = std.ArrayList(u8).init(alloc);
    defer line_buf.deinit();

    while (try reader.readUntilDelimiterOrEofAlloc(alloc, '\n', 1 << 20)) |line| {
        defer alloc.free(line);
        if (parseLogLine(line)) |entry| {
            if (std.mem.eql(u8, entry.level, "ERROR")) {
                std.debug.print("发现错误: {s}\n", .{entry.message});
            }
        }
    }
}