perf(DurFile): stream JSON parsing and flatten color maps

src/animations/DurFile.zig

@@ -19,25 +19,268 @@ const LogFile = ly_core.LogFile;
 const enums = @import("../enums.zig");
 const DurOffsetAlignment = enums.DurOffsetAlignment;
 
+const JsonTokenSlice = struct {
+    value: []const u8,
+    allocated: ?[]u8 = null,
+
+    fn deinit(self: JsonTokenSlice, allocator: Allocator) void {
+        if (self.allocated) |value| allocator.free(value);
+    }
+};
+
+fn freeJsonToken(allocator: Allocator, token: Json.Token) void {
+    switch (token) {
+        .allocated_string, .allocated_number => |value| allocator.free(value),
+        else => {},
+    }
+}
+
+fn jsonString(allocator: Allocator, reader: *Json.Reader) !JsonTokenSlice {
+    const token = try reader.nextAlloc(allocator, .alloc_if_needed);
+    errdefer freeJsonToken(allocator, token);
+
+    return switch (token) {
+        .string => |value| .{ .value = value },
+        .allocated_string => |value| .{ .value = value, .allocated = value },
+        else => error.UnexpectedToken,
+    };
+}
+
+fn jsonOwnedString(allocator: Allocator, reader: *Json.Reader) ![]u8 {
+    const token = try reader.nextAlloc(allocator, .alloc_always);
+    errdefer freeJsonToken(allocator, token);
+
+    return switch (token) {
+        .allocated_string => |value| value,
+        else => error.UnexpectedToken,
+    };
+}
+
+fn jsonNumber(allocator: Allocator, reader: *Json.Reader) !JsonTokenSlice {
+    const token = try reader.nextAlloc(allocator, .alloc_if_needed);
+    errdefer freeJsonToken(allocator, token);
+
+    return switch (token) {
+        .number => |value| .{ .value = value },
+        .allocated_number => |value| .{ .value = value, .allocated = value },
+        else => error.UnexpectedToken,
+    };
+}
+
+fn jsonI64(allocator: Allocator, reader: *Json.Reader) !i64 {
+    const number = try jsonNumber(allocator, reader);
+    defer number.deinit(allocator);
+
+    if (!Json.isNumberFormattedLikeAnInteger(number.value))
+        return error.InvalidNumber;
+
+    return std.fmt.parseInt(i64, number.value, 10) catch |err| switch (err) {
+        error.Overflow => error.Overflow,
+        error.InvalidCharacter => error.InvalidNumber,
+    };
+}
+
+fn jsonI32(allocator: Allocator, reader: *Json.Reader) !i32 {
+    const number = try jsonNumber(allocator, reader);
+    defer number.deinit(allocator);
+
+    if (Json.isNumberFormattedLikeAnInteger(number.value)) {
+        return std.fmt.parseInt(i32, number.value, 10) catch |err| switch (err) {
+            error.Overflow => error.Overflow,
+            error.InvalidCharacter => error.InvalidNumber,
+        };
+    }
+
+    const value = std.fmt.parseFloat(f64, number.value) catch return error.InvalidNumber;
+    if (!std.math.isFinite(value) or @round(value) != value)
+        return error.InvalidNumber;
+    if (value > @as(f64, @floatFromInt(std.math.maxInt(i32))) or
+        value < @as(f64, @floatFromInt(std.math.minInt(i32))))
+        return error.Overflow;
+
+    return @intFromFloat(value);
+}
+
+fn jsonFloat(allocator: Allocator, reader: *Json.Reader, allow_integer: bool) !f64 {
+    const number = try jsonNumber(allocator, reader);
+    defer number.deinit(allocator);
+
+    if (!allow_integer and Json.isNumberFormattedLikeAnInteger(number.value))
+        return error.UnexpectedToken;
+
+    const value = std.fmt.parseFloat(f64, number.value) catch return error.InvalidNumber;
+    if (!std.math.isFinite(value)) return error.Overflow;
+    return value;
+}
+
+fn jsonF64(allocator: Allocator, reader: *Json.Reader) !f64 {
+    return jsonFloat(allocator, reader, false);
+}
+
+fn jsonF32(allocator: Allocator, reader: *Json.Reader) !f32 {
+    return @floatCast(try jsonFloat(allocator, reader, true));
+}
+
 const Frame = struct {
     frameNumber: i32,
     delay: f32,
     contents: [][]u8,
-    colorMap: [][][]i32,
+    colorMap: [][2]i32,
+    colorMapHeight: usize,
 
-    // allocator must be outside of struct as it will fail the json parser
-    pub fn deinit(self: *const Frame, allocator: Allocator) void {
-        for (self.contents) |con| {
-            allocator.free(con);
-        }
-        allocator.free(self.contents);
+    fn colorAt(self: *const Frame, x: usize, y: usize) [2]i32 {
+        return self.colorMap[x * self.colorMapHeight + y];
+    }
 
-        for (self.colorMap) |cm| {
-            for (cm) |int2| {
-                allocator.free(int2);
+    fn fromJson(
+        allocator: Allocator,
+        reader: *Json.Reader,
+        expected_columns: ?usize,
+        expected_lines: ?usize,
+    ) !Frame {
+        if (.object_begin != try reader.next()) return error.UnexpectedToken;
+
+        var frame_number: ?i32 = null;
+        var delay: ?f32 = null;
+        var contents: ?[][]u8 = null;
+        var color_map: ?[][2]i32 = null;
+        var color_map_height: usize = 0;
+
+        errdefer if (contents) |value| freeContents(allocator, value);
+        errdefer if (color_map) |value| allocator.free(value);
+
+        while (try reader.peekNextTokenType() != .object_end) {
+            const key = try jsonString(allocator, reader);
+            defer key.deinit(allocator);
+
+            if (eql(u8, key.value, "frameNumber")) {
+                if (frame_number != null) return error.DuplicateField;
+                frame_number = try jsonI32(allocator, reader);
+            } else if (eql(u8, key.value, "delay")) {
+                if (delay != null) return error.DuplicateField;
+                delay = try jsonF32(allocator, reader);
+            } else if (eql(u8, key.value, "contents")) {
+                if (contents != null) return error.DuplicateField;
+                contents = try parseContents(allocator, reader);
+            } else if (eql(u8, key.value, "colorMap")) {
+                if (color_map != null) return error.DuplicateField;
+                const parsed = try parseColorMap(
+                    allocator,
+                    reader,
+                    expected_columns,
+                    expected_lines,
+                );
+                color_map = parsed.color_map;
+                color_map_height = parsed.height;
+            } else {
+                try reader.skipValue();
             }
-            allocator.free(cm);
         }
+        _ = try reader.next();
+
+        return .{
+            .frameNumber = frame_number orelse return error.MissingField,
+            .delay = delay orelse return error.MissingField,
+            .contents = contents orelse return error.MissingField,
+            .colorMap = color_map orelse return error.MissingField,
+            .colorMapHeight = color_map_height,
+        };
+    }
+
+    fn parseContents(allocator: Allocator, reader: *Json.Reader) ![][]u8 {
+        if (.array_begin != try reader.next()) return error.UnexpectedToken;
+
+        var contents: std.ArrayList([]u8) = .empty;
+        errdefer {
+            deinitContents(allocator, contents.items);
+            contents.deinit(allocator);
+        }
+
+        while (try reader.peekNextTokenType() != .array_end) {
+            const content = try jsonOwnedString(allocator, reader);
+            errdefer allocator.free(content);
+            try contents.append(allocator, content);
+        }
+        _ = try reader.next();
+
+        return try contents.toOwnedSlice(allocator);
+    }
+
+    const ParsedColorMap = struct {
+        color_map: [][2]i32,
+        height: usize,
+    };
+
+    fn parseColorMap(
+        allocator: Allocator,
+        reader: *Json.Reader,
+        expected_columns: ?usize,
+        expected_lines: ?usize,
+    ) !ParsedColorMap {
+        if (.array_begin != try reader.next()) return error.UnexpectedToken;
+
+        const expected_len = if (expected_columns != null and expected_lines != null)
+            std.math.mul(usize, expected_columns.?, expected_lines.?) catch return error.Overflow
+        else
+            null;
+
+        var color_map: std.ArrayList([2]i32) = if (expected_len) |len|
+            try .initCapacity(allocator, len)
+        else
+            .empty;
+        errdefer color_map.deinit(allocator);
+
+        var color_map_height: ?usize = null;
+        var column_count: usize = 0;
+
+        while (try reader.peekNextTokenType() != .array_end) {
+            if (.array_begin != try reader.next()) return error.UnexpectedToken;
+
+            var column_height: usize = 0;
+            while (try reader.peekNextTokenType() != .array_end) {
+                if (.array_begin != try reader.next()) return error.UnexpectedToken;
+
+                const foreground = try jsonI32(allocator, reader);
+                const background = try jsonI32(allocator, reader);
+
+                if (.array_end != try reader.next()) return error.LengthMismatch;
+
+                try color_map.append(allocator, .{ foreground, background });
+                column_height += 1;
+            }
+            _ = try reader.next();
+
+            if (color_map_height) |height| {
+                if (column_height != height) return error.LengthMismatch;
+            } else {
+                color_map_height = column_height;
+            }
+
+            column_count += 1;
+        }
+        _ = try reader.next();
+
+        if (expected_columns) |columns| {
+            if (column_count != columns) return error.LengthMismatch;
+        }
+
+        return .{
+            .color_map = try color_map.toOwnedSlice(allocator),
+            .height = color_map_height orelse 0,
+        };
+    }
+
+    fn deinitContents(allocator: Allocator, contents: []const []u8) void {
+        for (contents) |content| allocator.free(content);
+    }
+
+    fn freeContents(allocator: Allocator, contents: [][]u8) void {
+        deinitContents(allocator, contents);
+        allocator.free(contents);
+    }
+
+    pub fn deinit(self: *const Frame, allocator: Allocator) void {
+        freeContents(allocator, self.contents);
         allocator.free(self.colorMap);
     }
 };
@@ -51,7 +294,7 @@ const DurFormatRaw = struct {
     framerate: ?f64 = null,
     columns: ?i64 = null,
     lines: ?i64 = null,
-    frames: std.ArrayList(Frame) = undefined,
+    frames: std.ArrayList(Frame) = .empty,
 
     // Validate data and return a valid DurFormat
     // Consumes `self`, making it unusable after
@@ -81,6 +324,7 @@ const DurFormatRaw = struct {
 
         if (self.frames.items.len == 0) return error.NoFrames;
         const frames = self.frames;
+        self.frames = .empty;
 
         return .{
             .allocator = self.allocator,
@@ -94,51 +338,111 @@ const DurFormatRaw = struct {
         };
     }
 
-    fn parseFromJson(self: *DurFormatRaw, allocator: Allocator, dur_json_root: Json.Value) !void {
-        var dur_movie = if (dur_json_root.object.get("DurMovie")) |dm| dm.object else return error.NotValidFile;
+    fn parseFromJson(self: *DurFormatRaw, allocator: Allocator, reader: *Json.Reader) !void {
+        if (.object_begin != try reader.next()) return error.NotValidFile;
+
+        var dur_movie_seen = false;
+        while (try reader.peekNextTokenType() != .object_end) {
+            const key = try jsonString(allocator, reader);
+            defer key.deinit(allocator);
+
+            if (eql(u8, key.value, "DurMovie")) {
+                if (dur_movie_seen) return error.DuplicateField;
+                dur_movie_seen = true;
+                try parseDurMovie(self, allocator, reader);
+            } else {
+                try reader.skipValue();
+            }
+        }
+        _ = try reader.next();
+
+        if (!dur_movie_seen) return error.NotValidFile;
+        if (.end_of_document != try reader.next()) return error.UnexpectedToken;
+    }
+
+    fn parseDurMovie(self: *DurFormatRaw, allocator: Allocator, reader: *Json.Reader) !void {
+        if (.object_begin != try reader.next()) return error.NotValidFile;
 
         // Depending on the version, a dur file can have different json object names (ie: columns vs sizeX)
-        self.formatVersion = if (dur_movie.get("formatVersion")) |x| x.integer else null;
-        self.colorFormat = if (dur_movie.get("colorFormat")) |x| try allocator.dupe(u8, x.string) else null;
-        self.encoding = if (dur_movie.get("encoding")) |x| try allocator.dupe(u8, x.string) else null;
-        self.framerate = if (dur_movie.get("framerate")) |x| x.float else null;
-        self.columns = if (dur_movie.get("columns")) |x| x.integer else if (dur_movie.get("sizeX")) |x| x.integer else null;
+        var format_version_seen = false;
+        var color_format_seen = false;
+        var encoding_seen = false;
+        var framerate_seen = false;
+        var columns_seen = false;
+        var size_x_seen = false;
+        var lines_seen = false;
+        var size_y_seen = false;
+        var frames_seen = false;
 
-        self.lines = if (dur_movie.get("lines")) |x| x.integer else if (dur_movie.get("sizeY")) |x| x.integer else null;
+        while (try reader.peekNextTokenType() != .object_end) {
+            const key = try jsonString(allocator, reader);
+            defer key.deinit(allocator);
 
-        const frames = dur_movie.get("frames") orelse return error.NotValidFile;
-
-        self.frames = try .initCapacity(allocator, frames.array.items.len);
-
-        for (frames.array.items) |json_frame| {
-            var parsed_frame = try Json.parseFromValue(Frame, allocator, json_frame, .{});
-            defer parsed_frame.deinit();
-
-            const frame_val = parsed_frame.value;
-
-            // copy all fields to own the ptrs for deallocation, the parsed_frame has some other
-            // allocated memory making it difficult to deallocate without leaks
-            const frame: Frame = .{ .frameNumber = frame_val.frameNumber, .delay = frame_val.delay, .contents = try allocator.alloc([]u8, frame_val.contents.len), .colorMap = try allocator.alloc([][]i32, frame_val.colorMap.len) };
-
-            for (0..frame.contents.len) |i| {
-                frame.contents[i] = try allocator.dupe(u8, frame_val.contents[i]);
+            if (eql(u8, key.value, "formatVersion")) {
+                if (format_version_seen) return error.DuplicateField;
+                format_version_seen = true;
+                self.formatVersion = try jsonI64(allocator, reader);
+            } else if (eql(u8, key.value, "colorFormat")) {
+                if (color_format_seen) return error.DuplicateField;
+                color_format_seen = true;
+                self.colorFormat = try jsonOwnedString(allocator, reader);
+            } else if (eql(u8, key.value, "encoding")) {
+                if (encoding_seen) return error.DuplicateField;
+                encoding_seen = true;
+                self.encoding = try jsonOwnedString(allocator, reader);
+            } else if (eql(u8, key.value, "framerate")) {
+                if (framerate_seen) return error.DuplicateField;
+                framerate_seen = true;
+                self.framerate = try jsonF64(allocator, reader);
+            } else if (eql(u8, key.value, "columns")) {
+                if (columns_seen) return error.DuplicateField;
+                columns_seen = true;
+                self.columns = try jsonI64(allocator, reader);
+            } else if (eql(u8, key.value, "sizeX")) {
+                if (size_x_seen) return error.DuplicateField;
+                size_x_seen = true;
+                const size_x = try jsonI64(allocator, reader);
+                if (!columns_seen) self.columns = size_x;
+            } else if (eql(u8, key.value, "lines")) {
+                if (lines_seen) return error.DuplicateField;
+                lines_seen = true;
+                self.lines = try jsonI64(allocator, reader);
+            } else if (eql(u8, key.value, "sizeY")) {
+                if (size_y_seen) return error.DuplicateField;
+                size_y_seen = true;
+                const size_y = try jsonI64(allocator, reader);
+                if (!lines_seen) self.lines = size_y;
+            } else if (eql(u8, key.value, "frames")) {
+                if (frames_seen) return error.DuplicateField;
+                frames_seen = true;
+                try parseFrames(self, allocator, reader);
+            } else {
+                try reader.skipValue();
             }
+        }
+        _ = try reader.next();
 
-            // colorMap is stored as an 3d array where:
-            // the outer (i) most array is the horizontal position of the color
-            // the middle (j) is the vertical position of the color
-            // the inner (0/1) is the foreground/background color
-            for (0..frame.colorMap.len) |i| {
-                frame.colorMap[i] = try allocator.alloc([]i32, frame_val.colorMap[i].len);
-                for (0..frame.colorMap[i].len) |j| {
-                    frame.colorMap[i][j] = try allocator.alloc(i32, 2);
-                    frame.colorMap[i][j][0] = frame_val.colorMap[i][j][0];
-                    frame.colorMap[i][j][1] = frame_val.colorMap[i][j][1];
-                }
-            }
+        if (!frames_seen) return error.NotValidFile;
+    }
 
+    fn parseFrames(self: *DurFormatRaw, allocator: Allocator, reader: *Json.Reader) !void {
+        if (.array_begin != try reader.next()) return error.UnexpectedToken;
+
+        const expected_columns = if (self.columns) |columns|
+            std.math.cast(usize, columns)
+        else
+            null;
+        const expected_lines = if (self.lines) |lines|
+            std.math.cast(usize, lines)
+        else
+            null;
+
+        while (try reader.peekNextTokenType() != .array_end) {
+            var frame = try Frame.fromJson(allocator, reader, expected_columns, expected_lines);
+            errdefer frame.deinit(allocator);
             try self.frames.append(allocator, frame);
         }
+        _ = try reader.next();
     }
 
     pub fn createFromFile(self: *DurFormatRaw, allocator: Allocator, io: std.Io, file_path: []const u8) !void {
@@ -154,10 +458,7 @@ const DurFormatRaw = struct {
         var json_reader = Json.Reader.init(allocator, &decompress.reader);
         defer json_reader.deinit();
 
-        const json = try Json.parseFromTokenSource(Json.Value, allocator, &json_reader, .{});
-        defer json.deinit();
-
-        try parseFromJson(self, allocator, json.value);
+        try parseFromJson(self, allocator, &json_reader);
     }
 
     pub fn init(allocator: Allocator) DurFormatRaw {
@@ -167,6 +468,8 @@ const DurFormatRaw = struct {
     pub fn deinit(self: *DurFormatRaw) void {
         if (self.colorFormat) |str| self.allocator.free(str);
         if (self.encoding) |str| self.allocator.free(str);
+        for (self.frames.items) |frame| frame.deinit(self.allocator);
+        self.frames.deinit(self.allocator);
     }
 };
 
@@ -520,7 +823,7 @@ fn draw(self: *DurFile) void {
             const cell_x = @as(i32, @intCast(x)) + self.start_pos[VEC_X];
 
             const codepoint: u21 = iter.nextCodepoint().?;
-            const color_map = current_frame.colorMap[x][y];
+            const color_map = current_frame.colorAt(x, y);
 
             var color_map_0: u32 = @intCast(if (color_map[0] == -1) 0 else color_map[0]);
             var color_map_1: u32 = @intCast(if (color_map[1] == -1) 0 else color_map[1]);