const std = @import("std"); const Allocator = std.mem.Allocator; const ascii = std.ascii; const isDigit = ascii.isDigit; const isAlphabetic = ascii.isAlphabetic; const Token = @import("Token.zig"); const Literal = Token.Literal; const Scanner = @This(); const TokenType = @import("token_type.zig").TokenType; const lox = @import("lox.zig"); source: []const u8, tokens: std.ArrayList(Token) = .empty, start: u32 = 0, current: u32 = 0, line: u32 = 1, const keyword: std.StaticStringMap(TokenType) = .initComptime(.{ .{ "and", .@"and" }, .{ "class", .class }, .{ "else", .@"else" }, .{ "false", .false }, .{ "for", .@"for" }, .{ "fun", .fun }, .{ "if", .@"if" }, .{ "nil", .nil }, .{ "or", .@"or" }, .{ "print", .print }, .{ "return", .@"return" }, .{ "super", .super }, .{ "this", .this }, .{ "true", .true }, .{ "var", .@"var" }, .{ "while", .@"while" }, }); pub fn init(source: []const u8) Scanner { return .{ .source = source, }; } pub fn scanTokens(self: *Scanner, allocator: Allocator) ![]Token { while (!isAtEnd(self)) { // We are at the beginning of the next lexeme. self.start = self.current; try self.scanToken(allocator); } try self.tokens.append(allocator, .init(.eof, "", null, self.line)); return try self.tokens.toOwnedSlice(allocator); } fn scanToken(self: *Scanner, allocator: Allocator) !void { const c = self.advance(); switch (c) { '(' => try self.addToken(allocator, .left_paren, null), ')' => try self.addToken(allocator, .right_paren, null), '{' => try self.addToken(allocator, .left_brace, null), '}' => try self.addToken(allocator, .right_brace, null), ',' => try self.addToken(allocator, .comma, null), '.' => try self.addToken(allocator, .dot, null), '-' => try self.addToken(allocator, .minus, null), '+' => try self.addToken(allocator, .plus, null), ';' => try self.addToken(allocator, .semicolon, null), '*' => try self.addToken(allocator, .star, null), '!' => try self.addToken(allocator, if (self.match('=')) .bang_equal else .bang, null), '=' => try self.addToken(allocator, if (self.match('=')) .equal_equal else .equal, null), '<' => try self.addToken(allocator, if (self.match('=')) .less_equal else .less, null), '>' => try self.addToken(allocator, if (self.match('=')) .greater_equal else .greater, null), '/' => if (self.match('/')) { while (self.peek() != '\n' and !self.isAtEnd()) _ = self.advance(); } else { try self.addToken(allocator, .slash, null); }, ' ', '\r', '\t' => {}, '\n' => self.line += 1, '"' => try self.string(allocator), else => if (isDigit(c)) { try self.number(allocator); } else if (isAlpha(c)) { try self.identifier(allocator); } else { try lox.@"error"(self.line, "Unexpected character."); }, } } fn identifier(self: *Scanner, allocator: Allocator) !void { while (isAlphanumeric(self.peek())) _ = self.advance(); const text = self.source[self.start..self.current]; const @"type" = keyword.get(text) orelse .identifier; try self.addToken(allocator, @"type", null); } fn number(self: *Scanner, allocator: Allocator) !void { while (isDigit(self.peek())) _ = self.advance(); // Look for a fractional part. if (self.peek() == '.' and isDigit(self.peekNext())) { // Consume the "." _ = self.advance(); while (isDigit(self.peek())) _ = self.advance(); } try self.addToken(allocator, .number, .{ .number = std.fmt.parseFloat(f64, self.source[self.start..self.current]) catch unreachable }); } fn string(self: *Scanner, allocator: Allocator) !void { while (self.peek() != '"' and !self.isAtEnd()) { if (self.peek() == '\n') self.line += 1; _ = self.advance(); } if (self.isAtEnd()) { try lox.@"error"(self.line, "Unterminated string."); return; } // The closing ". _ = self.advance(); const value = self.source[self.start + 1 .. self.current - 1]; try self.addToken(allocator, .string, .{ .string = value }); } fn match(self: *Scanner, expected: u8) bool { if (self.isAtEnd()) return false; if (self.source[self.current] != expected) return false; self.current += 1; return true; } fn peek(self: *Scanner) u8 { if (self.isAtEnd()) return 0; return self.source[self.current]; } fn peekNext(self: *Scanner) u8 { if (self.current + 1 >= self.source.len) return 0; return self.source[self.current + 1]; } fn isAlpha(c: u8) bool { return isAlphabetic(c) or c == '_'; } fn isAlphanumeric(c: u8) bool { return isAlpha(c) or isDigit(c); } fn isAtEnd(self: *Scanner) bool { return self.current >= self.source.len; } fn advance(self: *Scanner) u8 { defer self.current += 1; return self.source[self.current]; } fn addToken(self: *Scanner, allocator: Allocator, @"type": TokenType, literal: ?Literal) !void { const text = self.source[self.start..self.current]; try self.tokens.append(allocator, .init(@"type", text, literal, self.line)); }