def tokenize
@output = []
while !@ss.eos?
@ss.skip(/\s*/)
tok = case
when t = @ss.scan(COMPARISON_OPERATOR) then [:comparison, t]
when t = @ss.scan(SINGLE_STRING_LITERAL) then [:string, t]
when t = @ss.scan(DOUBLE_STRING_LITERAL) then [:string, t]
when t = @ss.scan(NUMBER_LITERAL) then [:number, t]
when t = @ss.scan(IDENTIFIER) then [:id, t]
when t = @ss.scan(DOTDOT) then [:dotdot, t]
else
c = @ss.getch
if s = SPECIALS[c]
[s,c]
else
raise SyntaxError, "Unexpected character #{c}"
end
end
@output << tok
end
@output << [:end_of_string]
end