# File lib/rubypants.rb, line 403 def tokenize tag_soup = /([^<]*)(<!--.*?-->|<[^>]*>)/m tokens = [] prev_end = 0 scan(tag_soup) do tokens << [:text, $1] if $1 != "" tokens << [:tag, $2] prev_end = $~.end(0) end if prev_end < size tokens << [:text, self[prev_end..-1]] end tokens end