# File lib/rubypants.rb, line 403
  def tokenize
    tag_soup = /([^<]*)(<!--.*?-->|<[^>]*>)/m

    tokens = []

    prev_end = 0

    scan(tag_soup) do
      tokens << [:text, $1]  if $1 != ""
      tokens << [:tag, $2]
      prev_end = $~.end(0)
    end

    if prev_end < size
      tokens << [:text, self[prev_end..-1]]
    end

    tokens
  end