A simple implementation of an YAML lexer. It handles most cases. It is not a validating lexer.

Methods
Public Instance methods
step()

Step through a single iteration of the tokenization process. This will yield (potentially) many tokens, and possibly zero tokens.

    # File lib/syntax/lang/yaml.rb, line 11
11:     def step
12:       if bol?
13:         case
14:           when scan(/---(\s*.+)?$/)
15:             start_group :document, matched
16:           when scan(/(\s*)([a-zA-Z][-\w]*)(\s*):/)
17:             start_group :normal, subgroup(1)
18:             start_group :key, subgroup(2)
19:             start_group :normal, subgroup(3)
20:             start_group :punct, ":"
21:           when scan(/(\s*)-/)
22:             start_group :normal, subgroup(1)
23:             start_group :punct, "-"
24:           when scan(/\s*$/)
25:             start_group :normal, matched
26:           when scan(/#.*$/)
27:             start_group :comment, matched
28:           else
29:             append getch
30:         end
31:       else
32:         case
33:           when scan(/[\n\r]+/)
34:             start_group :normal, matched
35:           when scan(/[ \t]+/)
36:             start_group :normal, matched
37:           when scan(/!+(.*?^)?\S+/)
38:             start_group :type, matched
39:           when scan(/&\S+/)
40:             start_group :anchor, matched
41:           when scan(/\*\S+/)
42:             start_group :ref, matched
43:           when scan(/\d\d:\d\d:\d\d/)
44:             start_group :time, matched
45:           when scan(/\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d(\.\d+)? [-+]\d\d:\d\d/)
46:             start_group :date, matched
47:           when scan(/['"]/)
48:             start_group :punct, matched
49:             scan_string matched
50:           when scan(/:\w+/)
51:             start_group :symbol, matched
52:           when scan(/[:]/)
53:             start_group :punct, matched
54:           when scan(/#.*$/)
55:             start_group :comment, matched
56:           when scan(/>-?/)
57:             start_group :punct, matched
58:             start_group :normal, scan(/.*$/)
59:             append getch until eos? || bol?
60:             return if eos?
61:             indent = check(/ */)
62:             start_group :string
63:             loop do
64:               line = check_until(/[\n\r]|\Z/)
65:               break if line.nil?
66:               if line.chomp.length > 0
67:                 this_indent = line.chomp.match( /^\s*/ )[0]
68:                 break if this_indent.length < indent.length
69:               end
70:               append scan_until(/[\n\r]|\Z/)
71:             end
72:           else
73:             start_group :normal, scan_until(/(?=$|#)/)
74:         end
75:       end
76:     end