Feature: Keywords API

.rubocop.yml

@@ -100,6 +100,9 @@ Lint/NestedPercentLiteral:
 Lint/DuplicateBranch:
   Enabled: false
 
+Lint/NonLocalExitFromIterator:
+  Enabled: false
+
 Naming/BlockForwarding:
   Enabled: false
 

.rubocop_todo.yml

@@ -6,104 +6,24 @@
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 1
-# Configuration parameters: AllowComments, AllowEmptyLambdas.
-Lint/EmptyBlock:
-  Exclude:
-    - 'lib/rouge/lexers/nix.rb'
-
-# Offense count: 4
-Naming/ConstantName:
-  Exclude:
-    - 'lib/rouge/lexers/eiffel.rb'
-
-# Offense count: 5
-# This cop supports unsafe autocorrection (--autocorrect-all).
-# Configuration parameters: EnforcedStyleForLeadingUnderscores.
-# SupportedStylesForLeadingUnderscores: disallowed, required, optional
-Naming/MemoizedInstanceVariableName:
-  Exclude:
-    - 'lib/rouge/lexers/freefem.rb'
-    - 'lib/rouge/lexers/verilog.rb'
-
 # Offense count: 16
 # Configuration parameters: EnforcedStyle, AllowedPatterns, ForbiddenIdentifiers, ForbiddenPatterns.
 # SupportedStyles: snake_case, camelCase
 # ForbiddenIdentifiers: __id__, __send__
 Naming/MethodName:
   Exclude:
-    - 'lib/rouge/lexers/igorpro.rb'
     - 'lib/rouge/lexers/xpath.rb'
 
 # Offense count: 40
 # Configuration parameters: EnforcedStyle, AllowedIdentifiers, AllowedPatterns, ForbiddenIdentifiers, ForbiddenPatterns.
 # SupportedStyles: snake_case, camelCase
 Naming/VariableName:
   Exclude:
-    - 'lib/rouge/lexers/brightscript.rb'
-    - 'lib/rouge/lexers/dafny.rb'
-    - 'lib/rouge/lexers/freefem.rb'
-    - 'lib/rouge/lexers/igorpro.rb'
-    - 'lib/rouge/lexers/kotlin.rb'
     - 'lib/rouge/lexers/xpath.rb'
 
 # Offense count: 113
 Rouge/NoBuildingAlternationPatternInRegexp:
   Exclude:
-    - 'lib/rouge/lexers/apple_script.rb'
-    - 'lib/rouge/lexers/armasm.rb'
-    - 'lib/rouge/lexers/bbcbasic.rb'
-    - 'lib/rouge/lexers/bicep.rb'
-    - 'lib/rouge/lexers/cfscript.rb'
-    - 'lib/rouge/lexers/cmhg.rb'
-    - 'lib/rouge/lexers/crystal.rb'
-    - 'lib/rouge/lexers/csharp.rb'
-    - 'lib/rouge/lexers/d.rb'
-    - 'lib/rouge/lexers/dart.rb'
-    - 'lib/rouge/lexers/eiffel.rb'
-    - 'lib/rouge/lexers/elm.rb'
-    - 'lib/rouge/lexers/erlang.rb'
-    - 'lib/rouge/lexers/haskell.rb'
-    - 'lib/rouge/lexers/http.rb'
-    - 'lib/rouge/lexers/idris.rb'
-    - 'lib/rouge/lexers/isabelle.rb'
-    - 'lib/rouge/lexers/java.rb'
-    - 'lib/rouge/lexers/jsp.rb'
-    - 'lib/rouge/lexers/kotlin.rb'
-    - 'lib/rouge/lexers/lean.rb'
-    - 'lib/rouge/lexers/magik.rb'
-    - 'lib/rouge/lexers/make.rb'
-    - 'lib/rouge/lexers/mason.rb'
-    - 'lib/rouge/lexers/mosel.rb'
-    - 'lib/rouge/lexers/nix.rb'
-    - 'lib/rouge/lexers/p4.rb'
-    - 'lib/rouge/lexers/pascal.rb'
-    - 'lib/rouge/lexers/postscript.rb'
-    - 'lib/rouge/lexers/r.rb'
-    - 'lib/rouge/lexers/rust.rb'
-    - 'lib/rouge/lexers/scala.rb'
-    - 'lib/rouge/lexers/sml.rb'
-    - 'lib/rouge/lexers/stata.rb'
     - 'lib/rouge/lexers/vala.rb'
     - 'lib/rouge/lexers/wollok.rb'
     - 'lib/rouge/lexers/xojo.rb'
-
-# Offense count: 20
-Rouge/NoHugeCollections:
-  Exclude:
-    - 'lib/rouge/lexers/apple_script.rb'
-    - 'lib/rouge/lexers/cobol.rb'
-    - 'lib/rouge/lexers/common_lisp.rb'
-    - 'lib/rouge/lexers/css.rb'
-    - 'lib/rouge/lexers/datastudio.rb'
-    - 'lib/rouge/lexers/freefem.rb'
-    - 'lib/rouge/lexers/hql.rb'
-    - 'lib/rouge/lexers/idlang.rb'
-    - 'lib/rouge/lexers/igorpro.rb'
-    - 'lib/rouge/lexers/janet.rb'
-    - 'lib/rouge/lexers/openedge.rb'
-    - 'lib/rouge/lexers/plsql.rb'
-    - 'lib/rouge/lexers/racket.rb'
-    - 'lib/rouge/lexers/sas.rb'
-    - 'lib/rouge/lexers/sql.rb'
-    - 'lib/rouge/lexers/stata.rb'

Gemfile

@@ -8,6 +8,7 @@ gem 'rake'
 
 gem 'minitest', '>= 5.0'
 gem 'minitest-power_assert'
+gem 'minitest-focus'
 gem 'power_assert', '~> 2.0'
 
 # don't try to install redcarpet under jruby

docs/LexerDevelopment.md

@@ -267,6 +267,93 @@ rules, as if the regular expression had not matched.
 
 You can see an example of these more complex rules in [the Ruby lexer][ruby-lexer].
 
+#### Keywords, Builtins, and other word sets
+
+Most programming languages reserve certain words for use as identifiers that
+have a special meaning in the language. To make regular expressions that search
+for these words easier, many lexers will put the applicable keywords in a
+set and make them available in a particular way (be it as a local variable,
+an instance variable or what have you).
+
+For small sets, you can simply use a constant:
+
+```rb
+module Rouge
+  module Lexers
+    class YetAnotherLanguage < RegexLexer
+      # ...
+      KEYWORDS = Set.new %w(key words used in this language)
+      # ...
+    end
+  end
+end
+```
+
+If the keyword sets are very large (>75 elements or so), please put them in a constant in a separate file, which is lazily loaded:
+
+```rb
+# lib/rouge/lexers/my_lang.rb
+module Rouge
+  module Lexers
+    class MyLang < RegexLexer
+      tag :my_lang
+      # ...
+
+      lazy do
+        requre_relative 'my_lang/keywords'
+      end
+
+      # ...
+    end
+  end
+end
+
+# lib/rouge/lexers/my_lang/keywords.rb
+module Rouge
+  module Lexers
+    class MyLang
+      KEYWORDS = Set.new %w(massive set goes here)
+    end
+  end
+end
+```
+
+This way, users of Rouge who are not using your language will not have to load the keyword sets. These keywords can then be used with the special `#keywords` api:
+
+```rb
+state :my_cool_state do
+  # ...
+
+  # Use a "covering regex" here, like /\w+/, which matches the general form
+  # of keywords. This should be possible for almost all languages. If there is no
+  # appropriate regex that will cover all cases, consider special-casing some of
+  # the exceptions. If the language has *truly* free-form keyword sets (like Gherkin),
+  # please let us know and we'll consider allowing a large regex instead.
+  keywords %r/\w+/ do
+    # this method acts much like a normal #rule call - it takes a token type, and
+    # potentially an action block. Except instead of a regex, it takes a Set, which
+    # the match from above will be checked against, in the order they appear here.
+    rule KEYWORDS, Keyword
+    rule BUILTINS, Name::Builtin, :pop!
+
+    # symbols will be mapped to class methods on the lexer
+    rule :small_set, Name::Function
+
+    # or you can just inline small sets here
+    rule Set[%w(one two three four)], Num::Integer
+    # etc...
+
+    # optional: transform the match before checking set membership
+    transform(&:downcase)
+
+    # optional: a default rule if the match isn't contained in any of the sets.
+    # If this is not given, Rouge will simply fall through to the next rules after
+    # the keywords block.
+    default Name
+  end
+end
+```
+
 ### Additional Features
 
 While the properties and states are the minimum elements of a lexer that need
@@ -303,57 +390,6 @@ Filename Globs][conflict-globs] below.
 
 [conflict-globs]: #conflicting-filename-globs
 
-#### Special Words
-
-Most programming languages reserve certain words for use as identifiers that
-have a special meaning in the language. To make regular expressions that search
-for these words easier, many lexers will put the applicable keywords in a
-set and make them available in a particular way (be it as a local variable,
-an instance variable or what have you).
-
-For performance and safety, we strongly recommend lexers use a class method:
-
-```rb
-module Rouge
-  module Lexers
-    class YetAnotherLanguage < RegexLexer
-      ...
-
-      def self.keywords
-        @keywords ||= Set.new %w(key words used in this language)
-      end
-
-      ...
-    end
-  end
-end
-```
-
-These keywords can then be used like so:
-
-```rb
-rule /\w+/ do |m|
-  if self.class.keywords.include?(m[0])
-    token Keyword
-  elsif
-    token Name
-  end
-end
-```
-
-In some cases, you may want to interpolate your keywords into a regular
-expression. **We strongly recommend you avoid doing this.** Having a large
-number of rules that are searching for particular words is not as performant as
-a rule with a generic pattern with a block that checks whether the pattern is a
-member of a predefined set and assigns tokens, pushes new states, etc.
-
-If you do need to use interpolation, be careful to use the `\b` anchor to avoid
-inadvertently matching part of a longer word (eg. `if` matching `iff`)::
-
-```rb
-rule /\b(#{keywords.join('|')})\b/, Keyword
-```
-
 #### Startup
 
 ```rb

lib/rouge/lexer.rb

@@ -237,10 +237,10 @@ def eager_load!
         return if @_loaded
         @_loaded = true
 
-        lazy_procs.each { |b| instance_eval(&b) }
-
         superclass.eager_load! unless superclass == Lexer
 
+        lazy_procs.each { |b| instance_eval(&b) }
+
         self
       end
 

lib/rouge/lexers/actionscript.rb

@@ -98,7 +98,7 @@ def self.constants
       end
 
       def self.builtins
-        @builtins ||= %w(
+        @builtins ||= Set.new %w(
           void Function Math Class
           Object RegExp decodeURI
           decodeURIComponent encodeURI encodeURIComponent
@@ -129,22 +129,12 @@ def self.builtins
 
         rule %r/[{}]/, Punctuation, :statement
 
-        rule id do |m|
-          if self.class.keywords.include? m[0]
-            token Keyword
-            push :expr_start
-          elsif self.class.declarations.include? m[0]
-            token Keyword::Declaration
-            push :expr_start
-          elsif self.class.reserved.include? m[0]
-            token Keyword::Reserved
-          elsif self.class.constants.include? m[0]
-            token Keyword::Constant
-          elsif self.class.builtins.include? m[0]
-            token Name::Builtin
-          else
-            token Name::Other
-          end
+        keywords id do
+          rule :keywords, Keyword
+          rule :declarations, Keyword::Declaration
+          rule :reserved, Keyword::Reserved
+          rule :builtins, Name::Builtin
+          default Name::Other
         end
 
         rule %r/\-?[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?/, Num::Float